| 1 | //===- NVGPUTransformOps.cpp - Implementation of NVGPU transform ops ------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.h" |
| 10 | |
| 11 | #include "mlir/Analysis/SliceAnalysis.h" |
| 12 | #include "mlir/Conversion/GPUCommon/GPUCommonPass.h" |
| 13 | #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
| 14 | #include "mlir/Conversion/NVGPUToNVVM/NVGPUToNVVM.h" |
| 15 | #include "mlir/Dialect/Affine/IR/AffineOps.h" |
| 16 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 17 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 18 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 20 | #include "mlir/Dialect/LLVMIR/NVVMDialectDecl.h" |
| 21 | #include "mlir/Dialect/Linalg/IR/Linalg.h" |
| 22 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 23 | #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h" |
| 24 | #include "mlir/Dialect/NVGPU/Transforms/Transforms.h" |
| 25 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 26 | #include "mlir/Dialect/SCF/Transforms/Transforms.h" |
| 27 | #include "mlir/Dialect/Utils/IndexingUtils.h" |
| 28 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 29 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 30 | #include "mlir/IR/AffineExpr.h" |
| 31 | #include "mlir/IR/BuiltinTypes.h" |
| 32 | #include "mlir/IR/Value.h" |
| 33 | #include "llvm/ADT/ArrayRef.h" |
| 34 | |
| 35 | using namespace mlir; |
| 36 | using namespace mlir::linalg; |
| 37 | using namespace mlir::nvgpu; |
| 38 | using namespace mlir::NVVM; |
| 39 | using namespace mlir::transform; |
| 40 | |
| 41 | #define DEBUG_TYPE "nvgpu-transforms" |
| 42 | |
| 43 | //===----------------------------------------------------------------------===// |
| 44 | // Apply...ConversionPatternsOp |
| 45 | //===----------------------------------------------------------------------===// |
| 46 | |
| 47 | void ApplyNVGPUToNVVMConversionPatternsOp::populatePatterns( |
| 48 | TypeConverter &typeConverter, RewritePatternSet &patterns) { |
| 49 | auto &llvmTypeConverter = static_cast<LLVMTypeConverter &>(typeConverter); |
| 50 | /// device-side async tokens cannot be materialized in nvvm. We just |
| 51 | /// convert them to a dummy i32 type in order to easily drop them during |
| 52 | /// conversion. |
| 53 | nvgpu::populateCommonGPUTypeAndAttributeConversions(typeConverter&: llvmTypeConverter); |
| 54 | llvmTypeConverter.addConversion(callback: [&](DeviceAsyncTokenType type) -> Type { |
| 55 | return llvmTypeConverter.convertType( |
| 56 | t: IntegerType::get(context: type.getContext(), width: 32)); |
| 57 | }); |
| 58 | llvmTypeConverter.addConversion(callback: [&](MBarrierTokenType type) -> Type { |
| 59 | return llvmTypeConverter.convertType( |
| 60 | t: IntegerType::get(context: type.getContext(), width: 64)); |
| 61 | }); |
| 62 | llvmTypeConverter.addConversion(callback: [&](WarpgroupAccumulatorType type) -> Type { |
| 63 | Type elemType = type.getFragmented().getElementType(); |
| 64 | int64_t sizeM = type.getFragmented().getDimSize(idx: 0); |
| 65 | int64_t sizeN = type.getFragmented().getDimSize(idx: 1); |
| 66 | |
| 67 | unsigned numMembers; |
| 68 | if (elemType.isF32() || elemType.isInteger(width: 32)) |
| 69 | numMembers = sizeN / 2; |
| 70 | else if (elemType.isF16()) |
| 71 | numMembers = sizeN / 4; |
| 72 | else |
| 73 | llvm_unreachable("unsupported type for warpgroup accumulator" ); |
| 74 | |
| 75 | SmallVector<Type> innerStructBody; |
| 76 | for (unsigned i = 0; i < numMembers; i++) |
| 77 | innerStructBody.push_back(Elt: elemType); |
| 78 | auto innerStructType = |
| 79 | LLVM::LLVMStructType::getLiteral(context: type.getContext(), types: innerStructBody); |
| 80 | |
| 81 | SmallVector<Type> structBody; |
| 82 | for (int i = 0; i < sizeM; i += kWgmmaSizeM) |
| 83 | structBody.push_back(Elt: innerStructType); |
| 84 | |
| 85 | auto convertedType = |
| 86 | LLVM::LLVMStructType::getLiteral(context: type.getContext(), types: structBody); |
| 87 | return llvmTypeConverter.convertType(t: convertedType); |
| 88 | }); |
| 89 | llvmTypeConverter.addConversion(callback: [&](MBarrierGroupType type) -> Type { |
| 90 | return llvmTypeConverter.convertType( |
| 91 | t: getMBarrierMemrefType(context: type.getContext(), barrierType: type)); |
| 92 | }); |
| 93 | llvmTypeConverter.addConversion( |
| 94 | callback: [&](WarpgroupMatrixDescriptorType type) -> Type { |
| 95 | return llvmTypeConverter.convertType( |
| 96 | t: IntegerType::get(context: type.getContext(), width: 64)); |
| 97 | }); |
| 98 | llvmTypeConverter.addConversion(callback: [&](TensorMapDescriptorType type) -> Type { |
| 99 | return LLVM::LLVMPointerType::get(context: type.getContext()); |
| 100 | }); |
| 101 | populateNVGPUToNVVMConversionPatterns(converter: llvmTypeConverter, patterns); |
| 102 | } |
| 103 | |
| 104 | LogicalResult ApplyNVGPUToNVVMConversionPatternsOp::verifyTypeConverter( |
| 105 | TypeConverterBuilderOpInterface builder) { |
| 106 | if (builder.getTypeConverterType() != "LLVMTypeConverter" ) |
| 107 | return emitOpError(message: "expected LLVMTypeConverter" ); |
| 108 | return success(); |
| 109 | } |
| 110 | |
| 111 | //===---------------------------------------------------------------------===// |
| 112 | // CreateAsyncGroupsOp |
| 113 | //===---------------------------------------------------------------------===// |
| 114 | |
| 115 | void CreateAsyncGroupsOp::getEffects( |
| 116 | SmallVectorImpl<MemoryEffects::EffectInstance> &effects) { |
| 117 | consumesHandle(handles: getTargetMutable(), effects); |
| 118 | producesHandle(handles: getOperation()->getOpResults(), effects); |
| 119 | modifiesPayload(effects); |
| 120 | } |
| 121 | |
| 122 | DiagnosedSilenceableFailure |
| 123 | CreateAsyncGroupsOp::applyToOne(TransformRewriter &rewriter, Operation *target, |
| 124 | ApplyToEachResultList &results, |
| 125 | TransformState &state) { |
| 126 | createAsyncGroups(rewriter, op: target, bypassL1: getBypassL1()); |
| 127 | results.push_back(op: target); |
| 128 | return DiagnosedSilenceableFailure::success(); |
| 129 | } |
| 130 | |
| 131 | //===----------------------------------------------------------------------===// |
| 132 | // PipelineSharedMemoryCopiesOp |
| 133 | //===----------------------------------------------------------------------===// |
| 134 | |
| 135 | /// Returns true if the given type has the default memory space. |
| 136 | static bool hasDefaultMemorySpace(BaseMemRefType type) { |
| 137 | return !type.getMemorySpace() || type.getMemorySpaceAsInt() == 0; |
| 138 | } |
| 139 | |
| 140 | /// Returns true if the given type has the shared (workgroup) memory space. |
| 141 | static bool hasSharedMemorySpace(BaseMemRefType type) { |
| 142 | auto space = |
| 143 | dyn_cast_if_present<gpu::AddressSpaceAttr>(Val: type.getMemorySpace()); |
| 144 | return space && |
| 145 | space.getValue() == gpu::GPUDialect::getWorkgroupAddressSpace(); |
| 146 | } |
| 147 | |
| 148 | /// Returns the value produced by a load from the default memory space. Returns |
| 149 | /// null if the operation is not such a load. |
| 150 | static Value getValueLoadedFromGlobal(Operation *op) { |
| 151 | // TODO: consider an interface or leveraging the memory effects interface. |
| 152 | auto load = dyn_cast<vector::TransferReadOp>(Val: op); |
| 153 | if (!load) |
| 154 | return nullptr; |
| 155 | |
| 156 | auto loadType = dyn_cast<MemRefType>(Val: load.getBase().getType()); |
| 157 | if (!loadType || !hasDefaultMemorySpace(type: loadType)) |
| 158 | return nullptr; |
| 159 | return load; |
| 160 | } |
| 161 | |
| 162 | /// Returns true if the operation is storing the given value into shared memory. |
| 163 | static bool isStoreToShared(Operation *op, Value v) { |
| 164 | // TOD: consider an interface or leveraging the memory effects interface. |
| 165 | auto store = dyn_cast<vector::TransferWriteOp>(Val: op); |
| 166 | if (!store || store.getVector() != v) |
| 167 | return false; |
| 168 | |
| 169 | auto storeType = dyn_cast<MemRefType>(Val: store.getBase().getType()); |
| 170 | return storeType || hasSharedMemorySpace(type: storeType); |
| 171 | } |
| 172 | |
| 173 | /// Returns true if the operation is a load from the default memory space the |
| 174 | /// result of which is only stored into the shared memory space. |
| 175 | static bool isLoadFromGlobalStoredToShared(Operation *op) { |
| 176 | Value loaded = getValueLoadedFromGlobal(op); |
| 177 | if (!loaded || !loaded.hasOneUse()) |
| 178 | return false; |
| 179 | |
| 180 | return isStoreToShared(op: *loaded.getUsers().begin(), v: loaded); |
| 181 | } |
| 182 | |
| 183 | /// Populate `ops` with the set of operations that belong to the stage 0 of the |
| 184 | /// pipelined version of the given loop when pipelining copies to shared memory. |
| 185 | /// Specifically, this collects: |
| 186 | /// |
| 187 | /// 1. all loads from global memory, both sync and async; |
| 188 | /// 2. the barriers for async loads. |
| 189 | /// |
| 190 | /// In particular, barriers are omitted if they do not dominate at least one |
| 191 | /// async load for which there is not yet a barrier. |
| 192 | static LogicalResult |
| 193 | collectStage0PipeliningOps(scf::ForOp forOp, |
| 194 | llvm::SmallPtrSet<Operation *, 16> &ops) { |
| 195 | |
| 196 | llvm::SmallPtrSet<Operation *, 4> barriers; |
| 197 | for (Operation &op : *forOp.getBody()) { |
| 198 | // Bail on nested ops for now. |
| 199 | if (op.getNumRegions() > 0) |
| 200 | return failure(); |
| 201 | |
| 202 | if (isa<gpu::BarrierOp>(Val: op)) { |
| 203 | barriers.insert(Ptr: &op); |
| 204 | continue; |
| 205 | } |
| 206 | |
| 207 | if (isa<DeviceAsyncCopyOp, DeviceAsyncCreateGroupOp>(Val: op)) { |
| 208 | ops.insert(Ptr: &op); |
| 209 | ops.insert(I: std::make_move_iterator(i: barriers.begin()), |
| 210 | E: std::make_move_iterator(i: barriers.end())); |
| 211 | assert(barriers.empty() && |
| 212 | "expected to have moved the barriers into another set" ); |
| 213 | continue; |
| 214 | } |
| 215 | |
| 216 | if (isLoadFromGlobalStoredToShared(op: &op)) { |
| 217 | ops.insert(Ptr: &op); |
| 218 | continue; |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | return success(); |
| 223 | } |
| 224 | |
| 225 | /// Hook for the loop pipeliner that sets the "num groups in flight" attribute |
| 226 | /// of async wait operations corresponding to pipelined shared memory copies. |
| 227 | // TODO: this currently assumes that there are no groups that could be in flight |
| 228 | // in the existing code. |
| 229 | static void |
| 230 | setAsyncWaitGroupsInFlight(OpBuilder &builder, Operation *op, |
| 231 | scf::PipeliningOption::PipelinerPart part, |
| 232 | unsigned iteration, unsigned depth) { |
| 233 | // Based on the order of copies within the loop we need to set the number |
| 234 | // of copies in flight, unless it is already set. |
| 235 | auto waitOp = dyn_cast<DeviceAsyncWaitOp>(Val: op); |
| 236 | if (!waitOp || waitOp.getNumGroups()) |
| 237 | return; |
| 238 | |
| 239 | int numGroupInFlight = 0; |
| 240 | if (part == scf::PipeliningOption::PipelinerPart::Kernel || |
| 241 | part == scf::PipeliningOption::PipelinerPart::Prologue) { |
| 242 | numGroupInFlight = depth - 1; |
| 243 | } else { |
| 244 | // By construction there should be no wait op in the prologue as all the |
| 245 | // wait should be in the last stage. |
| 246 | assert(part == scf::PipeliningOption::PipelinerPart::Epilogue); |
| 247 | // Based on the schedule we pick we know how many groups are in flight for |
| 248 | // each iteration of the epilogue. |
| 249 | numGroupInFlight = depth - 1 - iteration; |
| 250 | } |
| 251 | waitOp.setNumGroups(numGroupInFlight); |
| 252 | } |
| 253 | |
| 254 | /// Hook for the loop pipeliner that populates `ops` with the stage information |
| 255 | /// as follows: |
| 256 | /// |
| 257 | /// - operations in `stage0Ops` (typically loads from global memory and |
| 258 | /// related barriers) are at stage 0; |
| 259 | /// - operations in the backward slice of any stage0Ops are all at stage 0; |
| 260 | /// - other operations are at stage `depth`; |
| 261 | /// - the internal order of the pipelined loop has ops at stage `depth` first, |
| 262 | /// then those at stage 0, with relative order within each group preserved. |
| 263 | /// |
| 264 | static void getPipelineStages( |
| 265 | scf::ForOp forOp, |
| 266 | std::vector<std::pair<Operation *, unsigned>> &opsWithPipelineStages, |
| 267 | unsigned depth, llvm::SmallPtrSetImpl<Operation *> &stage0Ops) { |
| 268 | SetVector<Operation *> dependencies; |
| 269 | BackwardSliceOptions options([&](Operation *visited) { |
| 270 | return visited->getBlock() == forOp.getBody(); |
| 271 | }); |
| 272 | options.inclusive = true; |
| 273 | for (Operation &op : forOp.getBody()->getOperations()) { |
| 274 | if (stage0Ops.contains(Ptr: &op)) { |
| 275 | LogicalResult result = getBackwardSlice(op: &op, backwardSlice: &dependencies, options); |
| 276 | assert(result.succeeded() && "expected a backward slice" ); |
| 277 | (void)result; |
| 278 | } |
| 279 | } |
| 280 | |
| 281 | for (Operation &op : forOp.getBody()->getOperations()) { |
| 282 | if (!dependencies.contains(key: &op) && !isa<scf::YieldOp>(Val: op)) |
| 283 | opsWithPipelineStages.emplace_back(args: &op, args&: depth); |
| 284 | } |
| 285 | for (Operation &op : forOp.getBody()->getOperations()) { |
| 286 | if (dependencies.contains(key: &op)) |
| 287 | opsWithPipelineStages.emplace_back(args: &op, args: 0); |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | /// Hook for the loop pipeliner. Replaces op with a predicated version and |
| 292 | /// returns the resulting operation. Returns the original op if the predication |
| 293 | /// isn't necessary for the given op. Returns null if predication is needed but |
| 294 | /// not supported. |
| 295 | static Operation *replaceOpWithPredicatedOp(RewriterBase &rewriter, |
| 296 | Operation *op, Value predicate) { |
| 297 | // Some operations may be fine to execute "speculatively" more times than the |
| 298 | // original number of iterations, in particular side-effect free operations |
| 299 | // and barriers, even if they cannot be predicated. |
| 300 | if (isMemoryEffectFree(op) || |
| 301 | isa<gpu::BarrierOp, DeviceAsyncCreateGroupOp, DeviceAsyncWaitOp>(Val: op)) { |
| 302 | return op; |
| 303 | } |
| 304 | |
| 305 | // Otherwise, only async copies can currently be predicated. |
| 306 | auto asyncCopyOp = dyn_cast<DeviceAsyncCopyOp>(Val: op); |
| 307 | if (!asyncCopyOp) |
| 308 | return nullptr; |
| 309 | |
| 310 | // Create srcElement Value based on `predicate`. The next lines generate |
| 311 | // the following code: |
| 312 | // |
| 313 | // srcElement = (pred) ? prevSrcElements : 0; |
| 314 | // |
| 315 | Location loc = asyncCopyOp->getLoc(); |
| 316 | Value dstElements = arith::ConstantOp::create( |
| 317 | builder&: rewriter, location: loc, value: asyncCopyOp.getDstElementsAttr()); |
| 318 | Value originalSrcElement = |
| 319 | asyncCopyOp.getSrcElements() ? asyncCopyOp.getSrcElements() : dstElements; |
| 320 | Value c0Index = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 321 | auto srcElements = arith::SelectOp::create(builder&: rewriter, location: loc, condition: predicate, |
| 322 | true_value: originalSrcElement, false_value: c0Index); |
| 323 | auto asyncCopyZeroFillOp = DeviceAsyncCopyOp::create( |
| 324 | builder&: rewriter, location: loc, asyncToken: DeviceAsyncTokenType::get(ctx: asyncCopyOp.getContext()), |
| 325 | dst: asyncCopyOp.getDst(), dstIndices: asyncCopyOp.getDstIndices(), src: asyncCopyOp.getSrc(), |
| 326 | srcIndices: asyncCopyOp.getSrcIndices(), dstElements: asyncCopyOp.getDstElements(), srcElements, |
| 327 | bypassL1: UnitAttr()); |
| 328 | rewriter.replaceOp(op: asyncCopyOp, newOp: asyncCopyZeroFillOp); |
| 329 | return asyncCopyZeroFillOp; |
| 330 | } |
| 331 | |
| 332 | /// Applies loop pipelining with the given depth to the given loop so that |
| 333 | /// copies into the shared memory are pipelined. Doesn't affect other loops. |
| 334 | /// Returns a pair containing the error state and the pipelined op, the latter |
| 335 | /// being null in case of any failure. The error state contains a definite error |
| 336 | /// if the IR has been modified and a silenceable error otherwise. |
| 337 | static std::tuple<DiagnosedSilenceableFailure, scf::ForOp> |
| 338 | pipelineForSharedCopies(RewriterBase &rewriter, scf::ForOp forOp, int64_t depth, |
| 339 | bool epiloguePeeling) { |
| 340 | llvm::SmallPtrSet<Operation *, 16> stage0Ops; |
| 341 | if (failed(Result: collectStage0PipeliningOps(forOp, ops&: stage0Ops))) { |
| 342 | return std::make_tuple( |
| 343 | args: emitSilenceableFailure(op: forOp, message: "cannot find stage 0 ops for pipelining" ), |
| 344 | args: scf::ForOp()); |
| 345 | } |
| 346 | if (stage0Ops.empty()) { |
| 347 | return std::make_tuple( |
| 348 | args: emitSilenceableFailure(op: forOp, message: "no shared memory copy" ), args: scf::ForOp()); |
| 349 | } |
| 350 | |
| 351 | scf::PipeliningOption options; |
| 352 | unsigned maxDepth = depth; |
| 353 | auto setAnnotation = [&](Operation *op, |
| 354 | scf::PipeliningOption::PipelinerPart part, |
| 355 | unsigned iteration) { |
| 356 | return setAsyncWaitGroupsInFlight(builder&: rewriter, op, part, iteration, depth: maxDepth); |
| 357 | }; |
| 358 | options.getScheduleFn = |
| 359 | [&](scf::ForOp schedulingFor, |
| 360 | std::vector<std::pair<Operation *, unsigned>> &ops) { |
| 361 | if (schedulingFor != forOp) |
| 362 | return; |
| 363 | return getPipelineStages(forOp, opsWithPipelineStages&: ops, depth: maxDepth, stage0Ops); |
| 364 | }; |
| 365 | options.annotateFn = setAnnotation; |
| 366 | if (!epiloguePeeling) { |
| 367 | options.peelEpilogue = false; |
| 368 | options.predicateFn = replaceOpWithPredicatedOp; |
| 369 | } |
| 370 | |
| 371 | OpBuilder::InsertionGuard guard(rewriter); |
| 372 | rewriter.setInsertionPoint(forOp); |
| 373 | bool modifiedIR; |
| 374 | FailureOr<scf::ForOp> maybePipelined = |
| 375 | pipelineForLoop(rewriter, forOp, options, modifiedIR: &modifiedIR); |
| 376 | if (succeeded(Result: maybePipelined)) { |
| 377 | return std::make_tuple(args: DiagnosedSilenceableFailure::success(), |
| 378 | args&: *maybePipelined); |
| 379 | } |
| 380 | return std::make_tuple( |
| 381 | args: modifiedIR |
| 382 | ? DiagnosedSilenceableFailure::definiteFailure() |
| 383 | : emitSilenceableFailure(op: forOp, message: "pipelining preconditions failed" ), |
| 384 | args: scf::ForOp()); |
| 385 | } |
| 386 | |
| 387 | DiagnosedSilenceableFailure PipelineSharedMemoryCopiesOp::applyToOne( |
| 388 | TransformRewriter &rewriter, scf::ForOp forOp, |
| 389 | ApplyToEachResultList &results, TransformState &state) { |
| 390 | auto [diag, pipelined] = pipelineForSharedCopies( |
| 391 | rewriter, forOp, depth: static_cast<int64_t>(getDepth()), epiloguePeeling: getPeelEpilogue()); |
| 392 | if (diag.succeeded()) { |
| 393 | results.push_back(op: pipelined); |
| 394 | return DiagnosedSilenceableFailure::success(); |
| 395 | } |
| 396 | if (diag.isDefiniteFailure()) { |
| 397 | auto diag = emitDefiniteFailure(message: "irreversible pipelining failure" ); |
| 398 | if (!getPeelEpilogue()) { |
| 399 | diag.attachNote(loc: forOp->getLoc()) << "couldn't predicate?" ; |
| 400 | diag.attachNote(loc: getLoc()) << "try setting " << getPeelEpilogueAttrName(); |
| 401 | } |
| 402 | return diag; |
| 403 | } |
| 404 | |
| 405 | return std::move(diag); |
| 406 | } |
| 407 | |
| 408 | //===----------------------------------------------------------------------===// |
| 409 | // RewriteMatmulAsMmaSyncOp |
| 410 | //===----------------------------------------------------------------------===// |
| 411 | |
| 412 | /// Helper struct to encode a pair of row/column indexings in the form of |
| 413 | /// affine expressions. |
| 414 | struct RowColIndexing : private std::pair<AffineExpr, AffineExpr> { |
| 415 | RowColIndexing(AffineExpr row, AffineExpr col) |
| 416 | : std::pair<AffineExpr, AffineExpr>(row, col) {} |
| 417 | |
| 418 | AffineExpr row() const { return first; }; |
| 419 | AffineExpr col() const { return second; }; |
| 420 | |
| 421 | void print(llvm::raw_ostream &os) const { |
| 422 | os << "- indexing: " << first << ", " << second; |
| 423 | } |
| 424 | }; |
| 425 | |
| 426 | /// Helper struct to provide a simple mapping from matmul operations to the |
| 427 | /// corresponding mma.sync operation. This is constrained to the case where the |
| 428 | /// matmul matches the mma.sync operation 1-1. |
| 429 | struct MmaSyncBuilder { |
| 430 | MmaSyncBuilder(OpBuilder &b, Location loc, OpFoldResult laneId) |
| 431 | : b(b), loc(loc), laneId(laneId) {} |
| 432 | |
| 433 | using IndexCalculator = |
| 434 | std::function<SmallVector<RowColIndexing>(MLIRContext *)>; |
| 435 | |
| 436 | /// Create the mma.sync operation corresponding to `linalgOp` along with all |
| 437 | /// the supporting load/store and vector operations. |
| 438 | FailureOr<Operation *> buildMmaSync(LinalgOp linalgOp); |
| 439 | |
| 440 | private: |
| 441 | struct MmaSyncInfo { |
| 442 | std::tuple<IndexCalculator, IndexCalculator, IndexCalculator> indexFns; |
| 443 | std::tuple<SmallVector<int64_t>, SmallVector<int64_t>, SmallVector<int64_t>> |
| 444 | vectorShapes; |
| 445 | SmallVector<int64_t> mmaShape; |
| 446 | bool tf32Enabled; |
| 447 | }; |
| 448 | |
| 449 | /// Return the specific index calculator for the given `linalgOp` or failure |
| 450 | /// if the op is not supported. This is the toplevel switch that should just |
| 451 | /// be Tablegen'd in the future. |
| 452 | FailureOr<MmaSyncInfo> getIndexCalculators(ArrayRef<int64_t> opShape, |
| 453 | TypeRange elementalTypes); |
| 454 | |
| 455 | //===--------------------------------------------------------------------===// |
| 456 | // Instruction-specific row, column indexing expression builders. |
| 457 | // These should all be declaratively specified via Tablegen in the future. |
| 458 | // The Tablegen specification should be as straightforward as possible to |
| 459 | // only model the existing size and type combinations. |
| 460 | //===--------------------------------------------------------------------===// |
| 461 | // |
| 462 | // TODO: Tablegen all this. |
| 463 | //===--------------------------------------------------------------------===// |
| 464 | // m16n8k4 tf32 case. |
| 465 | //===--------------------------------------------------------------------===// |
| 466 | /// From the NVIDIA doc: |
| 467 | /// groupID = %laneid >> 2 |
| 468 | /// threadIDInGroup = %laneid % 4 |
| 469 | /// row = groupID for a0 |
| 470 | /// groupID + 8 for a1 |
| 471 | /// col = threadIDInGroup |
| 472 | static SmallVector<RowColIndexing> m16n8k4tf32Lhs(MLIRContext *ctx) { |
| 473 | auto dim = getAffineDimExpr(position: 0, context: ctx); |
| 474 | AffineExpr groupID = dim.floorDiv(v: 4); |
| 475 | AffineExpr threadIDInGroup = dim % 4; |
| 476 | return {RowColIndexing{groupID, threadIDInGroup}, |
| 477 | RowColIndexing{groupID + 8, threadIDInGroup}}; |
| 478 | } |
| 479 | |
| 480 | /// From the NVIDIA doc: |
| 481 | /// groupID = %laneid >> 2 |
| 482 | /// threadIDInGroup = %laneid % 4 |
| 483 | /// row = threadIDInGroup |
| 484 | /// col = groupID |
| 485 | static SmallVector<RowColIndexing> m16n8k4tf32Rhs(MLIRContext *ctx) { |
| 486 | auto dim = getAffineDimExpr(position: 0, context: ctx); |
| 487 | AffineExpr groupID = dim.floorDiv(v: 4); |
| 488 | AffineExpr threadIDInGroup = dim % 4; |
| 489 | return {RowColIndexing{threadIDInGroup, groupID}}; |
| 490 | } |
| 491 | |
| 492 | /// From the NVIDIA doc: |
| 493 | /// groupID = %laneid >> 2 |
| 494 | /// threadIDInGroup = %laneid % 4 |
| 495 | /// row = groupID for c0 and c1 |
| 496 | /// groupID + 8 for c2 and c3 |
| 497 | /// col = (threadIDInGroup * 2) + (i & 0x1) for ci where i = {0,..,3} |
| 498 | static SmallVector<RowColIndexing> m16n8k4tf32Res(MLIRContext *ctx) { |
| 499 | auto dim = getAffineDimExpr(position: 0, context: ctx); |
| 500 | AffineExpr groupID = dim.floorDiv(v: 4); |
| 501 | AffineExpr threadIDInGroup = dim % 4; |
| 502 | return {RowColIndexing{groupID, threadIDInGroup * 2 + 0}, |
| 503 | RowColIndexing{groupID, threadIDInGroup * 2 + 1}, |
| 504 | RowColIndexing{groupID + 8, threadIDInGroup * 2 + 0}, |
| 505 | RowColIndexing{groupID + 8, threadIDInGroup * 2 + 1}}; |
| 506 | } |
| 507 | |
| 508 | //===--------------------------------------------------------------------===// |
| 509 | // m16n8k16 f16 case. |
| 510 | //===--------------------------------------------------------------------===// |
| 511 | /// From the NVIDIA doc: |
| 512 | /// groupID = %laneid >> 2 |
| 513 | /// threadIDInGroup = %laneid % 4 |
| 514 | /// |
| 515 | /// row = groupID for ai where 0 <= i < 2 || 4 <= i < 6 |
| 516 | /// groupID + 8 Otherwise |
| 517 | /// |
| 518 | /// col = (threadIDInGroup * 2) + (i & 0x1) for ai where i < 4 |
| 519 | /// (threadIDInGroup * 2) + (i & 0x1) + 8 for ai where i >= 4 |
| 520 | static SmallVector<RowColIndexing> m16n8k16f16Lhs(MLIRContext *ctx) { |
| 521 | auto dim = getAffineDimExpr(position: 0, context: ctx); |
| 522 | AffineExpr groupID = dim.floorDiv(v: 4); |
| 523 | AffineExpr threadIDInGroup = dim % 4; |
| 524 | // clang-format off |
| 525 | return { |
| 526 | RowColIndexing{groupID, threadIDInGroup * 2 + 0}, // i == 0 |
| 527 | RowColIndexing{groupID, threadIDInGroup * 2 + 1}, // i == 1 |
| 528 | RowColIndexing{groupID + 8, threadIDInGroup * 2 + 0}, // i == 2 |
| 529 | RowColIndexing{groupID + 8, threadIDInGroup * 2 + 1}, // i == 3 |
| 530 | RowColIndexing{groupID, threadIDInGroup * 2 + 0 + 8}, // i == 4 |
| 531 | RowColIndexing{groupID, threadIDInGroup * 2 + 1 + 8}, // i == 5 |
| 532 | RowColIndexing{groupID + 8, threadIDInGroup * 2 + 0 + 8}, // i == 6 |
| 533 | RowColIndexing{groupID + 8, threadIDInGroup * 2 + 1 + 8} // i == 7 |
| 534 | }; |
| 535 | // clang-format on |
| 536 | } |
| 537 | |
| 538 | /// From the NVIDIA doc: |
| 539 | /// groupID = %laneid >> 2 |
| 540 | /// threadIDInGroup = %laneid % 4 |
| 541 | /// |
| 542 | /// row = (threadIDInGroup * 2) + (i & 0x1) for bi where i < 2 |
| 543 | /// (threadIDInGroup * 2) + (i & 0x1) + 8 for bi where i >= 2 |
| 544 | /// |
| 545 | /// col = groupID |
| 546 | static SmallVector<RowColIndexing> m16n8k16f16Rhs(MLIRContext *ctx) { |
| 547 | auto dim = getAffineDimExpr(position: 0, context: ctx); |
| 548 | AffineExpr groupID = dim.floorDiv(v: 4); |
| 549 | AffineExpr threadIDInGroup = dim % 4; |
| 550 | // clang-format off |
| 551 | return { |
| 552 | RowColIndexing{threadIDInGroup * 2 + 0, groupID}, // i == 0 |
| 553 | RowColIndexing{threadIDInGroup * 2 + 1, groupID}, // i == 1 |
| 554 | RowColIndexing{threadIDInGroup * 2 + 0 + 8, groupID}, // i == 2 |
| 555 | RowColIndexing{threadIDInGroup * 2 + 1 + 8, groupID} // i == 3 |
| 556 | }; |
| 557 | // clang-format on |
| 558 | } |
| 559 | |
| 560 | /// From the NVIDIA doc: |
| 561 | /// groupID = %laneid >> 2 |
| 562 | /// threadIDInGroup = %laneid % 4 |
| 563 | /// |
| 564 | /// row = groupID for ci where i < 2 |
| 565 | /// groupID + 8 for ci where i >= 2 |
| 566 | /// |
| 567 | /// col = (threadIDInGroup * 2) + (i & 0x1) for ci where i = {0,..,3} |
| 568 | static SmallVector<RowColIndexing> m16n8k16f16Res(MLIRContext *ctx) { |
| 569 | auto dim = getAffineDimExpr(position: 0, context: ctx); |
| 570 | AffineExpr groupID = dim.floorDiv(v: 4); |
| 571 | AffineExpr threadIDInGroup = dim % 4; |
| 572 | // clang-format off |
| 573 | return { |
| 574 | RowColIndexing{groupID, threadIDInGroup * 2 + 0}, // i == 0 |
| 575 | RowColIndexing{groupID, threadIDInGroup * 2 + 1}, // i == 1 |
| 576 | RowColIndexing{groupID + 8, threadIDInGroup * 2 + 0}, // i == 2 |
| 577 | RowColIndexing{groupID + 8, threadIDInGroup * 2 + 1} // i == 3 |
| 578 | }; |
| 579 | // clang-format on |
| 580 | } |
| 581 | |
| 582 | //===--------------------------------------------------------------------===// |
| 583 | /// Helper functions to create customizable load and stores operations. The |
| 584 | /// specific shapes of each MMA instruction are passed via the |
| 585 | /// IndexCalculator callback. |
| 586 | //===--------------------------------------------------------------------===// |
| 587 | /// Build a list of memref.load operations indexed at `(row, col)` indices |
| 588 | /// that make sense for a particular MMA instruction and specified via the |
| 589 | /// IndexCalculator callback. |
| 590 | SmallVector<Value> buildMemRefLoads(OpBuilder &b, Location loc, |
| 591 | OpFoldResult laneId, Value memref, |
| 592 | const IndexCalculator &indexFn); |
| 593 | |
| 594 | /// Perform a distributed load of a vector operand of `vectorShape` for a |
| 595 | /// particular MMA instruction whose `(row, col)` indices are specified via |
| 596 | /// the IndexCalculator callback. Each `laneId` loads the subportion of the |
| 597 | /// data that makes sense for the particular MMA operation. |
| 598 | /// The `vectorShape` matches existing NVGPU dialect op specification but |
| 599 | /// could also be flattened in the future if needed for simplification. |
| 600 | Value buildMmaSyncMemRefLoadOperand(OpBuilder &b, Location loc, |
| 601 | OpFoldResult laneId, Value memref, |
| 602 | IndexCalculator indexFn, |
| 603 | ArrayRef<int64_t> vectorShape); |
| 604 | |
| 605 | /// Build a list of memref.store operations indexed at `(row, col)` indices |
| 606 | /// that make sense for a particular MMA instruction and specified via the |
| 607 | /// IndexCalculator callback. |
| 608 | SmallVector<Operation *> buildMemRefStores(OpBuilder &b, Location loc, |
| 609 | ValueRange toStore, |
| 610 | OpFoldResult laneId, Value memref, |
| 611 | const IndexCalculator &indexFn); |
| 612 | |
| 613 | /// Perform a distributed store of a vector operand of `vectorShape` for a |
| 614 | /// particular MMA instruction whose `(row, col)` indices are specified via |
| 615 | /// the IndexCalculator callback. Each `laneId` loads the subportion of the |
| 616 | /// data that makes sense for the particular MMA operation. |
| 617 | /// The `vectorShape` matches existing NVGPU dialect op specification but |
| 618 | /// could also be flattened in the future if needed for simplification. |
| 619 | SmallVector<Operation *> buildMmaSyncMemRefStoreOperand( |
| 620 | OpBuilder &b, Location loc, Value vectorToStore, OpFoldResult laneId, |
| 621 | Value memref, IndexCalculator indexFn, ArrayRef<int64_t> vectorShape); |
| 622 | |
| 623 | OpBuilder &b; |
| 624 | Location loc; |
| 625 | OpFoldResult laneId; |
| 626 | }; |
| 627 | |
| 628 | //===--------------------------------------------------------------------===// |
| 629 | /// Helper functions to create customizable load and stores operations. The |
| 630 | /// specific shapes of each MMA instruction are passed via the |
| 631 | /// IndexCalculator callback. |
| 632 | //===--------------------------------------------------------------------===// |
| 633 | |
| 634 | template <typename ApplyFn, typename ReduceFn> |
| 635 | static void foreachIndividualVectorElement(Value vector, ApplyFn applyFn, |
| 636 | ReduceFn reduceFn) { |
| 637 | VectorType vectorType = cast<VectorType>(Val: vector.getType()); |
| 638 | auto vectorShape = vectorType.getShape(); |
| 639 | auto strides = computeStrides(sizes: vectorShape); |
| 640 | for (int64_t idx = 0, e = vectorShape[0] * strides[0]; idx < e; ++idx) { |
| 641 | auto indices = delinearize(linearIndex: idx, strides); |
| 642 | reduceFn(applyFn(vector, idx, indices), idx, indices); |
| 643 | } |
| 644 | } |
| 645 | |
| 646 | SmallVector<Value> |
| 647 | MmaSyncBuilder::buildMemRefLoads(OpBuilder &b, Location loc, |
| 648 | OpFoldResult laneId, Value memref, |
| 649 | const IndexCalculator &indexFn) { |
| 650 | auto aff = [&](AffineExpr e) { |
| 651 | return affine::makeComposedFoldedAffineApply(b, loc, expr: e, operands: laneId); |
| 652 | }; |
| 653 | SmallVector<Value> res; |
| 654 | SmallVector<RowColIndexing> indexings = indexFn(b.getContext()); |
| 655 | for (auto indexing : indexings) { |
| 656 | Value row = getValueOrCreateConstantIndexOp(b, loc, ofr: aff(indexing.row())); |
| 657 | Value col = getValueOrCreateConstantIndexOp(b, loc, ofr: aff(indexing.col())); |
| 658 | auto load = memref::LoadOp::create(builder&: b, location: loc, memref, indices: ValueRange{row, col}); |
| 659 | res.push_back(Elt: load); |
| 660 | } |
| 661 | return res; |
| 662 | } |
| 663 | |
| 664 | Value MmaSyncBuilder::buildMmaSyncMemRefLoadOperand( |
| 665 | OpBuilder &b, Location loc, OpFoldResult laneId, Value memref, |
| 666 | IndexCalculator indexFn, ArrayRef<int64_t> vectorShape) { |
| 667 | auto loads = buildMemRefLoads(b, loc, laneId, memref, indexFn); |
| 668 | |
| 669 | Type elementType = getElementTypeOrSelf(type: memref.getType()); |
| 670 | auto vt = VectorType::get(shape: vectorShape, elementType); |
| 671 | Value res = vector::BroadcastOp::create(builder&: b, location: loc, result: vt, source: loads[0]); |
| 672 | foreachIndividualVectorElement( |
| 673 | vector: res, |
| 674 | /*applyFn=*/ |
| 675 | [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) { |
| 676 | return loads[linearIdx]; |
| 677 | }, |
| 678 | /*reduceFn=*/ |
| 679 | [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) { |
| 680 | res = vector::InsertOp::create(builder&: b, location: loc, valueToStore: v, dest: res, position: indices); |
| 681 | }); |
| 682 | |
| 683 | return res; |
| 684 | } |
| 685 | |
| 686 | SmallVector<Operation *> MmaSyncBuilder::buildMemRefStores( |
| 687 | OpBuilder &b, Location loc, ValueRange toStore, OpFoldResult laneId, |
| 688 | Value memref, const IndexCalculator &indexFn) { |
| 689 | auto aff = [&](AffineExpr e) { |
| 690 | return affine::makeComposedFoldedAffineApply(b, loc, expr: e, operands: laneId); |
| 691 | }; |
| 692 | SmallVector<Operation *> res; |
| 693 | for (auto [indexing, val] : |
| 694 | llvm::zip_equal(t: indexFn(b.getContext()), u&: toStore)) { |
| 695 | Value row = getValueOrCreateConstantIndexOp(b, loc, ofr: aff(indexing.row())); |
| 696 | Value col = getValueOrCreateConstantIndexOp(b, loc, ofr: aff(indexing.col())); |
| 697 | Operation *store = |
| 698 | memref::StoreOp::create(builder&: b, location: loc, valueToStore: val, memref, indices: ValueRange{row, col}); |
| 699 | res.push_back(Elt: store); |
| 700 | } |
| 701 | return res; |
| 702 | } |
| 703 | |
| 704 | SmallVector<Operation *> MmaSyncBuilder::buildMmaSyncMemRefStoreOperand( |
| 705 | OpBuilder &b, Location loc, Value vectorToStore, OpFoldResult laneId, |
| 706 | Value memref, IndexCalculator indexFn, ArrayRef<int64_t> vectorShape) { |
| 707 | SmallVector<Value> toStore; |
| 708 | toStore.reserve(N: 32); |
| 709 | foreachIndividualVectorElement( |
| 710 | vector: vectorToStore, |
| 711 | /*applyFn=*/ |
| 712 | [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) { |
| 713 | return vector::ExtractOp::create(builder&: b, location: loc, source: vectorToStore, position: indices); |
| 714 | }, |
| 715 | /*reduceFn=*/ |
| 716 | [&](Value v, int64_t linearIdx, ArrayRef<int64_t> indices) { |
| 717 | toStore.push_back(Elt: v); |
| 718 | }); |
| 719 | return buildMemRefStores(b, loc, toStore, laneId, memref, indexFn); |
| 720 | } |
| 721 | |
| 722 | static std::tuple<SmallVector<int64_t>, SmallVector<int64_t>, |
| 723 | SmallVector<int64_t>> |
| 724 | makeVectorShapes(ArrayRef<int64_t> lhs, ArrayRef<int64_t> rhs, |
| 725 | ArrayRef<int64_t> res) { |
| 726 | SmallVector<int64_t> vlhs(lhs); |
| 727 | SmallVector<int64_t> vrhs(rhs); |
| 728 | SmallVector<int64_t> vres(res); |
| 729 | return std::make_tuple(args&: vlhs, args&: vrhs, args&: vres); |
| 730 | } |
| 731 | |
| 732 | FailureOr<MmaSyncBuilder::MmaSyncInfo> |
| 733 | MmaSyncBuilder::getIndexCalculators(ArrayRef<int64_t> opShape, |
| 734 | TypeRange elementalTypes) { |
| 735 | // TODO: Tablegen all this. |
| 736 | Type f16 = b.getF16Type(); |
| 737 | Type f32 = b.getF32Type(); |
| 738 | if (opShape == ArrayRef<int64_t>{16, 8, 4} && |
| 739 | elementalTypes == TypeRange{f32, f32, f32}) { |
| 740 | return MmaSyncInfo{.indexFns: std::make_tuple(args: &MmaSyncBuilder::m16n8k4tf32Lhs, |
| 741 | args: &MmaSyncBuilder::m16n8k4tf32Rhs, |
| 742 | args: &MmaSyncBuilder::m16n8k4tf32Res), |
| 743 | .vectorShapes: makeVectorShapes(lhs: {2, 1}, rhs: {1, 1}, res: {2, 2}), |
| 744 | .mmaShape: SmallVector<int64_t>{opShape}, |
| 745 | /*tf32Enabled=*/true}; |
| 746 | } |
| 747 | // This is the version with f16 accumulation. |
| 748 | // TODO: version with f32 accumulation. |
| 749 | if (opShape == ArrayRef<int64_t>{16, 8, 16} && |
| 750 | elementalTypes == TypeRange{f16, f16, f16}) { |
| 751 | return MmaSyncInfo{.indexFns: std::make_tuple(args: &MmaSyncBuilder::m16n8k16f16Lhs, |
| 752 | args: &MmaSyncBuilder::m16n8k16f16Rhs, |
| 753 | args: &MmaSyncBuilder::m16n8k16f16Res), |
| 754 | .vectorShapes: makeVectorShapes(lhs: {4, 2}, rhs: {2, 2}, res: {2, 2}), |
| 755 | .mmaShape: SmallVector<int64_t>{opShape}, |
| 756 | /*tf32Enabled=*/false}; |
| 757 | } |
| 758 | return failure(); |
| 759 | } |
| 760 | |
| 761 | FailureOr<Operation *> MmaSyncBuilder::buildMmaSync(LinalgOp linalgOp) { |
| 762 | Value lhsMemRef = linalgOp.getDpsInputOperand(i: 0)->get(); |
| 763 | Value rhsMemRef = linalgOp.getDpsInputOperand(i: 1)->get(); |
| 764 | Value resMemRef = linalgOp.getDpsInitOperand(i: 0)->get(); |
| 765 | assert(cast<MemRefType>(lhsMemRef.getType()).getRank() == 2 && |
| 766 | "expected lhs to be a 2D memref" ); |
| 767 | assert(cast<MemRefType>(rhsMemRef.getType()).getRank() == 2 && |
| 768 | "expected rhs to be a 2D memref" ); |
| 769 | assert(cast<MemRefType>(resMemRef.getType()).getRank() == 2 && |
| 770 | "expected res to be a 2D memref" ); |
| 771 | |
| 772 | int64_t m = cast<MemRefType>(Val: lhsMemRef.getType()).getShape()[0]; |
| 773 | int64_t n = cast<MemRefType>(Val: rhsMemRef.getType()).getShape()[1]; |
| 774 | int64_t k = cast<MemRefType>(Val: lhsMemRef.getType()).getShape()[1]; |
| 775 | Type lhsType = getElementTypeOrSelf(type: lhsMemRef.getType()); |
| 776 | Type rhsType = getElementTypeOrSelf(type: rhsMemRef.getType()); |
| 777 | Type resType = getElementTypeOrSelf(type: resMemRef.getType()); |
| 778 | |
| 779 | FailureOr<MmaSyncInfo> maybeInfo = |
| 780 | getIndexCalculators(opShape: {m, n, k}, elementalTypes: {lhsType, rhsType, resType}); |
| 781 | if (failed(Result: maybeInfo)) |
| 782 | return failure(); |
| 783 | |
| 784 | const MmaSyncInfo &info = *maybeInfo; |
| 785 | auto [lhsIndexFn, rhsIndexFn, resIndexFn] = info.indexFns; |
| 786 | auto [lhsShape, rhsShape, resShape] = info.vectorShapes; |
| 787 | Value lhs = buildMmaSyncMemRefLoadOperand(b, loc, laneId, memref: lhsMemRef, |
| 788 | indexFn: lhsIndexFn, vectorShape: lhsShape); |
| 789 | Value rhs = buildMmaSyncMemRefLoadOperand(b, loc, laneId, memref: rhsMemRef, |
| 790 | indexFn: rhsIndexFn, vectorShape: rhsShape); |
| 791 | Value res = buildMmaSyncMemRefLoadOperand(b, loc, laneId, memref: resMemRef, |
| 792 | indexFn: resIndexFn, vectorShape: resShape); |
| 793 | res = |
| 794 | MmaSyncOp::create(builder&: b, location: loc, matrixA: lhs, matrixB: rhs, matrixC: res, mmaShape: info.mmaShape, tf32Enabled: info.tf32Enabled); |
| 795 | buildMmaSyncMemRefStoreOperand(b, loc, vectorToStore: res, laneId, memref: resMemRef, indexFn: resIndexFn, |
| 796 | vectorShape: resShape); |
| 797 | return res.getDefiningOp(); |
| 798 | } |
| 799 | |
| 800 | DiagnosedSilenceableFailure RewriteMatmulAsMmaSyncOp::applyToOne( |
| 801 | TransformRewriter &rewriter, LinalgOp linalgOp, |
| 802 | ApplyToEachResultList &results, TransformState &state) { |
| 803 | bool fail = true; |
| 804 | // TODO: more robust detection of matmulOp, with transposes etc. |
| 805 | if (isa_and_nonnull<linalg::MatmulOp>(Val: linalgOp.getOperation())) { |
| 806 | // Check to not let go the matmul with extended semantic, through this |
| 807 | // transform. |
| 808 | if (linalgOp.hasUserDefinedMaps()) { |
| 809 | return emitSilenceableError() |
| 810 | << "only matmul ops with non-extended semantics are supported" ; |
| 811 | } |
| 812 | Location loc = linalgOp.getLoc(); |
| 813 | // TODO: more robust computation of laneId, for now assume a single warp. |
| 814 | Value laneId = gpu::ThreadIdOp::create( |
| 815 | builder&: rewriter, location: loc, resultType: rewriter.getIndexType(), dimension: gpu::Dimension::x); |
| 816 | if (succeeded(Result: MmaSyncBuilder(rewriter, loc, laneId).buildMmaSync(linalgOp))) |
| 817 | fail = false; |
| 818 | } |
| 819 | |
| 820 | if (fail) { |
| 821 | DiagnosedSilenceableFailure diag = emitSilenceableError() |
| 822 | << "unsupported target op: " << linalgOp; |
| 823 | diag.attachNote(loc: linalgOp->getLoc()) << "target op" ; |
| 824 | return diag; |
| 825 | } |
| 826 | |
| 827 | rewriter.eraseOp(op: linalgOp); |
| 828 | return DiagnosedSilenceableFailure::success(); |
| 829 | } |
| 830 | |
| 831 | //===----------------------------------------------------------------------===// |
| 832 | // Hopper builders. |
| 833 | //===----------------------------------------------------------------------===// |
| 834 | |
| 835 | /// Helper to create the base Hopper-specific operations that are reused in |
| 836 | /// various other places. |
| 837 | struct HopperBuilder { |
| 838 | HopperBuilder(RewriterBase &rewriter, Location loc) |
| 839 | : rewriter(rewriter), loc(loc) {} |
| 840 | |
| 841 | TypedValue<MBarrierGroupType> |
| 842 | buildAndInitBarrierInSharedMemory(OpFoldResult numThreads); |
| 843 | |
| 844 | /// Create tma descriptor op to initiate transfer from global to shared |
| 845 | /// memory. This must be done before the launch op, on the host. |
| 846 | TypedValue<TensorMapDescriptorType> |
| 847 | buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref, |
| 848 | gpu::LaunchOp launchOp); |
| 849 | |
| 850 | /// Build a tma load from global memory to shared memory using `barrier` to |
| 851 | /// synchronize. Return the number of bytes that will be transferred. |
| 852 | OpFoldResult buildTmaAsyncLoad(TypedValue<TensorMapDescriptorType> globalDesc, |
| 853 | TypedValue<MemRefType> sharedMemref, |
| 854 | TypedValue<MBarrierGroupType> barrier, |
| 855 | SmallVectorImpl<Operation *> &loadOps); |
| 856 | void buildBarrierArriveTx(TypedValue<MBarrierGroupType> barrier, |
| 857 | ArrayRef<OpFoldResult> sizes); |
| 858 | |
| 859 | /// If threadIdx.x == 0 does TMA request + wait, else just wait. |
| 860 | /// Return the operation that performs the transfer on thread0. |
| 861 | // TODO: In the future, don't hardcode to thread 0 but elect a leader. |
| 862 | SmallVector<Operation *> buildPredicateLoadsOnThread0( |
| 863 | ArrayRef<TypedValue<TensorMapDescriptorType>> globalDescriptors, |
| 864 | ArrayRef<TypedValue<MemRefType>> sharedMemBuffers, |
| 865 | TypedValue<MBarrierGroupType> barrier); |
| 866 | |
| 867 | void buildTryWaitParity(TypedValue<MBarrierGroupType> barrier); |
| 868 | |
| 869 | RewriterBase &rewriter; |
| 870 | Location loc; |
| 871 | }; |
| 872 | |
| 873 | SmallVector<Operation *> HopperBuilder::buildPredicateLoadsOnThread0( |
| 874 | ArrayRef<TypedValue<TensorMapDescriptorType>> globalDescriptors, |
| 875 | ArrayRef<TypedValue<MemRefType>> sharedMemBuffers, |
| 876 | TypedValue<MBarrierGroupType> barrier) { |
| 877 | SmallVector<Operation *> loadOps; |
| 878 | Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 879 | Value tidx = gpu::ThreadIdOp::create(builder&: rewriter, location: loc, dimension: gpu::Dimension::x); |
| 880 | Value cond = arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, |
| 881 | lhs: tidx, rhs: zero); |
| 882 | // clang-format off |
| 883 | scf::IfOp::create(builder&: rewriter, |
| 884 | /*location=*/loc, |
| 885 | /*conditional=*/cond, |
| 886 | /*thenBuilder=*/ |
| 887 | [&](OpBuilder &lb, Location loc) { |
| 888 | SmallVector<OpFoldResult> sizes; |
| 889 | sizes.reserve(N: globalDescriptors.size()); |
| 890 | for (auto [desc, shmem] : llvm::zip_equal( |
| 891 | t&: globalDescriptors, u&: sharedMemBuffers)) { |
| 892 | OpFoldResult sz = buildTmaAsyncLoad(globalDesc: desc, sharedMemref: shmem, barrier, loadOps); |
| 893 | sizes.push_back(Elt: sz); |
| 894 | } |
| 895 | // TODO: Note that cutlass predeclares the barrier arrive tx before the tma.async.load. |
| 896 | // This may or may not have perf implications. |
| 897 | buildBarrierArriveTx(barrier, sizes); |
| 898 | scf::YieldOp::create(builder&: rewriter, location: loc); |
| 899 | }, |
| 900 | /*elseBuilder=*/ |
| 901 | [&](OpBuilder &lb, Location loc) { |
| 902 | // TODO: is this for no-thread divergence? |
| 903 | // Should we just yield the size and hoist? |
| 904 | buildBarrierArriveTx(barrier, sizes: getAsIndexOpFoldResult(ctx: rewriter.getContext(), val: 0)); |
| 905 | scf::YieldOp::create(builder&: rewriter, location: loc); |
| 906 | }); |
| 907 | // clang-format on |
| 908 | return loadOps; |
| 909 | } |
| 910 | |
| 911 | static Attribute getSharedAddressSpaceAttribute(OpBuilder &b) { |
| 912 | return gpu::AddressSpaceAttr::get( |
| 913 | context: b.getContext(), value: gpu::GPUDialect::getWorkgroupAddressSpace()); |
| 914 | // return b.getI64IntegerAttr(static_cast<int64_t>(kSharedMemorySpace)); |
| 915 | } |
| 916 | |
| 917 | TypedValue<MBarrierGroupType> |
| 918 | HopperBuilder::buildAndInitBarrierInSharedMemory(OpFoldResult numThreads) { |
| 919 | auto sharedMemorySpace = getSharedAddressSpaceAttribute(b&: rewriter); |
| 920 | Value barrier = MBarrierCreateOp::create( |
| 921 | builder&: rewriter, location: loc, |
| 922 | barriers: MBarrierGroupType::get(context: rewriter.getContext(), memorySpace: sharedMemorySpace)); |
| 923 | Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 924 | nvgpu::MBarrierInitOp::create( |
| 925 | builder&: rewriter, location: loc, barriers: barrier, |
| 926 | count: getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr: numThreads), mbarId: zero, |
| 927 | predicate: Value()); |
| 928 | gpu::BarrierOp::create(builder&: rewriter, location: loc); |
| 929 | return cast<TypedValue<MBarrierGroupType>>(Val&: barrier); |
| 930 | } |
| 931 | |
| 932 | TypedValue<TensorMapDescriptorType> |
| 933 | HopperBuilder::buildGlobalMemRefDescriptor(TypedValue<MemRefType> memref, |
| 934 | gpu::LaunchOp launchOp) { |
| 935 | OpBuilder::InsertionGuard guard(rewriter); |
| 936 | rewriter.setInsertionPoint(launchOp); |
| 937 | Value unrankedMemRef = memref::CastOp::create( |
| 938 | builder&: rewriter, location: loc, |
| 939 | dest: UnrankedMemRefType::get(elementType: memref.getType().getElementType(), |
| 940 | memorySpace: memref.getType().getMemorySpace()), |
| 941 | source: memref); |
| 942 | SmallVector<OpFoldResult> mixedSizes = |
| 943 | memref::getMixedSizes(builder&: rewriter, loc, value: memref); |
| 944 | SmallVector<Value> sizes = |
| 945 | getValueOrCreateConstantIndexOp(b&: rewriter, loc, valueOrAttrVec: mixedSizes); |
| 946 | |
| 947 | auto sharedMemorySpace = getSharedAddressSpaceAttribute(b&: rewriter); |
| 948 | Value desc = TmaCreateDescriptorOp::create( |
| 949 | builder&: rewriter, location: loc, |
| 950 | tensorMap: TensorMapDescriptorType::get(context: rewriter.getContext(), |
| 951 | tensor: MemRefType::Builder(memref.getType()) |
| 952 | .setMemorySpace(sharedMemorySpace), |
| 953 | swizzle: TensorMapSwizzleKind::SWIZZLE_NONE, |
| 954 | l2promo: TensorMapL2PromoKind::L2PROMO_NONE, |
| 955 | oob: TensorMapOOBKind::OOB_ZERO, |
| 956 | interleave: TensorMapInterleaveKind::INTERLEAVE_NONE), |
| 957 | tensor: unrankedMemRef, boxDimensions: sizes); |
| 958 | return cast<TypedValue<TensorMapDescriptorType>>(Val&: desc); |
| 959 | } |
| 960 | |
| 961 | OpFoldResult |
| 962 | HopperBuilder::buildTmaAsyncLoad(TypedValue<TensorMapDescriptorType> globalDesc, |
| 963 | TypedValue<MemRefType> sharedMemref, |
| 964 | TypedValue<MBarrierGroupType> barrier, |
| 965 | SmallVectorImpl<Operation *> &loadOps) { |
| 966 | MLIRContext *ctx = rewriter.getContext(); |
| 967 | Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 968 | Operation *loadOp = |
| 969 | TmaAsyncLoadOp::create(builder&: rewriter, location: loc, dst: sharedMemref, barriers: barrier, tensorMapDescriptor: globalDesc, |
| 970 | coordinates: ValueRange{zero, zero}, mbarId: zero, multicastMask: Value(), predicate: Value()); |
| 971 | loadOps.push_back(Elt: loadOp); |
| 972 | auto mixedSizes = memref::getMixedSizes(builder&: rewriter, loc, value: sharedMemref); |
| 973 | SmallVector<AffineExpr> symbols(mixedSizes.size()); |
| 974 | bindSymbolsList(ctx, exprs: llvm::MutableArrayRef{symbols}); |
| 975 | AffineExpr prodExprInBytes = |
| 976 | computeProduct(ctx, basis: symbols) * |
| 977 | (sharedMemref.getType().getElementTypeBitWidth() / 8); |
| 978 | auto res = affine::makeComposedFoldedAffineApply(b&: rewriter, loc, |
| 979 | expr: prodExprInBytes, operands: mixedSizes); |
| 980 | return res; |
| 981 | } |
| 982 | |
| 983 | void HopperBuilder::buildBarrierArriveTx(TypedValue<MBarrierGroupType> barrier, |
| 984 | ArrayRef<OpFoldResult> mixedSizes) { |
| 985 | assert(!mixedSizes.empty() && "expecte non-empty sizes" ); |
| 986 | MLIRContext *ctx = rewriter.getContext(); |
| 987 | SmallVector<AffineExpr> symbols(mixedSizes.size()); |
| 988 | bindSymbolsList(ctx, exprs: llvm::MutableArrayRef{symbols}); |
| 989 | AffineExpr sumExpr = computeSum(ctx, basis: symbols); |
| 990 | OpFoldResult size = |
| 991 | affine::makeComposedFoldedAffineApply(b&: rewriter, loc, expr: sumExpr, operands: mixedSizes); |
| 992 | Value sizeVal = getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr: size); |
| 993 | Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 994 | nvgpu::MBarrierArriveExpectTxOp::create(builder&: rewriter, location: loc, barriers: barrier, txcount: sizeVal, mbarId: zero, |
| 995 | predicate: Value()); |
| 996 | } |
| 997 | |
| 998 | void HopperBuilder::buildTryWaitParity(TypedValue<MBarrierGroupType> barrier) { |
| 999 | Type i1 = rewriter.getI1Type(); |
| 1000 | Value parity = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i1, value: 0); |
| 1001 | // 10M is an arbitrary, not too small or too big number to specify the number |
| 1002 | // of ticks before retry. |
| 1003 | // TODO: hoist this in a default dialect constant. |
| 1004 | Value ticksBeforeRetry = |
| 1005 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 10000000); |
| 1006 | Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 1007 | nvgpu::MBarrierTryWaitParityOp::create(builder&: rewriter, location: loc, barriers: barrier, phaseParity: parity, |
| 1008 | ticks: ticksBeforeRetry, mbarId: zero); |
| 1009 | } |
| 1010 | |
| 1011 | //===----------------------------------------------------------------------===// |
| 1012 | // RewriteCopyAsTmaOp |
| 1013 | //===----------------------------------------------------------------------===// |
| 1014 | |
| 1015 | /// Helper to create the tma operations corresponding to `linalg::CopyOp`. |
| 1016 | struct CopyBuilder : public HopperBuilder { |
| 1017 | CopyBuilder(RewriterBase &rewriter, Location loc) |
| 1018 | : HopperBuilder(rewriter, loc) {} |
| 1019 | |
| 1020 | SmallVector<Operation *> rewrite(ArrayRef<Operation *> copyOps); |
| 1021 | }; |
| 1022 | |
| 1023 | SmallVector<Operation *> CopyBuilder::rewrite(ArrayRef<Operation *> copyOps) { |
| 1024 | MLIRContext *ctx = rewriter.getContext(); |
| 1025 | if (copyOps.empty()) |
| 1026 | return SmallVector<Operation *>(); |
| 1027 | |
| 1028 | auto launchOp = copyOps.front()->getParentOfType<gpu::LaunchOp>(); |
| 1029 | assert(launchOp && "expected launch op" ); |
| 1030 | |
| 1031 | // 1. Init a barrier object in shared memory. |
| 1032 | OpBuilder::InsertionGuard g(rewriter); |
| 1033 | rewriter.setInsertionPoint(copyOps.front()); |
| 1034 | AffineExpr bx, by, bz; |
| 1035 | bindSymbols(ctx, exprs&: bx, exprs&: by, exprs&: bz); |
| 1036 | AffineExpr prod = computeProduct(ctx, basis: ArrayRef<AffineExpr>{bx, by, bz}); |
| 1037 | OpFoldResult numThreads = affine::makeComposedFoldedAffineApply( |
| 1038 | b&: rewriter, loc, expr: prod, |
| 1039 | operands: ArrayRef<OpFoldResult>{launchOp.getBlockSizeX(), launchOp.getBlockSizeY(), |
| 1040 | launchOp.getBlockSizeZ()}); |
| 1041 | |
| 1042 | TypedValue<MBarrierGroupType> barrier = |
| 1043 | buildAndInitBarrierInSharedMemory(numThreads); |
| 1044 | |
| 1045 | SmallVector<TypedValue<MemRefType>> shmems; |
| 1046 | SmallVector<TypedValue<TensorMapDescriptorType>> globalDescs; |
| 1047 | for (Operation *op : copyOps) { |
| 1048 | auto copyOp = cast<linalg::CopyOp>(Val: op); |
| 1049 | auto inMemRef = |
| 1050 | cast<TypedValue<MemRefType>>(Val: copyOp.getDpsInputOperand(i: 0)->get()); |
| 1051 | assert(inMemRef.getType().getRank() == 2 && |
| 1052 | "expected in to be a 2D memref" ); |
| 1053 | |
| 1054 | // 2. Build global memory descriptor. |
| 1055 | TypedValue<TensorMapDescriptorType> globalDesc = |
| 1056 | buildGlobalMemRefDescriptor(memref: inMemRef, launchOp); |
| 1057 | globalDescs.push_back(Elt: globalDesc); |
| 1058 | |
| 1059 | // 3. Shared memory and descriptor for the tmp array. |
| 1060 | auto shmem = |
| 1061 | cast<TypedValue<MemRefType>>(Val: copyOp.getDpsInitOperand(i: 0)->get()); |
| 1062 | shmems.push_back(Elt: shmem); |
| 1063 | } |
| 1064 | |
| 1065 | // 4. Load in from global memory to shared memory using tma. |
| 1066 | OpBuilder::InsertionGuard g2(rewriter); |
| 1067 | rewriter.setInsertionPoint(copyOps.front()); |
| 1068 | SmallVector<Operation *> results = |
| 1069 | buildPredicateLoadsOnThread0(globalDescriptors: globalDescs, sharedMemBuffers: shmems, barrier); |
| 1070 | |
| 1071 | // 5. Spin-loop until data is ready. |
| 1072 | buildTryWaitParity(barrier); |
| 1073 | |
| 1074 | // 6. Erase the ops that have now been rewritten. |
| 1075 | for (Operation *op : copyOps) |
| 1076 | rewriter.eraseOp(op); |
| 1077 | |
| 1078 | return results; |
| 1079 | } |
| 1080 | |
| 1081 | DiagnosedSilenceableFailure |
| 1082 | RewriteCopyAsTmaOp::apply(TransformRewriter &rewriter, |
| 1083 | TransformResults &results, TransformState &state) { |
| 1084 | auto payloadOps = state.getPayloadOps(value: getTarget()); |
| 1085 | gpu::LaunchOp commonLaunchOp; |
| 1086 | Operation *firstOp, *failingOp; |
| 1087 | if (llvm::any_of(Range&: payloadOps, P: [&](Operation *op) { |
| 1088 | if (!commonLaunchOp) { |
| 1089 | commonLaunchOp = op->getParentOfType<gpu::LaunchOp>(); |
| 1090 | firstOp = op; |
| 1091 | } |
| 1092 | auto fail = !op->getParentOfType<gpu::LaunchOp>() || |
| 1093 | commonLaunchOp != op->getParentOfType<gpu::LaunchOp>() || |
| 1094 | !isa<linalg::CopyOp>(Val: op); |
| 1095 | if (fail) |
| 1096 | failingOp = op; |
| 1097 | return fail; |
| 1098 | })) { |
| 1099 | DiagnosedSilenceableFailure diag = |
| 1100 | emitSilenceableError() |
| 1101 | << "target ops must be linalg::CopyOp nested under a common " |
| 1102 | "gpu.LaunchOp to be rewritten because the tma descriptors need to " |
| 1103 | "be created on the host.\nBut got: " |
| 1104 | << *firstOp << "\nand " << *failingOp; |
| 1105 | return diag; |
| 1106 | } |
| 1107 | |
| 1108 | // TODO: more robust detection of copy, with transposes etc. |
| 1109 | CopyBuilder(rewriter, getLoc()).rewrite(copyOps: llvm::to_vector(Range&: payloadOps)); |
| 1110 | |
| 1111 | return DiagnosedSilenceableFailure::success(); |
| 1112 | } |
| 1113 | |
| 1114 | //===----------------------------------------------------------------------===// |
| 1115 | // Transform op registration |
| 1116 | //===----------------------------------------------------------------------===// |
| 1117 | |
| 1118 | namespace { |
| 1119 | class NVGPUTransformDialectExtension |
| 1120 | : public TransformDialectExtension<NVGPUTransformDialectExtension> { |
| 1121 | public: |
| 1122 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(NVGPUTransformDialectExtension) |
| 1123 | |
| 1124 | NVGPUTransformDialectExtension() { |
| 1125 | declareGeneratedDialect<arith::ArithDialect>(); |
| 1126 | declareGeneratedDialect<affine::AffineDialect>(); |
| 1127 | declareGeneratedDialect<NVGPUDialect>(); |
| 1128 | declareGeneratedDialect<NVVM::NVVMDialect>(); |
| 1129 | declareGeneratedDialect<vector::VectorDialect>(); |
| 1130 | registerTransformOps< |
| 1131 | #define GET_OP_LIST |
| 1132 | #include "mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp.inc" |
| 1133 | >(); |
| 1134 | } |
| 1135 | }; |
| 1136 | } // namespace |
| 1137 | |
| 1138 | #define GET_OP_CLASSES |
| 1139 | #include "mlir/Dialect/NVGPU/TransformOps/NVGPUTransformOps.cpp.inc" |
| 1140 | |
| 1141 | void mlir::nvgpu::registerTransformDialectExtension(DialectRegistry ®istry) { |
| 1142 | registry.addExtensions<NVGPUTransformDialectExtension>(); |
| 1143 | } |
| 1144 | |