| 1 | //===- SparseGPUCodegen.cpp - Generates GPU code --------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This is a prototype GPU codegenerator for the sparsifier. |
| 10 | // The objective is to eventually use the right combination of |
| 11 | // direct code generation and libary calls into vendor-specific |
| 12 | // highly optimized sparse libraries (e.g. cuSparse for CUDA). |
| 13 | // |
| 14 | //===----------------------------------------------------------------------===// |
| 15 | |
| 16 | #include "Utils/CodegenUtils.h" |
| 17 | #include "Utils/LoopEmitter.h" |
| 18 | |
| 19 | #include "mlir/Dialect/Bufferization/IR/Bufferization.h" |
| 20 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 21 | #include "mlir/Dialect/Linalg/IR/Linalg.h" |
| 22 | #include "mlir/Dialect/Linalg/Utils/Utils.h" |
| 23 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 24 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 25 | #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h" |
| 26 | #include "mlir/Dialect/SparseTensor/IR/SparseTensorType.h" |
| 27 | #include "mlir/Dialect/SparseTensor/Transforms/Passes.h" |
| 28 | #include "mlir/IR/IRMapping.h" |
| 29 | #include "mlir/IR/Matchers.h" |
| 30 | #include "mlir/Interfaces/SideEffectInterfaces.h" |
| 31 | #include "llvm/Support/Casting.h" |
| 32 | |
| 33 | using namespace mlir; |
| 34 | using namespace mlir::sparse_tensor; |
| 35 | |
| 36 | namespace { |
| 37 | |
| 38 | // Sparse formats supported by cuSparse. |
| 39 | enum class CuSparseFormat { |
| 40 | kNone, |
| 41 | kCOO, |
| 42 | kCSR, |
| 43 | kCSC, |
| 44 | kBSR, |
| 45 | }; |
| 46 | |
| 47 | //===----------------------------------------------------------------------===// |
| 48 | // Helper methods. |
| 49 | //===----------------------------------------------------------------------===// |
| 50 | |
| 51 | /// Marks the given top module as a GPU container module. |
| 52 | static void markAsGPUContainer(ModuleOp topModule) { |
| 53 | topModule->setDiscardableAttr(name: gpu::GPUDialect::getContainerModuleAttrName(), |
| 54 | value: UnitAttr::get(context: topModule->getContext())); |
| 55 | } |
| 56 | |
| 57 | /// Constructs a new GPU module (for GPU kernels) inside the given top module, |
| 58 | /// or returns an existing GPU module if one was built previously. |
| 59 | static gpu::GPUModuleOp genGPUModule(OpBuilder &builder, ModuleOp topModule) { |
| 60 | for (auto op : topModule.getBodyRegion().getOps<gpu::GPUModuleOp>()) |
| 61 | return op; // existing |
| 62 | markAsGPUContainer(topModule); |
| 63 | builder.setInsertionPointToStart(topModule.getBody()); |
| 64 | return gpu::GPUModuleOp::create(builder, location: topModule->getLoc(), |
| 65 | name: "sparse_kernels" ); |
| 66 | } |
| 67 | |
| 68 | /// Constructs a new GPU kernel in the given GPU module. |
| 69 | static gpu::GPUFuncOp genGPUFunc(OpBuilder &builder, gpu::GPUModuleOp gpuModule, |
| 70 | SmallVectorImpl<Value> &args) { |
| 71 | // Get a unique kernel name. Not very creative, |
| 72 | // but we simply try kernel0, kernel1, etc. |
| 73 | unsigned kernelNumber = 0; |
| 74 | SmallString<16> kernelName; |
| 75 | do { |
| 76 | kernelName.clear(); |
| 77 | ("kernel" + Twine(kernelNumber++)).toStringRef(Out&: kernelName); |
| 78 | } while (gpuModule.lookupSymbol(name: kernelName)); |
| 79 | // Then we insert a new kernel with given arguments into the module. |
| 80 | builder.setInsertionPointToStart(gpuModule.getBody()); |
| 81 | SmallVector<Type> argsTp; |
| 82 | for (auto arg : args) |
| 83 | argsTp.push_back(Elt: arg.getType()); |
| 84 | FunctionType type = FunctionType::get(context: gpuModule->getContext(), inputs: argsTp, results: {}); |
| 85 | auto gpuFunc = |
| 86 | gpu::GPUFuncOp::create(builder, location: gpuModule->getLoc(), name: kernelName, type); |
| 87 | gpuFunc.setKernel(true); |
| 88 | return gpuFunc; |
| 89 | } |
| 90 | |
| 91 | /// Constructs code to launch GPU kernel. |
| 92 | static Value genLaunchGPUFunc(OpBuilder &builder, gpu::GPUFuncOp gpuFunc, |
| 93 | SmallVectorImpl<Value> &args, |
| 94 | SmallVectorImpl<Value> &tokens, |
| 95 | unsigned numThreads) { |
| 96 | Location loc = gpuFunc->getLoc(); |
| 97 | Value none = TypedValue<::mlir::IntegerType>{}; |
| 98 | Value one = constantIndex(builder, loc, i: 1); |
| 99 | Value numT = constantIndex(builder, loc, i: numThreads); |
| 100 | gpu::KernelDim3 gridSize = {.x: one, .y: one, .z: one}; |
| 101 | gpu::KernelDim3 blckSize = {.x: numT, .y: one, .z: one}; |
| 102 | return gpu::LaunchFuncOp::create(builder, location: loc, kernelFunc: gpuFunc, gridSize, blockSize: blckSize, |
| 103 | /*dynSharedMemSz*/ dynamicSharedMemorySize: none, kernelOperands: args, |
| 104 | asyncTokenType: builder.getType<gpu::AsyncTokenType>(), |
| 105 | asyncDependencies: tokens) |
| 106 | .getAsyncToken(); |
| 107 | } |
| 108 | |
| 109 | /// Maps the provided ranked host buffer into the device address space. |
| 110 | /// Writes from the host are guaranteed to be visible to device kernels |
| 111 | /// that are launched afterwards. Writes from the device are guaranteed |
| 112 | /// to be visible on the host after synchronizing with the device kernel |
| 113 | /// completion. Needs to cast the buffer to a unranked buffer. |
| 114 | static Value genHostRegisterMemref(OpBuilder &builder, Location loc, |
| 115 | Value mem) { |
| 116 | MemRefType memTp = cast<MemRefType>(Val: mem.getType()); |
| 117 | UnrankedMemRefType resTp = |
| 118 | UnrankedMemRefType::get(elementType: memTp.getElementType(), /*memorySpace=*/0); |
| 119 | Value cast = memref::CastOp::create(builder, location: loc, dest: resTp, source: mem); |
| 120 | gpu::HostRegisterOp::create(builder, location: loc, value: cast); |
| 121 | return cast; |
| 122 | } |
| 123 | |
| 124 | /// Unmaps the provided buffer, expecting the casted buffer. |
| 125 | static void genHostUnregisterMemref(OpBuilder &builder, Location loc, |
| 126 | Value cast) { |
| 127 | gpu::HostUnregisterOp::create(builder, location: loc, value: cast); |
| 128 | } |
| 129 | |
| 130 | /// Generates first wait in an asynchronous chain. |
| 131 | static Value genFirstWait(OpBuilder &builder, Location loc) { |
| 132 | Type tokenType = builder.getType<gpu::AsyncTokenType>(); |
| 133 | return gpu::WaitOp::create(builder, location: loc, asyncToken: tokenType, asyncDependencies: ValueRange()) |
| 134 | .getAsyncToken(); |
| 135 | } |
| 136 | |
| 137 | /// Generates last, blocking wait in an asynchronous chain. |
| 138 | static void genBlockingWait(OpBuilder &builder, Location loc, |
| 139 | ValueRange operands) { |
| 140 | gpu::WaitOp::create(builder, location: loc, asyncToken: Type(), asyncDependencies: operands); |
| 141 | } |
| 142 | |
| 143 | /// Allocates memory on the device. |
| 144 | /// TODO: A `host_shared` attribute could be used to indicate that |
| 145 | /// the buffer is visible by both host and device, but lowering |
| 146 | /// that feature does not seem to be fully supported yet. |
| 147 | static gpu::AllocOp genAllocMemRef(OpBuilder &builder, Location loc, Value mem, |
| 148 | Value token) { |
| 149 | auto tp = cast<ShapedType>(Val: mem.getType()); |
| 150 | auto elemTp = tp.getElementType(); |
| 151 | auto shape = tp.getShape(); |
| 152 | auto memTp = MemRefType::get(shape, elementType: elemTp); |
| 153 | SmallVector<Value> dynamicSizes; |
| 154 | for (unsigned r = 0, rank = tp.getRank(); r < rank; r++) { |
| 155 | if (shape[r] == ShapedType::kDynamic) { |
| 156 | Value dimOp = linalg::createOrFoldDimOp(b&: builder, loc, val: mem, dim: r); |
| 157 | dynamicSizes.push_back(Elt: dimOp); |
| 158 | } |
| 159 | } |
| 160 | return gpu::AllocOp::create(builder, location: loc, resultTypes: TypeRange({memTp, token.getType()}), |
| 161 | asyncDependencies: token, dynamicSizes, symbolOperands: ValueRange()); |
| 162 | } |
| 163 | |
| 164 | // Allocates a typed buffer on the host with given size. |
| 165 | static Value genHostBuffer(OpBuilder &builder, Location loc, Type type, |
| 166 | Value size) { |
| 167 | const auto memTp = MemRefType::get(shape: {ShapedType::kDynamic}, elementType: type); |
| 168 | return memref::AllocOp::create(builder, location: loc, memrefType: memTp, dynamicSizes: size).getResult(); |
| 169 | } |
| 170 | |
| 171 | // Allocates a typed buffer on the device with given size. |
| 172 | static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Type type, |
| 173 | Value size, Value token) { |
| 174 | const auto memTp = MemRefType::get(shape: {ShapedType::kDynamic}, elementType: type); |
| 175 | return gpu::AllocOp::create(builder, location: loc, resultTypes: TypeRange({memTp, token.getType()}), |
| 176 | asyncDependencies: token, dynamicSizes: size, symbolOperands: ValueRange()); |
| 177 | } |
| 178 | |
| 179 | // Allocates a void buffer on the device with given size. |
| 180 | static gpu::AllocOp genAllocBuffer(OpBuilder &builder, Location loc, Value size, |
| 181 | Value token) { |
| 182 | return genAllocBuffer(builder, loc, type: builder.getI8Type(), size, token); |
| 183 | } |
| 184 | |
| 185 | /// Deallocates memory from the device. |
| 186 | static Value genDeallocMemRef(OpBuilder &builder, Location loc, Value mem, |
| 187 | Value token) { |
| 188 | return gpu::DeallocOp::create(builder, location: loc, asyncToken: token.getType(), asyncDependencies: token, memref: mem) |
| 189 | .getAsyncToken(); |
| 190 | } |
| 191 | |
| 192 | /// Copies memory between host and device (direction is implicit). |
| 193 | static Value genCopyMemRef(OpBuilder &builder, Location loc, Value dst, |
| 194 | Value src, Value token) { |
| 195 | return gpu::MemcpyOp::create(builder, location: loc, asyncToken: token.getType(), asyncDependencies: token, dst, src) |
| 196 | .getAsyncToken(); |
| 197 | } |
| 198 | |
| 199 | /// Generates an alloc/copy pair. |
| 200 | static Value genAllocCopy(OpBuilder &builder, Location loc, Value b, |
| 201 | SmallVectorImpl<Value> &tokens) { |
| 202 | Value firstToken = genFirstWait(builder, loc); |
| 203 | auto alloc = genAllocMemRef(builder, loc, mem: b, token: firstToken); |
| 204 | Value devMem = alloc.getResult(i: 0); |
| 205 | Value depToken = alloc.getAsyncToken(); // copy-after-alloc |
| 206 | tokens.push_back(Elt: genCopyMemRef(builder, loc, dst: devMem, src: b, token: depToken)); |
| 207 | return devMem; |
| 208 | } |
| 209 | |
| 210 | /// Generates a memref from tensor operation. |
| 211 | static Value genTensorToMemref(PatternRewriter &rewriter, Location loc, |
| 212 | Value tensor) { |
| 213 | auto tensorType = llvm::cast<ShapedType>(Val: tensor.getType()); |
| 214 | auto memrefType = |
| 215 | MemRefType::get(shape: tensorType.getShape(), elementType: tensorType.getElementType()); |
| 216 | return bufferization::ToBufferOp::create(builder&: rewriter, location: loc, buffer: memrefType, tensor); |
| 217 | } |
| 218 | |
| 219 | /// Prepares the outlined arguments, passing scalars and buffers in. Here we |
| 220 | /// assume that the first buffer is the one allocated for output. We create |
| 221 | /// a set of properly chained asynchronous allocation/copy pairs to increase |
| 222 | /// overlap before launching the kernel. |
| 223 | static Value genParametersIn(OpBuilder &builder, Location loc, |
| 224 | SmallVectorImpl<Value> &scalars, |
| 225 | SmallVectorImpl<Value> &buffers, |
| 226 | SmallVectorImpl<Value> &args, |
| 227 | SmallVectorImpl<Value> &tokens, |
| 228 | bool useHostRegistrationForOut) { |
| 229 | Value out; |
| 230 | // Scalars are passed by value. |
| 231 | for (Value s : scalars) |
| 232 | args.push_back(Elt: s); |
| 233 | // Buffers are need to be made visible on device. |
| 234 | for (Value b : buffers) { |
| 235 | if (useHostRegistrationForOut) { |
| 236 | out = genHostRegisterMemref(builder, loc, mem: b); |
| 237 | args.push_back(Elt: b); |
| 238 | useHostRegistrationForOut = false; |
| 239 | continue; |
| 240 | } |
| 241 | args.push_back(Elt: genAllocCopy(builder, loc, b, tokens)); |
| 242 | } |
| 243 | return out; |
| 244 | } |
| 245 | |
| 246 | /// Finalizes the outlined arguments. The output buffer is copied depending |
| 247 | /// on the kernel token and then deallocated. All other buffers are simply |
| 248 | /// deallocated. Then we wait for all operations to complete. |
| 249 | /// |
| 250 | /// `copyBack` maps 1:1 to the `buffers` array. It tracks which buffers were |
| 251 | /// mutated by the kernel and require a device-to-host copy. An empty |
| 252 | /// `copyBack` array implies no buffers are "copied back". |
| 253 | static void genParametersOut(OpBuilder &builder, Location loc, Value out, |
| 254 | Value kernelToken, SmallVectorImpl<Value> &scalars, |
| 255 | SmallVectorImpl<Value> &buffers, |
| 256 | SmallVectorImpl<Value> &args, |
| 257 | SmallVectorImpl<Value> &tokens, |
| 258 | ArrayRef<bool> copyBack) { |
| 259 | unsigned base = scalars.size(); |
| 260 | |
| 261 | // `args` stores scalars followed by buffers. `base` is the index of the first |
| 262 | // buffer. `bufIdx` maps the current buffer to its exact 1:1 counterpart in |
| 263 | // the `copyBack` mask. |
| 264 | for (unsigned i = base, e = args.size(); i < e; i++) { |
| 265 | unsigned bufIdx = i - base; |
| 266 | Value firstToken; |
| 267 | |
| 268 | // Checks if the current buffer needs a device-to-host copy. |
| 269 | if (copyBack[bufIdx]) { |
| 270 | if (out && bufIdx == 0) { |
| 271 | genHostUnregisterMemref(builder, loc, cast: out); |
| 272 | out = Value(); |
| 273 | continue; |
| 274 | } |
| 275 | firstToken = |
| 276 | genCopyMemRef(builder, loc, dst: buffers[bufIdx], src: args[i], token: kernelToken); |
| 277 | } else { |
| 278 | firstToken = genFirstWait(builder, loc); |
| 279 | } |
| 280 | tokens.push_back(Elt: genDeallocMemRef(builder, loc, mem: args[i], token: firstToken)); |
| 281 | } |
| 282 | } |
| 283 | |
| 284 | /// Constructs code for new GPU kernel. |
| 285 | static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc, |
| 286 | scf::ParallelOp forallOp, |
| 287 | SmallVectorImpl<Value> &constants, |
| 288 | SmallVectorImpl<Value> &scalars, |
| 289 | SmallVectorImpl<Value> &buffers) { |
| 290 | Location loc = gpuFunc->getLoc(); |
| 291 | Block &block = gpuFunc.getBody().front(); |
| 292 | rewriter.setInsertionPointToStart(&block); |
| 293 | |
| 294 | // Re-generate the constants, recapture all arguments. |
| 295 | unsigned arg = 0; |
| 296 | IRMapping irMap; |
| 297 | for (Value c : constants) |
| 298 | irMap.map(from: c, to: rewriter.clone(op&: *c.getDefiningOp())->getResult(idx: 0)); |
| 299 | for (Value s : scalars) |
| 300 | irMap.map(from: s, to: block.getArgument(i: arg++)); |
| 301 | for (Value b : buffers) |
| 302 | irMap.map(from: b, to: block.getArgument(i: arg++)); |
| 303 | |
| 304 | // Assume 1-dimensional grid/block configuration (only x dimension), |
| 305 | // so that: |
| 306 | // row = blockIdx.x * blockDim.x + threadIdx.x |
| 307 | // inc = blockDim.x * gridDim.x |
| 308 | Value bid = gpu::BlockIdOp::create(builder&: rewriter, location: loc, dimension: gpu::Dimension::x); |
| 309 | Value bsz = gpu::BlockDimOp::create(builder&: rewriter, location: loc, dimension: gpu::Dimension::x); |
| 310 | Value tid = gpu::ThreadIdOp::create(builder&: rewriter, location: loc, dimension: gpu::Dimension::x); |
| 311 | Value gsz = gpu::GridDimOp::create(builder&: rewriter, location: loc, dimension: gpu::Dimension::x); |
| 312 | Value mul = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: bid, rhs: bsz); |
| 313 | Value row = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: mul, rhs: tid); |
| 314 | Value inc = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: bsz, rhs: gsz); |
| 315 | |
| 316 | // Construct the iteration over the computational space that |
| 317 | // accounts for the fact that the total number of threads and |
| 318 | // the amount of work to be done usually do not match precisely. |
| 319 | // for (r = row; r < N; r += inc) { |
| 320 | // <loop-body> |
| 321 | // } |
| 322 | Value upper = irMap.lookup(from: forallOp.getUpperBound()[0]); |
| 323 | scf::ForOp forOp = scf::ForOp::create(builder&: rewriter, location: loc, lowerBound: row, upperBound: upper, step: inc); |
| 324 | // The scf.for builder creates an empty block. scf.for does not allow multiple |
| 325 | // blocks in its region, so delete the block before `cloneRegionBefore` adds |
| 326 | // an additional block. |
| 327 | rewriter.eraseBlock(block: forOp.getBody()); |
| 328 | rewriter.cloneRegionBefore(region&: forallOp.getRegion(), parent&: forOp.getRegion(), |
| 329 | before: forOp.getRegion().begin(), mapping&: irMap); |
| 330 | // Replace the scf.reduce terminator. |
| 331 | rewriter.setInsertionPoint(forOp.getBody()->getTerminator()); |
| 332 | rewriter.replaceOpWithNewOp<scf::YieldOp>(op: forOp.getBody()->getTerminator()); |
| 333 | |
| 334 | // Done. |
| 335 | rewriter.setInsertionPointAfter(forOp); |
| 336 | gpu::ReturnOp::create(builder&: rewriter, location: gpuFunc->getLoc()); |
| 337 | } |
| 338 | |
| 339 | //===----------------------------------------------------------------------===// |
| 340 | // Library helper methods. |
| 341 | //===----------------------------------------------------------------------===// |
| 342 | |
| 343 | /// Helper to detect a + b with arguments taken from given block. |
| 344 | static bool matchAddOfArgs(Block *block, Value val) { |
| 345 | if (auto *def = val.getDefiningOp()) { |
| 346 | if (isa<arith::AddFOp, arith::AddIOp>(Val: def)) { |
| 347 | Value a = block->getArguments()[0]; |
| 348 | Value b = block->getArguments()[1]; |
| 349 | return (def->getOperand(idx: 0) == a && def->getOperand(idx: 1) == b) || |
| 350 | (def->getOperand(idx: 0) == b && def->getOperand(idx: 1) == a); |
| 351 | } |
| 352 | } |
| 353 | return false; |
| 354 | } |
| 355 | |
| 356 | /// Helper to detect a * b with arguments taken from given block. |
| 357 | static bool matchMulOfArgs(Block *block, Value val) { |
| 358 | if (auto *def = val.getDefiningOp()) { |
| 359 | if (isa<arith::MulFOp, arith::MulIOp>(Val: def)) { |
| 360 | Value a = block->getArguments()[0]; |
| 361 | Value b = block->getArguments()[1]; |
| 362 | return (def->getOperand(idx: 0) == a && def->getOperand(idx: 1) == b) || |
| 363 | (def->getOperand(idx: 0) == b && def->getOperand(idx: 1) == a); |
| 364 | } |
| 365 | } |
| 366 | return false; |
| 367 | } |
| 368 | |
| 369 | /// Helper to detect x = x + a * b |
| 370 | static bool matchSumOfMultOfArgs(linalg::GenericOp op) { |
| 371 | auto yieldOp = cast<linalg::YieldOp>(Val: op.getRegion().front().getTerminator()); |
| 372 | if (auto *def = yieldOp.getOperand(i: 0).getDefiningOp()) { |
| 373 | if (isa<arith::AddFOp, arith::AddIOp>(Val: def)) { |
| 374 | Value x = op.getBlock()->getArguments()[2]; |
| 375 | return (def->getOperand(idx: 0) == x && |
| 376 | matchMulOfArgs(block: op.getBlock(), val: def->getOperand(idx: 1))) || |
| 377 | (def->getOperand(idx: 1) == x && |
| 378 | matchMulOfArgs(block: op.getBlock(), val: def->getOperand(idx: 0))); |
| 379 | } |
| 380 | } |
| 381 | return false; |
| 382 | } |
| 383 | |
| 384 | // Helper to detect c += spy(s) x (a * b) |
| 385 | static bool matchSumReductionOfMulUnary(linalg::GenericOp op) { |
| 386 | auto yieldOp = cast<linalg::YieldOp>(Val: op.getRegion().front().getTerminator()); |
| 387 | // The linalg yields a custom reduce result. |
| 388 | Value s_out = op.getBlock()->getArguments()[2]; |
| 389 | if (auto redOp = |
| 390 | yieldOp.getOperand(i: 0).getDefiningOp<sparse_tensor::ReduceOp>()) { |
| 391 | // The reduce consumes the output. |
| 392 | Value other; |
| 393 | if (s_out == redOp->getOperand(idx: 0)) |
| 394 | other = redOp->getOperand(idx: 1); |
| 395 | else if (s_out == redOp->getOperand(idx: 1)) |
| 396 | other = redOp->getOperand(idx: 0); |
| 397 | else |
| 398 | return false; |
| 399 | // The reduce op also consumes an unary which also consumes the output |
| 400 | // and does not define an absent value. |
| 401 | if (auto unOp = other.getDefiningOp<sparse_tensor::UnaryOp>()) { |
| 402 | if (s_out != unOp->getOperand(idx: 0) || !unOp.getAbsentRegion().empty()) |
| 403 | return false; |
| 404 | // And the bodies are as expected. |
| 405 | auto yieldUn = cast<sparse_tensor::YieldOp>( |
| 406 | Val: unOp.getRegion(i: 0).front().getTerminator()); |
| 407 | auto yieldRed = cast<sparse_tensor::YieldOp>( |
| 408 | Val: redOp.getRegion().front().getTerminator()); |
| 409 | return matchMulOfArgs(block: op.getBlock(), val: yieldUn.getOperand(i: 0)) && |
| 410 | matchAddOfArgs(block: &redOp.getRegion().front(), val: yieldRed.getOperand(i: 0)); |
| 411 | } |
| 412 | } |
| 413 | return false; |
| 414 | } |
| 415 | |
| 416 | /// Test for dense tensor. |
| 417 | static bool isDenseTensor(Value v) { |
| 418 | auto sTp = getSparseTensorType(val: v); |
| 419 | return sTp.getDimRank() == sTp.getLvlRank() && sTp.isAllDense(); |
| 420 | } |
| 421 | |
| 422 | /// Test for suitable positions/coordinates width. |
| 423 | static bool isAdmissibleMetaData(SparseTensorType &aTp) { |
| 424 | return (aTp.getPosWidth() == 0 || aTp.getPosWidth() >= 16) && |
| 425 | (aTp.getCrdWidth() == 0 || aTp.getCrdWidth() >= 16); |
| 426 | } |
| 427 | |
| 428 | /// Test for sorted COO matrix with suitable metadata. |
| 429 | static bool isAdmissibleCOO(SparseTensorType &aTp) { |
| 430 | return aTp.getDimRank() == 2 && aTp.getLvlRank() == 2 && aTp.isIdentity() && |
| 431 | aTp.isCompressedLvl(l: 0) && aTp.isOrderedLvl(l: 0) && !aTp.isUniqueLvl(l: 0) && |
| 432 | aTp.isSingletonLvl(l: 1) && aTp.isOrderedLvl(l: 1) && aTp.isUniqueLvl(l: 1) && |
| 433 | isAdmissibleMetaData(aTp); |
| 434 | } |
| 435 | |
| 436 | /// Test for CSR matrix with suitable metadata. |
| 437 | static bool isAdmissibleCSR(SparseTensorType &aTp) { |
| 438 | return aTp.getDimRank() == 2 && aTp.getLvlRank() == 2 && aTp.isIdentity() && |
| 439 | aTp.isDenseLvl(l: 0) && aTp.isCompressedLvl(l: 1) && aTp.isOrderedLvl(l: 1) && |
| 440 | aTp.isUniqueLvl(l: 1) && isAdmissibleMetaData(aTp); |
| 441 | } |
| 442 | |
| 443 | /// Test for CSC matrix with suitable metadata. |
| 444 | static bool isAdmissibleCSC(SparseTensorType &aTp) { |
| 445 | return aTp.getDimRank() == 2 && aTp.getLvlRank() == 2 && !aTp.isIdentity() && |
| 446 | aTp.isPermutation() && aTp.isDenseLvl(l: 0) && aTp.isCompressedLvl(l: 1) && |
| 447 | aTp.isOrderedLvl(l: 1) && aTp.isUniqueLvl(l: 1) && isAdmissibleMetaData(aTp); |
| 448 | } |
| 449 | |
| 450 | /// Test for BSR matrix with suitable metadata. |
| 451 | static bool isAdmissibleBSR(SparseTensorType &aTp) { |
| 452 | if (aTp.getDimRank() == 2 && aTp.getLvlRank() == 4 && aTp.isDenseLvl(l: 0) && |
| 453 | aTp.isCompressedLvl(l: 1) && aTp.isOrderedLvl(l: 1) && aTp.isUniqueLvl(l: 1) && |
| 454 | aTp.isDenseLvl(l: 2) && aTp.isDenseLvl(l: 3) && isAdmissibleMetaData(aTp)) { |
| 455 | // CuSparse only supports "square" blocks currently. |
| 456 | SmallVector<unsigned> dims = getBlockSize(dimToLvl: aTp.getDimToLvl()); |
| 457 | assert(dims.size() == 2); |
| 458 | return dims[0] == dims[1] && dims[0] > 1; |
| 459 | } |
| 460 | return false; |
| 461 | } |
| 462 | |
| 463 | /// Test for 2:4 matrix with suitable metadata. |
| 464 | static bool isAdmissible24(SparseTensorType &aTp) { |
| 465 | return aTp.getDimRank() == 2 && aTp.getLvlRank() == 3 && aTp.isDenseLvl(l: 0) && |
| 466 | aTp.isDenseLvl(l: 1) && aTp.isNOutOfMLvl(l: 2) && isAdmissibleMetaData(aTp); |
| 467 | } |
| 468 | |
| 469 | /// Test for conversion into 2:4 matrix. |
| 470 | static bool isConversionInto24(Value v) { |
| 471 | if (auto cnv = v.getDefiningOp<ConvertOp>()) { |
| 472 | Value a = cnv.getResult(); |
| 473 | Value d = cnv.getSource(); |
| 474 | SparseTensorType aTp = getSparseTensorType(val: a); |
| 475 | return isDenseTensor(v: d) && isAdmissible24(aTp); |
| 476 | } |
| 477 | return false; |
| 478 | } |
| 479 | |
| 480 | /// Returns a suitable sparse format for the operation and given operand |
| 481 | /// types with cuSparse, or kNone if none is available. |
| 482 | static CuSparseFormat getCuSparseFormat(SparseTensorType aTp, |
| 483 | SparseTensorType bTp, |
| 484 | SparseTensorType cTp, bool enableRT, |
| 485 | bool isMatVec) { |
| 486 | // The other operands have a dense type. |
| 487 | if (bTp.hasEncoding() || cTp.hasEncoding()) |
| 488 | return CuSparseFormat::kNone; |
| 489 | // Now check for suitable operand type for the main operand. |
| 490 | if (isAdmissibleCOO(aTp)) |
| 491 | #ifdef CUSPARSE_COO_AOS |
| 492 | return isMatVec ? CuSparseFormat::kCOO : CuSparseFormat::kNone; |
| 493 | #else |
| 494 | return enableRT ? CuSparseFormat::kCOO : CuSparseFormat::kNone; |
| 495 | #endif |
| 496 | if (isAdmissibleCSR(aTp)) |
| 497 | return CuSparseFormat::kCSR; |
| 498 | if (isAdmissibleCSC(aTp)) |
| 499 | return CuSparseFormat::kCSC; |
| 500 | if (isAdmissibleBSR(aTp)) |
| 501 | return CuSparseFormat::kBSR; |
| 502 | return CuSparseFormat::kNone; |
| 503 | } |
| 504 | |
| 505 | /// Generates the first positions/coordinates of a sparse matrix. |
| 506 | static Value genFirstPosOrCrds(OpBuilder &builder, Location loc, Value a, |
| 507 | CuSparseFormat format, bool enableRT) { |
| 508 | if (format == CuSparseFormat::kCOO) { |
| 509 | // Library uses SoA COO, direct IR uses AoS COO. |
| 510 | if (enableRT) |
| 511 | return ToCoordinatesOp::create(builder, location: loc, tensor: a, level: 0); |
| 512 | return ToCoordinatesBufferOp::create(builder, location: loc, tensor: a); |
| 513 | } |
| 514 | // Formats CSR/CSC and BSR use positions at 1. |
| 515 | return ToPositionsOp::create(builder, location: loc, tensor: a, level: 1); |
| 516 | } |
| 517 | |
| 518 | /// Generates the second coordinates of a sparse matrix. |
| 519 | static Value genSecondCrds(OpBuilder &builder, Location loc, Value a, |
| 520 | CuSparseFormat format, bool enableRT) { |
| 521 | bool isCOO = format == CuSparseFormat::kCOO; |
| 522 | if (isCOO && !enableRT) |
| 523 | return Value(); // nothing needed |
| 524 | // Formats CSR/CSC and BSR use coordinates at 1. |
| 525 | return ToCoordinatesOp::create(builder, location: loc, tensor: a, level: 1); |
| 526 | } |
| 527 | |
| 528 | /// Generates the sparse matrix handle. |
| 529 | static Operation *genSpMat(OpBuilder &builder, Location loc, |
| 530 | SparseTensorType &aTp, Type handleTp, Type tokenTp, |
| 531 | Value token, Value sz1, Value sz2, Value nseA, |
| 532 | Value rowA, Value colA, Value valA, |
| 533 | CuSparseFormat format, bool enableRT) { |
| 534 | if (format == CuSparseFormat::kCOO) { |
| 535 | // Library uses SoA COO, direct IR uses AoS COO. |
| 536 | if (enableRT) { |
| 537 | assert(colA); |
| 538 | return gpu::CreateCooOp::create(builder, location: loc, spmat: handleTp, asyncToken: tokenTp, asyncDependencies: token, |
| 539 | rows: sz1, cols: sz2, nnz: nseA, rowIdxs: rowA, colIdxs: colA, values: valA); |
| 540 | } |
| 541 | #ifdef CUSPARSE_COO_AOS |
| 542 | assert(!colA); |
| 543 | return gpu::CreateCooAoSOp::create(builder, loc, handleTp, tokenTp, token, |
| 544 | sz1, sz2, nseA, rowA, valA); |
| 545 | #else |
| 546 | llvm_unreachable("gpu::CreateCooAoSOp is deprecated" ); |
| 547 | #endif |
| 548 | } |
| 549 | assert(colA); |
| 550 | if (format == CuSparseFormat::kCSR) |
| 551 | return gpu::CreateCsrOp::create(builder, location: loc, spmat: handleTp, asyncToken: tokenTp, asyncDependencies: token, rows: sz1, |
| 552 | cols: sz2, nnz: nseA, rowPos: rowA, colIdxs: colA, values: valA); |
| 553 | if (format == CuSparseFormat::kCSC) |
| 554 | return gpu::CreateCscOp::create(builder, location: loc, spmat: handleTp, asyncToken: tokenTp, asyncDependencies: token, rows: sz1, |
| 555 | cols: sz2, nnz: nseA, colPos: rowA, rowIdxs: colA, values: valA); |
| 556 | // BSR requires a bit more work since we need to pass in the block size |
| 557 | // and all others sizes in terms of blocks (#block-rows, #block-cols, |
| 558 | // #nonzero-blocks). |
| 559 | assert(format == CuSparseFormat::kBSR); |
| 560 | SmallVector<unsigned> dims = getBlockSize(dimToLvl: aTp.getDimToLvl()); |
| 561 | assert(dims.size() == 2 && dims[0] == dims[1]); |
| 562 | uint64_t b = dims[0]; |
| 563 | Value bSz = constantIndex(builder, loc, i: b); |
| 564 | Value bRows = arith::DivUIOp::create(builder, location: loc, lhs: sz1, rhs: bSz); |
| 565 | Value bCols = arith::DivUIOp::create(builder, location: loc, lhs: sz2, rhs: bSz); |
| 566 | Value bNum = arith::DivUIOp::create(builder, location: loc, lhs: nseA, |
| 567 | rhs: constantIndex(builder, loc, i: b * b)); |
| 568 | return gpu::CreateBsrOp::create(builder, location: loc, spmat: handleTp, asyncToken: tokenTp, asyncDependencies: token, brows: bRows, |
| 569 | bcols: bCols, bnnz: bNum, rBlockSize: bSz, cBlockSize: bSz, bRowPos: rowA, bColIdxs: colA, values: valA); |
| 570 | } |
| 571 | |
| 572 | /// Returns whether the operation swaps matrix dimensions. |
| 573 | static bool isTransposed(gpu::TransposeMode mode) { |
| 574 | switch (mode) { |
| 575 | case gpu::TransposeMode::NON_TRANSPOSE: |
| 576 | return false; |
| 577 | case gpu::TransposeMode::TRANSPOSE: |
| 578 | case gpu::TransposeMode::CONJUGATE_TRANSPOSE: |
| 579 | return true; |
| 580 | } |
| 581 | llvm_unreachable("unhandled transpose mode" ); |
| 582 | } |
| 583 | |
| 584 | /// Match and rewrite SpMV kernel. |
| 585 | static LogicalResult rewriteSpMV(PatternRewriter &rewriter, |
| 586 | linalg::GenericOp op, bool enableRT, |
| 587 | gpu::TransposeMode modeA) { |
| 588 | Location loc = op.getLoc(); |
| 589 | Value a = op.getOperand(i: 0); |
| 590 | Value x = op.getOperand(i: 1); |
| 591 | Value y = op.getOperand(i: 2); // we have y = op(A)x |
| 592 | SmallVector<Value> tokens; |
| 593 | |
| 594 | // Only admissible sparse matrix format and dense vectors (no BSR). |
| 595 | SparseTensorType aTp = getSparseTensorType(val: a); |
| 596 | SparseTensorType xTp = getSparseTensorType(val: x); |
| 597 | SparseTensorType yTp = getSparseTensorType(val: y); |
| 598 | auto format = getCuSparseFormat(aTp, bTp: xTp, cTp: yTp, enableRT, /*isMatVec=*/true); |
| 599 | if (format == CuSparseFormat::kNone || format == CuSparseFormat::kBSR) |
| 600 | return failure(); |
| 601 | |
| 602 | // Start sparse kernel and copy data from host to device. |
| 603 | // a : memR/memC/memV -> rowA,colA,valA |
| 604 | // x : memX -> vecX |
| 605 | // y : memY -> vecY |
| 606 | Value nseA = NumberOfEntriesOp::create(builder&: rewriter, location: loc, tensor: a); |
| 607 | Value rows = linalg::createOrFoldDimOp(b&: rewriter, loc, val: a, dim: 0); |
| 608 | Value cols = linalg::createOrFoldDimOp(b&: rewriter, loc, val: a, dim: 1); |
| 609 | bool isTranspose = isTransposed(mode: modeA); |
| 610 | Value szX = isTranspose ? rows : cols; |
| 611 | Value szY = isTranspose ? cols : rows; |
| 612 | Value memR = genFirstPosOrCrds(builder&: rewriter, loc, a, format, enableRT); |
| 613 | Value memC = genSecondCrds(builder&: rewriter, loc, a, format, enableRT); // or empty |
| 614 | Value memV = ToValuesOp::create(builder&: rewriter, location: loc, tensor: a); |
| 615 | Value rowA = genAllocCopy(builder&: rewriter, loc, b: memR, tokens); |
| 616 | Value colA = memC ? genAllocCopy(builder&: rewriter, loc, b: memC, tokens) : Value(); |
| 617 | Value valA = genAllocCopy(builder&: rewriter, loc, b: memV, tokens); |
| 618 | Value memX = genTensorToMemref(rewriter, loc, tensor: x); |
| 619 | Value vecX = genAllocCopy(builder&: rewriter, loc, b: memX, tokens); |
| 620 | Value memY = genTensorToMemref(rewriter, loc, tensor: y); |
| 621 | Value vecY = genAllocCopy(builder&: rewriter, loc, b: memY, tokens); |
| 622 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 623 | tokens.clear(); |
| 624 | |
| 625 | // Create sparse environment and sparse matrix/dense vector handles. |
| 626 | Type indexTp = rewriter.getIndexType(); |
| 627 | Type dnTensorHandleTp = rewriter.getType<gpu::SparseDnTensorHandleType>(); |
| 628 | Type spmatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>(); |
| 629 | Type tokenTp = rewriter.getType<gpu::AsyncTokenType>(); |
| 630 | Value token = genFirstWait(builder&: rewriter, loc); |
| 631 | Operation *spGenA = |
| 632 | genSpMat(builder&: rewriter, loc, aTp, handleTp: spmatHandleTp, tokenTp, token, sz1: rows, sz2: cols, |
| 633 | nseA, rowA, colA, valA, format, enableRT); |
| 634 | Value spMatA = spGenA->getResult(idx: 0); |
| 635 | token = spGenA->getResult(idx: 1); |
| 636 | auto dvecX = gpu::CreateDnTensorOp::create(builder&: rewriter, location: loc, dnTensor: dnTensorHandleTp, |
| 637 | asyncToken: tokenTp, asyncDependencies: token, memref: vecX, dims: szX); |
| 638 | Value dnX = dvecX.getResult(i: 0); |
| 639 | token = dvecX.getAsyncToken(); |
| 640 | auto dvecY = gpu::CreateDnTensorOp::create(builder&: rewriter, location: loc, dnTensor: dnTensorHandleTp, |
| 641 | asyncToken: tokenTp, asyncDependencies: token, memref: vecY, dims: szY); |
| 642 | Value dnY = dvecY.getResult(i: 0); |
| 643 | token = dvecY.getAsyncToken(); |
| 644 | auto dnYType = llvm::cast<ShapedType>(Val: y.getType()).getElementType(); |
| 645 | |
| 646 | // Precompute buffersize for SpMV. |
| 647 | auto bufferComp = gpu::SpMVBufferSizeOp::create( |
| 648 | builder&: rewriter, location: loc, bufferSz: indexTp, asyncToken: tokenTp, asyncDependencies: token, modeA, spmatA: spMatA, dnX, dnY, |
| 649 | /*computeType=*/dnYType); |
| 650 | Value bufferSz = bufferComp.getResult(i: 0); |
| 651 | token = bufferComp.getAsyncToken(); |
| 652 | auto buf = genAllocBuffer(builder&: rewriter, loc, size: bufferSz, token); |
| 653 | Value buffer = buf.getResult(i: 0); |
| 654 | token = buf.getAsyncToken(); |
| 655 | |
| 656 | // Perform the SpMV. |
| 657 | auto spmvComp = gpu::SpMVOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, modeA, |
| 658 | spmatA: spMatA, dnX, dnY, |
| 659 | /*computeType=*/dnYType, buffer); |
| 660 | token = spmvComp.getAsyncToken(); |
| 661 | |
| 662 | // Copy data back to host and free all the resoures. |
| 663 | token = gpu::DestroySpMatOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, spmat: spMatA) |
| 664 | .getAsyncToken(); |
| 665 | token = gpu::DestroyDnTensorOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, dnTensor: dnX) |
| 666 | .getAsyncToken(); |
| 667 | token = gpu::DestroyDnTensorOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, dnTensor: dnY) |
| 668 | .getAsyncToken(); |
| 669 | token = genDeallocMemRef(builder&: rewriter, loc, mem: rowA, token); |
| 670 | if (colA) |
| 671 | token = genDeallocMemRef(builder&: rewriter, loc, mem: colA, token); |
| 672 | token = genDeallocMemRef(builder&: rewriter, loc, mem: valA, token); |
| 673 | token = genDeallocMemRef(builder&: rewriter, loc, mem: buffer, token); |
| 674 | token = genDeallocMemRef(builder&: rewriter, loc, mem: vecX, token); |
| 675 | token = genCopyMemRef(builder&: rewriter, loc, dst: memY, src: vecY, token); |
| 676 | token = genDeallocMemRef(builder&: rewriter, loc, mem: vecY, token); |
| 677 | tokens.push_back(Elt: token); |
| 678 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 679 | tokens.clear(); |
| 680 | |
| 681 | // Done. |
| 682 | rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, args: y.getType(), args&: memY); |
| 683 | return success(); |
| 684 | } |
| 685 | |
| 686 | /// Match and rewrite SpMM kernel. |
| 687 | static LogicalResult rewriteSpMM(PatternRewriter &rewriter, |
| 688 | linalg::GenericOp op, bool enableRT, |
| 689 | gpu::TransposeMode modeA, |
| 690 | gpu::TransposeMode modeB) { |
| 691 | Location loc = op.getLoc(); |
| 692 | Value a = op.getOperand(i: 0); |
| 693 | Value b = op.getOperand(i: 1); |
| 694 | Value c = op.getOperand(i: 2); // we have C = op(A)op(B) |
| 695 | SmallVector<Value> tokens; |
| 696 | |
| 697 | // Only admissible sparse matrix format and dense matrices (no BSR). |
| 698 | SparseTensorType aTp = getSparseTensorType(val: a); |
| 699 | SparseTensorType bTp = getSparseTensorType(val: b); |
| 700 | SparseTensorType cTp = getSparseTensorType(val: c); |
| 701 | auto format = getCuSparseFormat(aTp, bTp, cTp, enableRT, /*isMatVec=*/false); |
| 702 | if (format == CuSparseFormat::kNone || format == CuSparseFormat::kBSR) |
| 703 | return failure(); |
| 704 | |
| 705 | // Start sparse kernel and copy data from host to device. |
| 706 | // a : memR/memC/memV -> rowA,colA,valA |
| 707 | // b : bufB -> matB |
| 708 | // c : bufC -> matC |
| 709 | Value nseA = NumberOfEntriesOp::create(builder&: rewriter, location: loc, tensor: a); |
| 710 | Value rows = linalg::createOrFoldDimOp(b&: rewriter, loc, val: a, dim: 0); |
| 711 | Value cols = linalg::createOrFoldDimOp(b&: rewriter, loc, val: a, dim: 1); |
| 712 | bool isTransposeA = isTransposed(mode: modeA); |
| 713 | bool isTransposeB = isTransposed(mode: modeB); |
| 714 | Value szm = isTransposeA ? cols : rows; |
| 715 | Value szk = isTransposeA ? rows : cols; |
| 716 | Value szn = linalg::createOrFoldDimOp(b&: rewriter, loc, val: b, dim: isTransposeB ? 0 : 1); |
| 717 | SmallVector<Value> bSizes = isTransposeB ? SmallVector<Value>{szn, szk} |
| 718 | : SmallVector<Value>{szk, szn}; |
| 719 | Value memR = genFirstPosOrCrds(builder&: rewriter, loc, a, format, enableRT); |
| 720 | Value memC = genSecondCrds(builder&: rewriter, loc, a, format, enableRT); // or empty |
| 721 | Value memV = ToValuesOp::create(builder&: rewriter, location: loc, tensor: a); |
| 722 | Value rowA = genAllocCopy(builder&: rewriter, loc, b: memR, tokens); |
| 723 | Value colA = memC ? genAllocCopy(builder&: rewriter, loc, b: memC, tokens) : Value(); |
| 724 | Value valA = genAllocCopy(builder&: rewriter, loc, b: memV, tokens); |
| 725 | Value bufB = genTensorToMemref(rewriter, loc, tensor: b); |
| 726 | Value matB = genAllocCopy(builder&: rewriter, loc, b: bufB, tokens); |
| 727 | Value bufC = genTensorToMemref(rewriter, loc, tensor: c); |
| 728 | Value matC = genAllocCopy(builder&: rewriter, loc, b: bufC, tokens); |
| 729 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 730 | tokens.clear(); |
| 731 | |
| 732 | // Create sparse environment and sparse matrix/dense matrix handles. |
| 733 | Type indexTp = rewriter.getIndexType(); |
| 734 | Type dnTensorHandleTp = rewriter.getType<gpu::SparseDnTensorHandleType>(); |
| 735 | Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>(); |
| 736 | Type tokenTp = rewriter.getType<gpu::AsyncTokenType>(); |
| 737 | Value token = genFirstWait(builder&: rewriter, loc); |
| 738 | Operation *spGenA = |
| 739 | genSpMat(builder&: rewriter, loc, aTp, handleTp: spMatHandleTp, tokenTp, token, sz1: rows, sz2: cols, |
| 740 | nseA, rowA, colA, valA, format, enableRT); |
| 741 | Value spMatA = spGenA->getResult(idx: 0); |
| 742 | token = spGenA->getResult(idx: 1); |
| 743 | auto dmatB = gpu::CreateDnTensorOp::create(builder&: rewriter, location: loc, dnTensor: dnTensorHandleTp, |
| 744 | asyncToken: tokenTp, asyncDependencies: token, memref: matB, dims: bSizes); |
| 745 | Value dnB = dmatB.getResult(i: 0); |
| 746 | token = dmatB.getAsyncToken(); |
| 747 | auto dmatC = |
| 748 | gpu::CreateDnTensorOp::create(builder&: rewriter, location: loc, dnTensor: dnTensorHandleTp, asyncToken: tokenTp, |
| 749 | asyncDependencies: token, memref: matC, dims: SmallVector<Value>{szm, szn}); |
| 750 | Value dnC = dmatC.getResult(i: 0); |
| 751 | token = dmatC.getAsyncToken(); |
| 752 | auto dmatCType = llvm::cast<ShapedType>(Val: c.getType()).getElementType(); |
| 753 | |
| 754 | // Precompute buffersize for SpMM. |
| 755 | auto bufferComp = gpu::SpMMBufferSizeOp::create( |
| 756 | builder&: rewriter, location: loc, bufferSzs: indexTp, asyncToken: tokenTp, asyncDependencies: token, modeA, modeB, spmatA: spMatA, dnmatB: dnB, dnmatC: dnC, |
| 757 | /*computeType=*/dmatCType); |
| 758 | Value bufferSz = bufferComp.getResult(i: 0); |
| 759 | token = bufferComp.getAsyncToken(); |
| 760 | auto buf = genAllocBuffer(builder&: rewriter, loc, size: bufferSz, token); |
| 761 | Value buffer = buf.getResult(i: 0); |
| 762 | token = buf.getAsyncToken(); |
| 763 | auto dnCType = llvm::cast<ShapedType>(Val: c.getType()).getElementType(); |
| 764 | |
| 765 | // Perform the SpMM. |
| 766 | auto spmmComp = gpu::SpMMOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, modeA, |
| 767 | modeB, spmatA: spMatA, dnmatB: dnB, dnmatC: dnC, |
| 768 | /*computeType=*/dnCType, buffers: buffer); |
| 769 | token = spmmComp.getAsyncToken(); |
| 770 | |
| 771 | // Copy data back to host and free all the resoures. |
| 772 | token = gpu::DestroySpMatOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, spmat: spMatA) |
| 773 | .getAsyncToken(); |
| 774 | token = gpu::DestroyDnTensorOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, dnTensor: dnB) |
| 775 | .getAsyncToken(); |
| 776 | token = gpu::DestroyDnTensorOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, dnTensor: dnC) |
| 777 | .getAsyncToken(); |
| 778 | token = genDeallocMemRef(builder&: rewriter, loc, mem: rowA, token); |
| 779 | if (colA) |
| 780 | token = genDeallocMemRef(builder&: rewriter, loc, mem: colA, token); |
| 781 | token = genDeallocMemRef(builder&: rewriter, loc, mem: valA, token); |
| 782 | token = genDeallocMemRef(builder&: rewriter, loc, mem: buffer, token); |
| 783 | token = genDeallocMemRef(builder&: rewriter, loc, mem: matB, token); |
| 784 | token = genCopyMemRef(builder&: rewriter, loc, dst: bufC, src: matC, token); |
| 785 | token = genDeallocMemRef(builder&: rewriter, loc, mem: matC, token); |
| 786 | tokens.push_back(Elt: token); |
| 787 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 788 | tokens.clear(); |
| 789 | |
| 790 | // Done. |
| 791 | rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, args: c.getType(), args&: bufC); |
| 792 | return success(); |
| 793 | } |
| 794 | |
| 795 | // Match and rewrite SpGEMM kernel. |
| 796 | static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter, |
| 797 | linalg::GenericOp op, bool enableRT) { |
| 798 | Location loc = op.getLoc(); |
| 799 | Value a = op.getOperand(i: 0); |
| 800 | Value b = op.getOperand(i: 1); |
| 801 | Value c = op.getOperand(i: 2); // we have C = AB |
| 802 | SmallVector<Value> tokens; |
| 803 | |
| 804 | // Only CSR <- CSR x CSR supported. |
| 805 | auto format = CuSparseFormat::kCSR; |
| 806 | SparseTensorType aTp = getSparseTensorType(val: a); |
| 807 | SparseTensorType bTp = getSparseTensorType(val: b); |
| 808 | SparseTensorType cTp = getSparseTensorType(val: c); |
| 809 | if (!isAdmissibleCSR(aTp) || !isAdmissibleCSR(aTp&: bTp) || !isAdmissibleCSR(aTp&: cTp)) |
| 810 | return failure(); |
| 811 | |
| 812 | // Start sparse kernel and copy data from host to device. |
| 813 | // a : amemR/amemC/amemV -> rowA,colA,valA |
| 814 | // b : bmemR/bmemC/bmemV -> rowB,colB,valB |
| 815 | // c : materializes |
| 816 | auto dnCType = cTp.getElementType(); |
| 817 | Value nseA = NumberOfEntriesOp::create(builder&: rewriter, location: loc, tensor: a); |
| 818 | Value nseB = NumberOfEntriesOp::create(builder&: rewriter, location: loc, tensor: b); |
| 819 | Value szm = linalg::createOrFoldDimOp(b&: rewriter, loc, val: a, dim: 0); |
| 820 | Value szk = linalg::createOrFoldDimOp(b&: rewriter, loc, val: a, dim: 1); |
| 821 | Value szn = linalg::createOrFoldDimOp(b&: rewriter, loc, val: b, dim: 1); |
| 822 | Value amemR = genFirstPosOrCrds(builder&: rewriter, loc, a, format, enableRT); |
| 823 | Value amemC = genSecondCrds(builder&: rewriter, loc, a, format, enableRT); // not empty |
| 824 | Value amemV = ToValuesOp::create(builder&: rewriter, location: loc, tensor: a); |
| 825 | Value bmemR = genFirstPosOrCrds(builder&: rewriter, loc, a: b, format, enableRT); |
| 826 | Value bmemC = genSecondCrds(builder&: rewriter, loc, a: b, format, enableRT); // not empty |
| 827 | Value bmemV = ToValuesOp::create(builder&: rewriter, location: loc, tensor: b); |
| 828 | Value rowA = genAllocCopy(builder&: rewriter, loc, b: amemR, tokens); |
| 829 | Value colA = genAllocCopy(builder&: rewriter, loc, b: amemC, tokens); |
| 830 | Value valA = genAllocCopy(builder&: rewriter, loc, b: amemV, tokens); |
| 831 | Value rowB = genAllocCopy(builder&: rewriter, loc, b: bmemR, tokens); |
| 832 | Value colB = genAllocCopy(builder&: rewriter, loc, b: bmemC, tokens); |
| 833 | Value valB = genAllocCopy(builder&: rewriter, loc, b: bmemV, tokens); |
| 834 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 835 | tokens.clear(); |
| 836 | |
| 837 | // Create sparse environment and sparse matrix/dense vector handles. |
| 838 | Type indexTp = rewriter.getIndexType(); |
| 839 | Type spmatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>(); |
| 840 | Type descTp = rewriter.getType<gpu::SparseSpGEMMOpHandleType>(); |
| 841 | Type tokenTp = rewriter.getType<gpu::AsyncTokenType>(); |
| 842 | Value token = genFirstWait(builder&: rewriter, loc); |
| 843 | Operation *spGenA = |
| 844 | genSpMat(builder&: rewriter, loc, aTp, handleTp: spmatHandleTp, tokenTp, token, sz1: szm, sz2: szk, |
| 845 | nseA, rowA, colA, valA, format, enableRT); |
| 846 | Value spMatA = spGenA->getResult(idx: 0); |
| 847 | token = spGenA->getResult(idx: 1); |
| 848 | Operation *spGenB = |
| 849 | genSpMat(builder&: rewriter, loc, aTp&: bTp, handleTp: spmatHandleTp, tokenTp, token, sz1: szk, sz2: szn, |
| 850 | nseA: nseB, rowA: rowB, colA: colB, valA: valB, format, enableRT); |
| 851 | Value spMatB = spGenB->getResult(idx: 0); |
| 852 | token = spGenB->getResult(idx: 1); |
| 853 | |
| 854 | // Sparse matrix C materializes (also assumes beta == 0). |
| 855 | Value zero = constantIndex(builder&: rewriter, loc, i: 0); |
| 856 | Value one = constantIndex(builder&: rewriter, loc, i: 1); |
| 857 | Value mplus1 = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: szm, rhs: one); |
| 858 | auto e1 = genAllocBuffer(builder&: rewriter, loc, type: cTp.getPosType(), size: mplus1, token); |
| 859 | Value rowC = e1.getResult(i: 0); |
| 860 | token = e1.getAsyncToken(); |
| 861 | auto e2 = genAllocBuffer(builder&: rewriter, loc, type: cTp.getCrdType(), size: zero, token); |
| 862 | Value colC = e2.getResult(i: 0); // no free needed |
| 863 | token = e2.getAsyncToken(); |
| 864 | auto e3 = genAllocBuffer(builder&: rewriter, loc, type: dnCType, size: zero, token); |
| 865 | Value valC = e3.getResult(i: 0); // no free needed |
| 866 | token = e3.getAsyncToken(); |
| 867 | Operation *spGenC = |
| 868 | genSpMat(builder&: rewriter, loc, aTp&: cTp, handleTp: spmatHandleTp, tokenTp, token, sz1: szm, sz2: szn, |
| 869 | nseA: zero, rowA: rowC, colA: colC, valA: valC, format, enableRT); |
| 870 | Value spMatC = spGenC->getResult(idx: 0); |
| 871 | token = spGenC->getResult(idx: 1); |
| 872 | |
| 873 | // Precompute buffersizes for SpGEMM. |
| 874 | Operation *descOp = |
| 875 | gpu::SpGEMMCreateDescrOp::create(builder&: rewriter, location: loc, desc: descTp, asyncToken: tokenTp, asyncDependencies: token); |
| 876 | Value desc = descOp->getResult(idx: 0); |
| 877 | token = descOp->getResult(idx: 1); |
| 878 | Operation *work1 = gpu::SpGEMMWorkEstimationOrComputeOp::create( |
| 879 | builder&: rewriter, location: loc, bufferSzNew: indexTp, asyncToken: tokenTp, asyncDependencies: token, desc, |
| 880 | modeA: gpu::TransposeMode::NON_TRANSPOSE, modeB: gpu::TransposeMode::NON_TRANSPOSE, |
| 881 | spmatA: spMatA, spmatB: spMatB, spmatC: spMatC, computeType: dnCType, bufferSz: zero, buffer: valC, |
| 882 | kind: gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION); |
| 883 | Value bufferSz1 = work1->getResult(idx: 0); |
| 884 | token = work1->getResult(idx: 1); |
| 885 | auto buf1 = genAllocBuffer(builder&: rewriter, loc, size: bufferSz1, token); |
| 886 | Value buffer1 = buf1.getResult(i: 0); |
| 887 | token = buf1.getAsyncToken(); |
| 888 | Operation *work2 = gpu::SpGEMMWorkEstimationOrComputeOp::create( |
| 889 | builder&: rewriter, location: loc, bufferSzNew: indexTp, asyncToken: tokenTp, asyncDependencies: token, desc, |
| 890 | modeA: gpu::TransposeMode::NON_TRANSPOSE, modeB: gpu::TransposeMode::NON_TRANSPOSE, |
| 891 | spmatA: spMatA, spmatB: spMatB, spmatC: spMatC, computeType: dnCType, bufferSz: bufferSz1, buffer: buffer1, |
| 892 | kind: gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION); |
| 893 | token = work2->getResult(idx: 1); |
| 894 | |
| 895 | // Compute step. |
| 896 | Operation *compute1 = gpu::SpGEMMWorkEstimationOrComputeOp::create( |
| 897 | builder&: rewriter, location: loc, bufferSzNew: indexTp, asyncToken: tokenTp, asyncDependencies: token, desc, |
| 898 | modeA: gpu::TransposeMode::NON_TRANSPOSE, modeB: gpu::TransposeMode::NON_TRANSPOSE, |
| 899 | spmatA: spMatA, spmatB: spMatB, spmatC: spMatC, computeType: dnCType, bufferSz: zero, buffer: valC, |
| 900 | kind: gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE); |
| 901 | Value bufferSz2 = compute1->getResult(idx: 0); |
| 902 | token = compute1->getResult(idx: 1); |
| 903 | auto buf2 = genAllocBuffer(builder&: rewriter, loc, size: bufferSz2, token); |
| 904 | Value buffer2 = buf2.getResult(i: 0); |
| 905 | token = buf2.getAsyncToken(); |
| 906 | Operation *compute2 = gpu::SpGEMMWorkEstimationOrComputeOp::create( |
| 907 | builder&: rewriter, location: loc, bufferSzNew: indexTp, asyncToken: tokenTp, asyncDependencies: token, desc, |
| 908 | modeA: gpu::TransposeMode::NON_TRANSPOSE, modeB: gpu::TransposeMode::NON_TRANSPOSE, |
| 909 | spmatA: spMatA, spmatB: spMatB, spmatC: spMatC, computeType: dnCType, bufferSz: bufferSz2, buffer: buffer2, |
| 910 | kind: gpu::SpGEMMWorkEstimationOrComputeKind::COMPUTE); |
| 911 | token = compute2->getResult(idx: 1); |
| 912 | |
| 913 | // Get sizes. |
| 914 | Operation *sizes = gpu::SpMatGetSizeOp::create( |
| 915 | builder&: rewriter, location: loc, rows: indexTp, cols: indexTp, nnz: indexTp, asyncToken: tokenTp, asyncDependencies: token, spmat: spMatC); |
| 916 | Value nnz = sizes->getResult(idx: 2); |
| 917 | token = sizes->getResult(idx: 3); |
| 918 | auto a2 = genAllocBuffer(builder&: rewriter, loc, type: cTp.getCrdType(), size: nnz, token); |
| 919 | colC = a2.getResult(i: 0); |
| 920 | token = a2.getAsyncToken(); |
| 921 | auto a3 = genAllocBuffer(builder&: rewriter, loc, type: dnCType, size: nnz, token); |
| 922 | valC = a3.getResult(i: 0); |
| 923 | token = a3.getAsyncToken(); |
| 924 | |
| 925 | // Update C with new pointers and copy final product back into C. |
| 926 | Operation *update = gpu::SetCsrPointersOp::create( |
| 927 | builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, spmat: spMatC, positions: rowC, coordinates: colC, values: valC); |
| 928 | token = update->getResult(idx: 0); |
| 929 | Operation *copy = gpu::SpGEMMCopyOp::create( |
| 930 | builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, desc, modeA: gpu::TransposeMode::NON_TRANSPOSE, |
| 931 | modeB: gpu::TransposeMode::NON_TRANSPOSE, spmatA: spMatA, spmatB: spMatB, spmatC: spMatC, computeType: dnCType); |
| 932 | token = copy->getResult(idx: 0); |
| 933 | |
| 934 | // Allocate buffers on host. |
| 935 | Value rowH = genHostBuffer(builder&: rewriter, loc, type: cTp.getPosType(), size: mplus1); |
| 936 | Value colH = genHostBuffer(builder&: rewriter, loc, type: cTp.getCrdType(), size: nnz); |
| 937 | Value valH = genHostBuffer(builder&: rewriter, loc, type: dnCType, size: nnz); |
| 938 | |
| 939 | // Copy data back to host and free all the resoures. |
| 940 | token = gpu::SpGEMMDestroyDescrOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, desc) |
| 941 | .getAsyncToken(); |
| 942 | token = gpu::DestroySpMatOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, spmat: spMatA) |
| 943 | .getAsyncToken(); |
| 944 | token = gpu::DestroySpMatOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, spmat: spMatB) |
| 945 | .getAsyncToken(); |
| 946 | token = gpu::DestroySpMatOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, spmat: spMatC) |
| 947 | .getAsyncToken(); |
| 948 | token = genCopyMemRef(builder&: rewriter, loc, dst: rowH, src: rowC, token); |
| 949 | token = genCopyMemRef(builder&: rewriter, loc, dst: colH, src: colC, token); |
| 950 | token = genCopyMemRef(builder&: rewriter, loc, dst: valH, src: valC, token); |
| 951 | token = genDeallocMemRef(builder&: rewriter, loc, mem: rowA, token); |
| 952 | token = genDeallocMemRef(builder&: rewriter, loc, mem: colA, token); |
| 953 | token = genDeallocMemRef(builder&: rewriter, loc, mem: valA, token); |
| 954 | token = genDeallocMemRef(builder&: rewriter, loc, mem: rowB, token); |
| 955 | token = genDeallocMemRef(builder&: rewriter, loc, mem: colB, token); |
| 956 | token = genDeallocMemRef(builder&: rewriter, loc, mem: valB, token); |
| 957 | token = genDeallocMemRef(builder&: rewriter, loc, mem: rowC, token); |
| 958 | token = genDeallocMemRef(builder&: rewriter, loc, mem: colC, token); |
| 959 | token = genDeallocMemRef(builder&: rewriter, loc, mem: valC, token); |
| 960 | token = genDeallocMemRef(builder&: rewriter, loc, mem: buffer1, token); |
| 961 | token = genDeallocMemRef(builder&: rewriter, loc, mem: buffer2, token); |
| 962 | tokens.push_back(Elt: token); |
| 963 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 964 | tokens.clear(); |
| 965 | |
| 966 | // Done. |
| 967 | Value vt = bufferization::ToTensorOp::create( |
| 968 | builder&: rewriter, location: loc, result: memref::getTensorTypeFromMemRefType(type: valH.getType()), buffer: valH); |
| 969 | Value rt = bufferization::ToTensorOp::create( |
| 970 | builder&: rewriter, location: loc, result: memref::getTensorTypeFromMemRefType(type: rowH.getType()), buffer: rowH); |
| 971 | Value ct = bufferization::ToTensorOp::create( |
| 972 | builder&: rewriter, location: loc, result: memref::getTensorTypeFromMemRefType(type: colH.getType()), buffer: colH); |
| 973 | rewriter.replaceOpWithNewOp<AssembleOp>(op, args: c.getType(), args: ValueRange{rt, ct}, |
| 974 | args&: vt); |
| 975 | return success(); |
| 976 | } |
| 977 | |
| 978 | // Match and rewrite 2:4 SpMM kernel. |
| 979 | static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter, |
| 980 | linalg::GenericOp op) { |
| 981 | Location loc = op.getLoc(); |
| 982 | Value A = op.getOperand(i: 0); |
| 983 | Value B = op.getOperand(i: 1); |
| 984 | Value C = op.getOperand(i: 2); // we have C = AB |
| 985 | SmallVector<Value> tokens; |
| 986 | |
| 987 | // The cuSparselt API currently only allows pruning and compression |
| 988 | // to occur on the device. So we recognize the pattern |
| 989 | // A' = convert A ; dense to 2:4 |
| 990 | // C = A'B ; 2:4 matrix mult |
| 991 | // and then perform compression and matrix multiplication on device. |
| 992 | auto cnv = A.getDefiningOp<ConvertOp>(); |
| 993 | assert(cnv); |
| 994 | A = cnv.getSource(); |
| 995 | |
| 996 | // All input should be dense tensors. |
| 997 | if (!isDenseTensor(v: A) || !isDenseTensor(v: B) || !isDenseTensor(v: C)) |
| 998 | return failure(); |
| 999 | |
| 1000 | // Start sparse kernel and copy data from host to device. |
| 1001 | // a : bufA -> matA |
| 1002 | // b : bufB -> matB |
| 1003 | // c : bufC -> matC |
| 1004 | Value bufA = genTensorToMemref(rewriter, loc, tensor: A); |
| 1005 | Value matA = genAllocCopy(builder&: rewriter, loc, b: bufA, tokens); |
| 1006 | Value bufB = genTensorToMemref(rewriter, loc, tensor: B); |
| 1007 | Value matB = genAllocCopy(builder&: rewriter, loc, b: bufB, tokens); |
| 1008 | Value bufC = genTensorToMemref(rewriter, loc, tensor: C); |
| 1009 | Value matC = genAllocCopy(builder&: rewriter, loc, b: bufC, tokens); |
| 1010 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 1011 | tokens.clear(); |
| 1012 | |
| 1013 | // Create sparse environment and sparse matrix/dense vector handles. |
| 1014 | Value szm = linalg::createOrFoldDimOp(b&: rewriter, loc, val: matA, dim: 0); |
| 1015 | Value szk = linalg::createOrFoldDimOp(b&: rewriter, loc, val: matB, dim: 0); |
| 1016 | Value szn = linalg::createOrFoldDimOp(b&: rewriter, loc, val: matC, dim: 1); |
| 1017 | Type indexTp = rewriter.getIndexType(); |
| 1018 | Type dnTensorHandleTp = rewriter.getType<gpu::SparseDnTensorHandleType>(); |
| 1019 | Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>(); |
| 1020 | Type tokenTp = rewriter.getType<gpu::AsyncTokenType>(); |
| 1021 | Value token = genFirstWait(builder&: rewriter, loc); |
| 1022 | Operation *spGenA = gpu::Create2To4SpMatOp::create( |
| 1023 | builder&: rewriter, location: loc, spMat: spMatHandleTp, asyncToken: tokenTp, asyncDependencies: token, rows: szm, cols: szk, |
| 1024 | pruneFlag: gpu::Prune2To4SpMatFlag::PRUNE_AND_CHECK, memref: matA); |
| 1025 | Value spMatA = spGenA->getResult(idx: 0); |
| 1026 | token = spGenA->getResult(idx: 1); |
| 1027 | auto dmatB = |
| 1028 | gpu::CreateDnTensorOp::create(builder&: rewriter, location: loc, dnTensor: dnTensorHandleTp, asyncToken: tokenTp, |
| 1029 | asyncDependencies: token, memref: matB, dims: SmallVector<Value>{szk, szn}); |
| 1030 | Value dnB = dmatB.getResult(i: 0); |
| 1031 | token = dmatB.getAsyncToken(); |
| 1032 | auto dmatC = |
| 1033 | gpu::CreateDnTensorOp::create(builder&: rewriter, location: loc, dnTensor: dnTensorHandleTp, asyncToken: tokenTp, |
| 1034 | asyncDependencies: token, memref: matC, dims: SmallVector<Value>{szm, szn}); |
| 1035 | Value dnC = dmatC.getResult(i: 0); |
| 1036 | token = dmatC.getAsyncToken(); |
| 1037 | auto dmatCType = llvm::cast<ShapedType>(Val: matC.getType()).getElementType(); |
| 1038 | |
| 1039 | // Precompute buffersize for SpMM. |
| 1040 | SmallVector<Type> bufferTypes_{indexTp, indexTp, indexTp}; |
| 1041 | TypeRange bufferTypes(bufferTypes_); |
| 1042 | auto bufferComp = gpu::SpMMBufferSizeOp::create( |
| 1043 | builder&: rewriter, location: loc, bufferSzs: bufferTypes, asyncToken: tokenTp, asyncDependencies: token, |
| 1044 | modeA: gpu::TransposeMode::NON_TRANSPOSE, modeB: gpu::TransposeMode::NON_TRANSPOSE, |
| 1045 | spmatA: spMatA, dnmatB: dnB, dnmatC: dnC, |
| 1046 | /*computeType=*/dmatCType); |
| 1047 | token = bufferComp.getAsyncToken(); |
| 1048 | |
| 1049 | // Allocate buffers on host. |
| 1050 | Value bufferSz1 = bufferComp.getResult(i: 0); |
| 1051 | auto buf1 = genAllocBuffer(builder&: rewriter, loc, size: bufferSz1, token); |
| 1052 | Value buffer1 = buf1.getResult(i: 0); |
| 1053 | token = buf1.getAsyncToken(); |
| 1054 | Value bufferSz2 = bufferComp.getResult(i: 1); |
| 1055 | auto buf2 = genAllocBuffer(builder&: rewriter, loc, size: bufferSz2, token); |
| 1056 | Value buffer2 = buf2.getResult(i: 0); |
| 1057 | token = buf2.getAsyncToken(); |
| 1058 | Value bufferSz3 = bufferComp.getResult(i: 2); |
| 1059 | auto buf3 = genAllocBuffer(builder&: rewriter, loc, size: bufferSz3, token); |
| 1060 | Value buffer3 = buf3.getResult(i: 0); |
| 1061 | token = buf3.getAsyncToken(); |
| 1062 | |
| 1063 | // Perform the SpMM. |
| 1064 | auto dnCType = llvm::cast<ShapedType>(Val: matC.getType()).getElementType(); |
| 1065 | auto spmmComp = gpu::SpMMOp::create( |
| 1066 | builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, spmatA: spMatA, dnmatB: dnB, dnmatC: dnC, /*computeType=*/dnCType, |
| 1067 | buffers: SmallVector<Value>{buffer1, buffer2, buffer3}); |
| 1068 | token = spmmComp.getAsyncToken(); |
| 1069 | |
| 1070 | // Copy data back to host and free all the resources. |
| 1071 | token = gpu::DestroySpMatOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, spmat: spMatA) |
| 1072 | .getAsyncToken(); |
| 1073 | token = gpu::DestroyDnTensorOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, dnTensor: dnB) |
| 1074 | .getAsyncToken(); |
| 1075 | token = gpu::DestroyDnTensorOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, dnTensor: dnC) |
| 1076 | .getAsyncToken(); |
| 1077 | token = genDeallocMemRef(builder&: rewriter, loc, mem: buffer1, token); |
| 1078 | token = genDeallocMemRef(builder&: rewriter, loc, mem: buffer2, token); |
| 1079 | token = genDeallocMemRef(builder&: rewriter, loc, mem: buffer3, token); |
| 1080 | token = genDeallocMemRef(builder&: rewriter, loc, mem: matA, token); |
| 1081 | token = genDeallocMemRef(builder&: rewriter, loc, mem: matB, token); |
| 1082 | token = genCopyMemRef(builder&: rewriter, loc, dst: bufC, src: matC, token); |
| 1083 | token = genDeallocMemRef(builder&: rewriter, loc, mem: matC, token); |
| 1084 | tokens.push_back(Elt: token); |
| 1085 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 1086 | tokens.clear(); |
| 1087 | |
| 1088 | // Done. |
| 1089 | rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, args: C.getType(), args&: bufC); |
| 1090 | return success(); |
| 1091 | } |
| 1092 | |
| 1093 | /// Match and rewrite SDDMM kernel. |
| 1094 | static LogicalResult rewriteSDDMM(PatternRewriter &rewriter, |
| 1095 | linalg::GenericOp op, bool enableRT) { |
| 1096 | Location loc = op.getLoc(); |
| 1097 | Value a = op.getOperand(i: 0); |
| 1098 | Value b = op.getOperand(i: 1); |
| 1099 | Value c = op.getOperand(i: 2); |
| 1100 | SmallVector<Value> tokens; |
| 1101 | |
| 1102 | // Only admissible sparse matrix format (no COO/CSC) and dense matrices. |
| 1103 | SparseTensorType aTp = getSparseTensorType(val: a); |
| 1104 | SparseTensorType bTp = getSparseTensorType(val: b); |
| 1105 | SparseTensorType cTp = getSparseTensorType(val: c); |
| 1106 | auto format = getCuSparseFormat(aTp: cTp, bTp, cTp: aTp, enableRT, /*isMatVec=*/false); |
| 1107 | if (format == CuSparseFormat::kNone || format == CuSparseFormat::kCOO || |
| 1108 | format == CuSparseFormat::kCSC) |
| 1109 | return failure(); |
| 1110 | |
| 1111 | // The SDDMM does the in-place operation. |
| 1112 | // Start sparse kernel and copy data from host to device. |
| 1113 | // a : bufA -> matA |
| 1114 | // b : bufB -> matB |
| 1115 | // c : memR/memC/memV -> rowC,colC,valC |
| 1116 | Value nseC = NumberOfEntriesOp::create(builder&: rewriter, location: loc, tensor: c); |
| 1117 | Value szm = linalg::createOrFoldDimOp(b&: rewriter, loc, val: a, dim: 0); |
| 1118 | Value szk = linalg::createOrFoldDimOp(b&: rewriter, loc, val: a, dim: 1); |
| 1119 | Value szn = linalg::createOrFoldDimOp(b&: rewriter, loc, val: b, dim: 1); |
| 1120 | Value bufA = genTensorToMemref(rewriter, loc, tensor: a); |
| 1121 | Value matA = genAllocCopy(builder&: rewriter, loc, b: bufA, tokens); |
| 1122 | Value bufB = genTensorToMemref(rewriter, loc, tensor: b); |
| 1123 | Value matB = genAllocCopy(builder&: rewriter, loc, b: bufB, tokens); |
| 1124 | Value memR = genFirstPosOrCrds(builder&: rewriter, loc, a: c, format, enableRT); |
| 1125 | Value memC = genSecondCrds(builder&: rewriter, loc, a: c, format, enableRT); // or empty |
| 1126 | Value memV = ToValuesOp::create(builder&: rewriter, location: loc, tensor: c); |
| 1127 | Value rowC = genAllocCopy(builder&: rewriter, loc, b: memR, tokens); |
| 1128 | Value colC = memC ? genAllocCopy(builder&: rewriter, loc, b: memC, tokens) : Value(); |
| 1129 | Value valC = genAllocCopy(builder&: rewriter, loc, b: memV, tokens); |
| 1130 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 1131 | tokens.clear(); |
| 1132 | |
| 1133 | // Create sparse environment and sparse matrix/dense matrix handles. |
| 1134 | Type indexTp = rewriter.getIndexType(); |
| 1135 | Type dnMatHandleTp = rewriter.getType<gpu::SparseDnTensorHandleType>(); |
| 1136 | Type spMatHandleTp = rewriter.getType<gpu::SparseSpMatHandleType>(); |
| 1137 | Type tokenTp = rewriter.getType<gpu::AsyncTokenType>(); |
| 1138 | Value token = genFirstWait(builder&: rewriter, loc); |
| 1139 | auto dmatA = |
| 1140 | gpu::CreateDnTensorOp::create(builder&: rewriter, location: loc, dnTensor: dnMatHandleTp, asyncToken: tokenTp, |
| 1141 | asyncDependencies: token, memref: matA, dims: SmallVector<Value>{szm, szk}); |
| 1142 | Value dnA = dmatA.getResult(i: 0); |
| 1143 | token = dmatA.getAsyncToken(); |
| 1144 | auto dmatB = |
| 1145 | gpu::CreateDnTensorOp::create(builder&: rewriter, location: loc, dnTensor: dnMatHandleTp, asyncToken: tokenTp, |
| 1146 | asyncDependencies: token, memref: matB, dims: SmallVector<Value>{szk, szn}); |
| 1147 | Value dnB = dmatB.getResult(i: 0); |
| 1148 | token = dmatB.getAsyncToken(); |
| 1149 | Operation *spGenC = |
| 1150 | genSpMat(builder&: rewriter, loc, aTp&: cTp, handleTp: spMatHandleTp, tokenTp, token, sz1: szm, sz2: szn, |
| 1151 | nseA: nseC, rowA: rowC, colA: colC, valA: valC, format, enableRT); |
| 1152 | Value spMatC = spGenC->getResult(idx: 0); |
| 1153 | token = spGenC->getResult(idx: 1); |
| 1154 | auto dnCType = llvm::cast<ShapedType>(Val: c.getType()).getElementType(); |
| 1155 | |
| 1156 | // Precompute buffersize for SDDMM. |
| 1157 | auto bufferComp = gpu::SDDMMBufferSizeOp::create( |
| 1158 | builder&: rewriter, location: loc, bufferSz: indexTp, asyncToken: tokenTp, asyncDependencies: token, dnmatA: dnA, dnmatB: dnB, spmatC: spMatC, computeType: dnCType); |
| 1159 | Value bufferSz = bufferComp.getResult(i: 0); |
| 1160 | token = bufferComp.getAsyncToken(); |
| 1161 | auto buf = genAllocBuffer(builder&: rewriter, loc, size: bufferSz, token); |
| 1162 | Value buffer = buf.getResult(i: 0); |
| 1163 | token = buf.getAsyncToken(); |
| 1164 | |
| 1165 | // Perform the SDDMM. |
| 1166 | auto sddmmComp = gpu::SDDMMOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, dnmatA: dnA, dnmatB: dnB, |
| 1167 | spmatC: spMatC, computeType: dnCType, buffer); |
| 1168 | token = sddmmComp.getAsyncToken(); |
| 1169 | |
| 1170 | // Copy data back to host and free all the resoures. |
| 1171 | token = gpu::DestroyDnTensorOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, dnTensor: dnA) |
| 1172 | .getAsyncToken(); |
| 1173 | token = gpu::DestroyDnTensorOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, dnTensor: dnB) |
| 1174 | .getAsyncToken(); |
| 1175 | token = gpu::DestroySpMatOp::create(builder&: rewriter, location: loc, asyncToken: tokenTp, asyncDependencies: token, spmat: spMatC) |
| 1176 | .getAsyncToken(); |
| 1177 | token = genDeallocMemRef(builder&: rewriter, loc, mem: buffer, token); |
| 1178 | token = genDeallocMemRef(builder&: rewriter, loc, mem: matA, token); |
| 1179 | token = genDeallocMemRef(builder&: rewriter, loc, mem: matB, token); |
| 1180 | token = genDeallocMemRef(builder&: rewriter, loc, mem: rowC, token); |
| 1181 | if (colC) |
| 1182 | token = genDeallocMemRef(builder&: rewriter, loc, mem: colC, token); |
| 1183 | token = genCopyMemRef(builder&: rewriter, loc, dst: memV, src: valC, token); |
| 1184 | token = genDeallocMemRef(builder&: rewriter, loc, mem: valC, token); |
| 1185 | tokens.push_back(Elt: token); |
| 1186 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 1187 | tokens.clear(); |
| 1188 | |
| 1189 | // Done. |
| 1190 | rewriter.replaceOpWithNewOp<sparse_tensor::LoadOp>(op, args&: c); |
| 1191 | return success(); |
| 1192 | } |
| 1193 | |
| 1194 | //===----------------------------------------------------------------------===// |
| 1195 | // Rewriting rules for direct code generation. |
| 1196 | //===----------------------------------------------------------------------===// |
| 1197 | |
| 1198 | /// Proof-of-concept rewriter. This rule generates a GPU implementation |
| 1199 | /// for each outermost forall loop generated by the sparsifier. |
| 1200 | /// TODO: right now works with parallelization-strategy=dense-outer-loop |
| 1201 | /// but give this its own flags in the future |
| 1202 | struct ForallRewriter : public OpRewritePattern<scf::ParallelOp> { |
| 1203 | using OpRewritePattern<scf::ParallelOp>::OpRewritePattern; |
| 1204 | |
| 1205 | ForallRewriter(MLIRContext *context, unsigned nT) |
| 1206 | : OpRewritePattern(context), numThreads(nT) {}; |
| 1207 | |
| 1208 | LogicalResult matchAndRewrite(scf::ParallelOp forallOp, |
| 1209 | PatternRewriter &rewriter) const override { |
| 1210 | // Reject inadmissible loop form. |
| 1211 | // Essentially only accept a loop, generated by the sparsifier, |
| 1212 | // of the form |
| 1213 | // forall (i = 0; i < N; i++) |
| 1214 | // so that cyclic scheduling over the threads is easy. |
| 1215 | if (!forallOp->hasDiscardableAttr( |
| 1216 | name: LoopEmitter::getLoopEmitterLoopAttrName()) || |
| 1217 | forallOp.getNumReductions() != 0 || forallOp.getNumLoops() != 1 || |
| 1218 | !matchPattern(value: forallOp.getLowerBound()[0], pattern: m_Zero()) || |
| 1219 | !matchPattern(value: forallOp.getStep()[0], pattern: m_One())) |
| 1220 | return failure(); |
| 1221 | // Collect every value that is computed outside the parallel loop. |
| 1222 | SetVector<Value> invariants; // stable iteration! |
| 1223 | forallOp->walk(callback: [&](Operation *op) { |
| 1224 | // Collect all values of admissible ops. |
| 1225 | for (OpOperand &o : op->getOpOperands()) { |
| 1226 | Value val = o.get(); |
| 1227 | Block *block; |
| 1228 | if (auto arg = dyn_cast<BlockArgument>(Val&: val)) |
| 1229 | block = arg.getOwner(); |
| 1230 | else |
| 1231 | block = val.getDefiningOp()->getBlock(); |
| 1232 | if (!forallOp.getRegion().findAncestorBlockInRegion(block&: *block)) |
| 1233 | invariants.insert(X: val); |
| 1234 | } |
| 1235 | }); |
| 1236 | // Outline the outside values as proper parameters. Fail when sharing |
| 1237 | // value between host and device is not straightforward. |
| 1238 | SmallVector<Value> constants; |
| 1239 | SmallVector<Value> scalars; |
| 1240 | SmallVector<Value> buffers; |
| 1241 | // A boolean mask aligned 1:1 with the `buffers` array, tracking which |
| 1242 | // of those buffers were mutated by the loop. If true, the corresponding |
| 1243 | // buffer needs to be "copied back" using a device-to-host copy. |
| 1244 | SmallVector<bool> copyBack; |
| 1245 | for (Value val : invariants) { |
| 1246 | Type tp = val.getType(); |
| 1247 | if (val.getDefiningOp<arith::ConstantOp>()) |
| 1248 | constants.push_back(Elt: val); |
| 1249 | else if (isa<FloatType>(Val: tp) || tp.isIntOrIndex()) |
| 1250 | scalars.push_back(Elt: val); |
| 1251 | else if (isa<MemRefType>(Val: tp)) { |
| 1252 | buffers.push_back(Elt: val); |
| 1253 | |
| 1254 | // Determine if the buffer needs to be "copied back" from device |
| 1255 | // to host by checking for `memref.store` and the write memory effect. |
| 1256 | bool isWrite = false; |
| 1257 | for (Operation *user : val.getUsers()) { |
| 1258 | if (isa<memref::StoreOp>(Val: user)) { |
| 1259 | isWrite = true; |
| 1260 | break; |
| 1261 | } |
| 1262 | if (auto memInterface = dyn_cast<MemoryEffectOpInterface>(Val: user)) { |
| 1263 | if (memInterface.getEffectOnValue<MemoryEffects::Write>(value: val)) { |
| 1264 | isWrite = true; |
| 1265 | break; |
| 1266 | } |
| 1267 | } |
| 1268 | } |
| 1269 | copyBack.push_back(Elt: isWrite); |
| 1270 | } else |
| 1271 | return failure(); // don't know how to share |
| 1272 | } |
| 1273 | // Pass outlined non-constant values. |
| 1274 | // TODO: Experiment with `useHostRegistrationForOut` to see if we want to |
| 1275 | // keep the feature at all (either through a heuristic or compiler |
| 1276 | // option for gpu codegen). |
| 1277 | Location loc = forallOp->getLoc(); |
| 1278 | SmallVector<Value> args; |
| 1279 | SmallVector<Value> tokens; |
| 1280 | Value out = genParametersIn(builder&: rewriter, loc, scalars, buffers, args, tokens, |
| 1281 | /*useHostRegistrationForOut=*/false); |
| 1282 | // Set up GPU module and construct GPU function. |
| 1283 | auto saveIp = rewriter.saveInsertionPoint(); |
| 1284 | ModuleOp topModule = forallOp->getParentOfType<ModuleOp>(); |
| 1285 | auto gpuModule = genGPUModule(builder&: rewriter, topModule); |
| 1286 | auto gpuFunc = genGPUFunc(builder&: rewriter, gpuModule, args); |
| 1287 | genGPUCode(rewriter, gpuFunc, forallOp, constants, scalars, buffers); |
| 1288 | // Generate code that launches the kernel asynchronously, blocking on all |
| 1289 | // opens tokens and yielding a new token for the output. |
| 1290 | // TODO: Passing in tokens to launch up does not seem to be properly lowered |
| 1291 | // by cubin yet, hence the current blocking wait. |
| 1292 | rewriter.restoreInsertionPoint(ip: saveIp); |
| 1293 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 1294 | tokens.clear(); |
| 1295 | Value kernelToken = |
| 1296 | genLaunchGPUFunc(builder&: rewriter, gpuFunc, args, tokens, numThreads); |
| 1297 | // Finalize the outlined arguments. |
| 1298 | genParametersOut(builder&: rewriter, loc, out, kernelToken, scalars, buffers, args, |
| 1299 | tokens, copyBack); |
| 1300 | genBlockingWait(builder&: rewriter, loc, operands: tokens); |
| 1301 | rewriter.eraseOp(op: forallOp); |
| 1302 | return success(); |
| 1303 | } |
| 1304 | |
| 1305 | private: |
| 1306 | unsigned numThreads; |
| 1307 | }; |
| 1308 | |
| 1309 | //===----------------------------------------------------------------------===// |
| 1310 | // Rewriting rules for library recognition and code generation. |
| 1311 | //===----------------------------------------------------------------------===// |
| 1312 | |
| 1313 | /// Proof-of-concept rewriter. This rule recognizes certain math kernels |
| 1314 | /// and replaces these with corresponding calls into a sparse library. |
| 1315 | struct LinalgOpRewriter : public OpRewritePattern<linalg::GenericOp> { |
| 1316 | using OpRewritePattern<linalg::GenericOp>::OpRewritePattern; |
| 1317 | |
| 1318 | LinalgOpRewriter(MLIRContext *context, bool rt) |
| 1319 | : OpRewritePattern(context), enableRT(rt) {} |
| 1320 | |
| 1321 | LogicalResult matchAndRewrite(linalg::GenericOp op, |
| 1322 | PatternRewriter &rewriter) const override { |
| 1323 | if (op.getNumDpsInits() != 1) |
| 1324 | return failure(); // reject multi-output |
| 1325 | // The rewrites below query the sparse tensor encoding of the operands, |
| 1326 | // which is only defined for ranked tensor types. Reject buffer semantics |
| 1327 | // rather than casting a memref operand to a tensor type. |
| 1328 | if (!op.hasPureTensorSemantics()) |
| 1329 | return failure(); |
| 1330 | |
| 1331 | const unsigned numLoops = op.getNumLoops(); |
| 1332 | const unsigned numTensors = op->getNumOperands(); |
| 1333 | const auto iteratorTypes = op.getIteratorTypesArray(); |
| 1334 | SmallVector<AffineMap, 4> maps = op.getIndexingMapsArray(); |
| 1335 | |
| 1336 | using MapList = ArrayRef<ArrayRef<AffineExpr>>; |
| 1337 | auto infer = [&](MapList m) { |
| 1338 | return AffineMap::inferFromExprList(exprsList: m, context: op.getContext()); |
| 1339 | }; |
| 1340 | AffineExpr i, j, k; |
| 1341 | bindDims(ctx: getContext(), exprs&: i, exprs&: j, exprs&: k); |
| 1342 | |
| 1343 | // TODO: more robust patterns, transposed versions, more kernels, |
| 1344 | // identify alpha and beta and pass them to the CUDA calls. |
| 1345 | |
| 1346 | // Recognize a SpMV kernel. |
| 1347 | if (numLoops == 2 && numTensors == 3 && |
| 1348 | linalg::isParallelIterator(iteratorType: iteratorTypes[0]) && |
| 1349 | linalg::isReductionIterator(iteratorType: iteratorTypes[1]) && |
| 1350 | matchSumOfMultOfArgs(op)) { |
| 1351 | if (maps == infer({{i, j}, {j}, {i}})) |
| 1352 | return rewriteSpMV(rewriter, op, enableRT, |
| 1353 | modeA: gpu::TransposeMode::NON_TRANSPOSE); |
| 1354 | // TODO: recognize conjugate transpose patterns for SpMV. |
| 1355 | if (maps == infer({{j, i}, {j}, {i}})) |
| 1356 | return rewriteSpMV(rewriter, op, enableRT, |
| 1357 | modeA: gpu::TransposeMode::TRANSPOSE); |
| 1358 | } |
| 1359 | |
| 1360 | // Recognize a SpGEMM, 2:4-SpMM, or SpMM kernel. |
| 1361 | if (numLoops == 3 && numTensors == 3 && |
| 1362 | linalg::isParallelIterator(iteratorType: iteratorTypes[0]) && |
| 1363 | linalg::isParallelIterator(iteratorType: iteratorTypes[1]) && |
| 1364 | linalg::isReductionIterator(iteratorType: iteratorTypes[2]) && |
| 1365 | matchSumOfMultOfArgs(op)) { |
| 1366 | if (maps == infer({{i, k}, {k, j}, {i, j}})) { |
| 1367 | if (!isDenseTensor(v: op.getOperand(i: 0)) && |
| 1368 | !isDenseTensor(v: op.getOperand(i: 1))) |
| 1369 | return rewriteSpGEMM(rewriter, op, enableRT); |
| 1370 | if (isConversionInto24(v: op.getOperand(i: 0))) |
| 1371 | return rewrite2To4SpMM(rewriter, op); |
| 1372 | return rewriteSpMM(rewriter, op, enableRT, |
| 1373 | modeA: gpu::TransposeMode::NON_TRANSPOSE, |
| 1374 | modeB: gpu::TransposeMode::NON_TRANSPOSE); |
| 1375 | } |
| 1376 | // TODO: recognize conjugate transpose patterns for either SpMM input. |
| 1377 | if (maps == infer({{k, i}, {k, j}, {i, j}})) |
| 1378 | return rewriteSpMM(rewriter, op, enableRT, |
| 1379 | modeA: gpu::TransposeMode::TRANSPOSE, |
| 1380 | modeB: gpu::TransposeMode::NON_TRANSPOSE); |
| 1381 | if (maps == infer({{i, k}, {j, k}, {i, j}})) |
| 1382 | return rewriteSpMM(rewriter, op, enableRT, |
| 1383 | modeA: gpu::TransposeMode::NON_TRANSPOSE, |
| 1384 | modeB: gpu::TransposeMode::TRANSPOSE); |
| 1385 | if (maps == infer({{k, i}, {j, k}, {i, j}})) |
| 1386 | return rewriteSpMM(rewriter, op, enableRT, |
| 1387 | modeA: gpu::TransposeMode::TRANSPOSE, |
| 1388 | modeB: gpu::TransposeMode::TRANSPOSE); |
| 1389 | } |
| 1390 | |
| 1391 | // Recognize a SDDMM kernel. |
| 1392 | if (numLoops == 3 && numTensors == 3 && |
| 1393 | linalg::isParallelIterator(iteratorType: iteratorTypes[0]) && |
| 1394 | linalg::isParallelIterator(iteratorType: iteratorTypes[1]) && |
| 1395 | linalg::isReductionIterator(iteratorType: iteratorTypes[2]) && |
| 1396 | maps == infer({{i, k}, {k, j}, {i, j}}) && |
| 1397 | matchSumReductionOfMulUnary(op)) { |
| 1398 | return rewriteSDDMM(rewriter, op, enableRT); |
| 1399 | } |
| 1400 | |
| 1401 | return failure(); |
| 1402 | } |
| 1403 | |
| 1404 | private: |
| 1405 | bool enableRT; |
| 1406 | }; |
| 1407 | |
| 1408 | } // namespace |
| 1409 | |
| 1410 | //===----------------------------------------------------------------------===// |
| 1411 | // Public method for populating GPU rewriting rules. |
| 1412 | // |
| 1413 | // Currently two set of rewriting rules are made available. The first set |
| 1414 | // implements direct code generation, currently by means of convering the |
| 1415 | // outermost paralell loop into GPU threads. The second set implements |
| 1416 | // libary recognition of a set of sparse operations. Eventually, the right |
| 1417 | // combination of these two approaches has to be found. |
| 1418 | //===----------------------------------------------------------------------===// |
| 1419 | |
| 1420 | void mlir::populateSparseGPUCodegenPatterns(RewritePatternSet &patterns, |
| 1421 | unsigned numThreads) { |
| 1422 | patterns.add<ForallRewriter>(arg: patterns.getContext(), args&: numThreads); |
| 1423 | } |
| 1424 | |
| 1425 | void mlir::populateSparseGPULibgenPatterns(RewritePatternSet &patterns, |
| 1426 | bool enableRT) { |
| 1427 | patterns.add<LinalgOpRewriter>(arg: patterns.getContext(), args&: enableRT); |
| 1428 | } |
| 1429 | |