| 1 | //===- SCFToSPIRV.cpp - SCF to SPIR-V Patterns ----------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements patterns to convert SCF dialect to SPIR-V dialect. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h" |
| 14 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 15 | #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h" |
| 16 | #include "mlir/Dialect/SPIRV/IR/TargetAndABI.h" |
| 17 | #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h" |
| 18 | #include "mlir/Transforms/DialectConversion.h" |
| 19 | #include "llvm/Support/FormatVariadic.h" |
| 20 | |
| 21 | using namespace mlir; |
| 22 | |
| 23 | //===----------------------------------------------------------------------===// |
| 24 | // Context |
| 25 | //===----------------------------------------------------------------------===// |
| 26 | |
| 27 | namespace mlir { |
| 28 | struct ScfToSPIRVContextImpl { |
| 29 | // Map between the spirv region control flow operation (spirv.mlir.loop or |
| 30 | // spirv.mlir.selection) to the VariableOp created to store the region |
| 31 | // results. The order of the VariableOp matches the order of the results. |
| 32 | DenseMap<Operation *, SmallVector<spirv::VariableOp, 8>> outputVars; |
| 33 | }; |
| 34 | } // namespace mlir |
| 35 | |
| 36 | /// We use ScfToSPIRVContext to store information about the lowering of the scf |
| 37 | /// region that need to be used later on. When we lower scf.for/scf.if we create |
| 38 | /// VariableOp to store the results. We need to keep track of the VariableOp |
| 39 | /// created as we need to insert stores into them when lowering Yield. Those |
| 40 | /// StoreOp cannot be created earlier as they may use a different type than |
| 41 | /// yield operands. |
| 42 | ScfToSPIRVContext::ScfToSPIRVContext() { |
| 43 | impl = std::make_unique<::ScfToSPIRVContextImpl>(); |
| 44 | } |
| 45 | |
| 46 | ScfToSPIRVContext::~ScfToSPIRVContext() = default; |
| 47 | |
| 48 | namespace { |
| 49 | |
| 50 | //===----------------------------------------------------------------------===// |
| 51 | // Helper Functions |
| 52 | //===----------------------------------------------------------------------===// |
| 53 | |
| 54 | /// Replaces SCF op outputs with SPIR-V variable loads. |
| 55 | /// We create VariableOp to handle the results value of the control flow region. |
| 56 | /// spirv.mlir.loop/spirv.mlir.selection currently don't yield value. Right |
| 57 | /// after the loop we load the value from the allocation and use it as the SCF |
| 58 | /// op result. |
| 59 | template <typename ScfOp, typename OpTy> |
| 60 | void replaceSCFOutputValue(ScfOp scfOp, OpTy newOp, |
| 61 | ConversionPatternRewriter &rewriter, |
| 62 | ScfToSPIRVContextImpl *scfToSPIRVContext, |
| 63 | ArrayRef<Type> returnTypes) { |
| 64 | |
| 65 | Location loc = scfOp.getLoc(); |
| 66 | auto &allocas = scfToSPIRVContext->outputVars[newOp]; |
| 67 | // Clearing the allocas is necessary in case a dialect conversion path failed |
| 68 | // previously, and this is the second attempt of this conversion. |
| 69 | allocas.clear(); |
| 70 | SmallVector<Value, 8> resultValue; |
| 71 | for (Type convertedType : returnTypes) { |
| 72 | auto pointerType = |
| 73 | spirv::PointerType::get(pointeeType: convertedType, storageClass: spirv::StorageClass::Function); |
| 74 | rewriter.setInsertionPoint(newOp); |
| 75 | auto alloc = spirv::VariableOp::create(builder&: rewriter, location: loc, pointer: pointerType, |
| 76 | storage_class: spirv::StorageClass::Function, |
| 77 | /*initializer=*/nullptr); |
| 78 | allocas.push_back(alloc); |
| 79 | rewriter.setInsertionPointAfter(newOp); |
| 80 | Value loadResult = spirv::LoadOp::create(builder&: rewriter, location: loc, basePtr: alloc); |
| 81 | resultValue.push_back(Elt: loadResult); |
| 82 | } |
| 83 | rewriter.replaceOp(scfOp, resultValue); |
| 84 | } |
| 85 | |
| 86 | Region::iterator getBlockIt(Region ®ion, unsigned index) { |
| 87 | return std::next(x: region.begin(), n: index); |
| 88 | } |
| 89 | |
| 90 | //===----------------------------------------------------------------------===// |
| 91 | // Conversion Patterns |
| 92 | //===----------------------------------------------------------------------===// |
| 93 | |
| 94 | /// Common class for all vector to GPU patterns. |
| 95 | template <typename OpTy> |
| 96 | class SCFToSPIRVPattern : public OpConversionPattern<OpTy> { |
| 97 | public: |
| 98 | SCFToSPIRVPattern(MLIRContext *context, const SPIRVTypeConverter &converter, |
| 99 | ScfToSPIRVContextImpl *scfToSPIRVContext) |
| 100 | : OpConversionPattern<OpTy>::OpConversionPattern(converter, context), |
| 101 | scfToSPIRVContext(scfToSPIRVContext), typeConverter(converter) {} |
| 102 | |
| 103 | protected: |
| 104 | ScfToSPIRVContextImpl *scfToSPIRVContext; |
| 105 | // FIXME: We explicitly keep a reference of the type converter here instead of |
| 106 | // passing it to OpConversionPattern during construction. This effectively |
| 107 | // bypasses the conversion framework's automation on type conversion. This is |
| 108 | // needed right now because the conversion framework will unconditionally |
| 109 | // legalize all types used by SCF ops upon discovering them, for example, the |
| 110 | // types of loop carried values. We use SPIR-V variables for those loop |
| 111 | // carried values. Depending on the available capabilities, the SPIR-V |
| 112 | // variable can be different, for example, cooperative matrix or normal |
| 113 | // variable. We'd like to detach the conversion of the loop carried values |
| 114 | // from the SCF ops (which is mainly a region). So we need to "mark" types |
| 115 | // used by SCF ops as legal, if to use the conversion framework for type |
| 116 | // conversion. There isn't a straightforward way to do that yet, as when |
| 117 | // converting types, ops aren't taken into consideration. Therefore, we just |
| 118 | // bypass the framework's type conversion for now. |
| 119 | const SPIRVTypeConverter &typeConverter; |
| 120 | }; |
| 121 | |
| 122 | //===----------------------------------------------------------------------===// |
| 123 | // scf::ForOp |
| 124 | //===----------------------------------------------------------------------===// |
| 125 | |
| 126 | /// Pattern to convert a scf::ForOp within kernel functions into spirv::LoopOp. |
| 127 | struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> { |
| 128 | using SCFToSPIRVPattern::SCFToSPIRVPattern; |
| 129 | |
| 130 | LogicalResult |
| 131 | matchAndRewrite(scf::ForOp forOp, OpAdaptor adaptor, |
| 132 | ConversionPatternRewriter &rewriter) const override { |
| 133 | // scf::ForOp can be lowered to the structured control flow represented by |
| 134 | // spirv::LoopOp by making the continue block of the spirv::LoopOp the loop |
| 135 | // latch and the merge block the exit block. The resulting spirv::LoopOp has |
| 136 | // a single back edge from the continue to header block, and a single exit |
| 137 | // from header to merge. |
| 138 | auto loc = forOp.getLoc(); |
| 139 | auto loopControl = spirv::LoopControl::None; |
| 140 | if (auto attr = forOp->getDiscardableAttrOfType<spirv::LoopControlAttr>( |
| 141 | name: spirv::getLoopControlAttrName())) |
| 142 | loopControl = attr.getValue(); |
| 143 | auto loopOp = spirv::LoopOp::create(builder&: rewriter, location: loc, loopControl); |
| 144 | loopOp.addEntryAndMergeBlock(builder&: rewriter); |
| 145 | |
| 146 | OpBuilder::InsertionGuard guard(rewriter); |
| 147 | // Create the block for the header. |
| 148 | Block * = rewriter.createBlock(parent: &loopOp.getBody(), |
| 149 | insertPt: getBlockIt(region&: loopOp.getBody(), index: 1)); |
| 150 | rewriter.setInsertionPointAfter(loopOp); |
| 151 | |
| 152 | // Create the new induction variable to use. |
| 153 | Value adapLowerBound = adaptor.getLowerBound(); |
| 154 | BlockArgument newIndVar = |
| 155 | header->addArgument(type: adapLowerBound.getType(), loc: adapLowerBound.getLoc()); |
| 156 | for (Value arg : adaptor.getInitArgs()) |
| 157 | header->addArgument(type: arg.getType(), loc: arg.getLoc()); |
| 158 | Block *body = forOp.getBody(); |
| 159 | |
| 160 | // Apply signature conversion to the body of the forOp. It has a single |
| 161 | // block, with argument which is the induction variable. That has to be |
| 162 | // replaced with the new induction variable. |
| 163 | TypeConverter::SignatureConversion signatureConverter( |
| 164 | body->getNumArguments()); |
| 165 | signatureConverter.remapInput(origInputNo: 0, replacements: newIndVar); |
| 166 | for (unsigned i = 1, e = body->getNumArguments(); i < e; i++) |
| 167 | signatureConverter.remapInput(origInputNo: i, replacements: header->getArgument(i)); |
| 168 | body = rewriter.applySignatureConversion(block: &forOp.getRegion().front(), |
| 169 | conversion&: signatureConverter); |
| 170 | |
| 171 | // Move the blocks from the forOp into the loopOp. This is the body of the |
| 172 | // loopOp. |
| 173 | rewriter.inlineRegionBefore(region&: forOp->getRegion(index: 0), parent&: loopOp.getBody(), |
| 174 | before: getBlockIt(region&: loopOp.getBody(), index: 2)); |
| 175 | |
| 176 | SmallVector<Value, 8> args(1, adaptor.getLowerBound()); |
| 177 | args.append(in_start: adaptor.getInitArgs().begin(), in_end: adaptor.getInitArgs().end()); |
| 178 | // Branch into it from the entry. |
| 179 | rewriter.setInsertionPointToEnd(&(loopOp.getBody().front())); |
| 180 | spirv::BranchOp::create(builder&: rewriter, location: loc, successor: header, arguments: args); |
| 181 | |
| 182 | // Generate the rest of the loop header. |
| 183 | rewriter.setInsertionPointToEnd(header); |
| 184 | auto *mergeBlock = loopOp.getMergeBlock(); |
| 185 | Value cmpOp; |
| 186 | if (forOp.getUnsignedCmp()) { |
| 187 | cmpOp = spirv::ULessThanOp::create(builder&: rewriter, location: loc, result: rewriter.getI1Type(), |
| 188 | operand1: newIndVar, operand2: adaptor.getUpperBound()); |
| 189 | } else { |
| 190 | cmpOp = spirv::SLessThanOp::create(builder&: rewriter, location: loc, result: rewriter.getI1Type(), |
| 191 | operand1: newIndVar, operand2: adaptor.getUpperBound()); |
| 192 | } |
| 193 | |
| 194 | spirv::BranchConditionalOp::create(builder&: rewriter, location: loc, condition: cmpOp, trueBlock: body, |
| 195 | trueArguments: ArrayRef<Value>(), falseBlock: mergeBlock, |
| 196 | falseArguments: ArrayRef<Value>()); |
| 197 | |
| 198 | // Generate instructions to increment the step of the induction variable and |
| 199 | // branch to the header. |
| 200 | Block *continueBlock = loopOp.getContinueBlock(); |
| 201 | rewriter.setInsertionPointToEnd(continueBlock); |
| 202 | |
| 203 | // Add the step to the induction variable and branch to the header. |
| 204 | Value updatedIndVar = spirv::IAddOp::create( |
| 205 | builder&: rewriter, location: loc, result: newIndVar.getType(), operand1: newIndVar, operand2: adaptor.getStep()); |
| 206 | spirv::BranchOp::create(builder&: rewriter, location: loc, successor: header, arguments: updatedIndVar); |
| 207 | |
| 208 | // Infer the return types from the init operands. Vector type may get |
| 209 | // converted to CooperativeMatrix or to Vector type, to avoid having complex |
| 210 | // extra logic to figure out the right type we just infer it from the Init |
| 211 | // operands. |
| 212 | SmallVector<Type, 8> initTypes; |
| 213 | for (auto arg : adaptor.getInitArgs()) |
| 214 | initTypes.push_back(Elt: arg.getType()); |
| 215 | replaceSCFOutputValue(scfOp: forOp, newOp: loopOp, rewriter, scfToSPIRVContext, |
| 216 | returnTypes: initTypes); |
| 217 | |
| 218 | // Store init values so a zero-trip loop returns them instead of undef. |
| 219 | // Skip the stores if the loop is known to always execute at least once. |
| 220 | std::optional<APInt> tripCount = forOp.getStaticTripCount(); |
| 221 | if (!tripCount || tripCount->isZero()) { |
| 222 | auto &allocas = scfToSPIRVContext->outputVars[loopOp]; |
| 223 | rewriter.setInsertionPoint(loopOp); |
| 224 | for (auto [alloca, init] : llvm::zip(t&: allocas, u: adaptor.getInitArgs())) |
| 225 | spirv::StoreOp::create(builder&: rewriter, location: loc, ptr: alloca, value: init); |
| 226 | } |
| 227 | return success(); |
| 228 | } |
| 229 | }; |
| 230 | |
| 231 | //===----------------------------------------------------------------------===// |
| 232 | // scf::IfOp |
| 233 | //===----------------------------------------------------------------------===// |
| 234 | |
| 235 | /// Pattern to convert a scf::IfOp within kernel functions into |
| 236 | /// spirv::SelectionOp. |
| 237 | struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> { |
| 238 | using SCFToSPIRVPattern::SCFToSPIRVPattern; |
| 239 | |
| 240 | LogicalResult |
| 241 | matchAndRewrite(scf::IfOp ifOp, OpAdaptor adaptor, |
| 242 | ConversionPatternRewriter &rewriter) const override { |
| 243 | // When lowering `scf::IfOp` we explicitly create a selection header block |
| 244 | // before the control flow diverges and a merge block where control flow |
| 245 | // subsequently converges. |
| 246 | auto loc = ifOp.getLoc(); |
| 247 | |
| 248 | // Compute return types. |
| 249 | SmallVector<Type, 8> returnTypes; |
| 250 | for (auto result : ifOp.getResults()) { |
| 251 | auto convertedType = typeConverter.convertType(t: result.getType()); |
| 252 | if (!convertedType) |
| 253 | return rewriter.notifyMatchFailure( |
| 254 | arg&: loc, |
| 255 | msg: llvm::formatv(Fmt: "failed to convert type '{0}'" , Vals: result.getType())); |
| 256 | |
| 257 | returnTypes.push_back(Elt: convertedType); |
| 258 | } |
| 259 | |
| 260 | // Create `spirv.selection` operation, selection header block and merge |
| 261 | // block. |
| 262 | auto selectionControl = spirv::SelectionControl::None; |
| 263 | if (auto attr = ifOp->getDiscardableAttrOfType<spirv::SelectionControlAttr>( |
| 264 | name: spirv::getSelectionControlAttrName())) |
| 265 | selectionControl = attr.getValue(); |
| 266 | auto selectionOp = |
| 267 | spirv::SelectionOp::create(builder&: rewriter, location: loc, selectionControl); |
| 268 | auto *mergeBlock = rewriter.createBlock(parent: &selectionOp.getBody(), |
| 269 | insertPt: selectionOp.getBody().end()); |
| 270 | spirv::MergeOp::create(builder&: rewriter, location: loc); |
| 271 | |
| 272 | OpBuilder::InsertionGuard guard(rewriter); |
| 273 | auto * = |
| 274 | rewriter.createBlock(insertBefore: &selectionOp.getBody().front()); |
| 275 | |
| 276 | // Inline `then` region before the merge block and branch to it. |
| 277 | auto &thenRegion = ifOp.getThenRegion(); |
| 278 | auto *thenBlock = &thenRegion.front(); |
| 279 | rewriter.setInsertionPointToEnd(&thenRegion.back()); |
| 280 | spirv::BranchOp::create(builder&: rewriter, location: loc, successor: mergeBlock); |
| 281 | rewriter.inlineRegionBefore(region&: thenRegion, before: mergeBlock); |
| 282 | |
| 283 | auto *elseBlock = mergeBlock; |
| 284 | // If `else` region is not empty, inline that region before the merge block |
| 285 | // and branch to it. |
| 286 | if (!ifOp.getElseRegion().empty()) { |
| 287 | auto &elseRegion = ifOp.getElseRegion(); |
| 288 | elseBlock = &elseRegion.front(); |
| 289 | rewriter.setInsertionPointToEnd(&elseRegion.back()); |
| 290 | spirv::BranchOp::create(builder&: rewriter, location: loc, successor: mergeBlock); |
| 291 | rewriter.inlineRegionBefore(region&: elseRegion, before: mergeBlock); |
| 292 | } |
| 293 | |
| 294 | // Create a `spirv.BranchConditional` operation for selection header block. |
| 295 | rewriter.setInsertionPointToEnd(selectionHeaderBlock); |
| 296 | spirv::BranchConditionalOp::create(builder&: rewriter, location: loc, condition: adaptor.getCondition(), |
| 297 | trueBlock: thenBlock, trueArguments: ArrayRef<Value>(), falseBlock: elseBlock, |
| 298 | falseArguments: ArrayRef<Value>()); |
| 299 | |
| 300 | replaceSCFOutputValue(scfOp: ifOp, newOp: selectionOp, rewriter, scfToSPIRVContext, |
| 301 | returnTypes); |
| 302 | return success(); |
| 303 | } |
| 304 | }; |
| 305 | |
| 306 | //===----------------------------------------------------------------------===// |
| 307 | // scf::IndexSwitchOp |
| 308 | //===----------------------------------------------------------------------===// |
| 309 | |
| 310 | /// Pattern to convert a scf::IndexSwitchOp within kernel functions into |
| 311 | /// spirv::SelectionOp with a spirv::SwitchOp header. |
| 312 | struct IndexSwitchOpConversion final : SCFToSPIRVPattern<scf::IndexSwitchOp> { |
| 313 | using SCFToSPIRVPattern::SCFToSPIRVPattern; |
| 314 | |
| 315 | LogicalResult |
| 316 | matchAndRewrite(scf::IndexSwitchOp switchOp, OpAdaptor adaptor, |
| 317 | ConversionPatternRewriter &rewriter) const override { |
| 318 | Location loc = switchOp.getLoc(); |
| 319 | |
| 320 | // Compute return types. |
| 321 | SmallVector<Type, 8> returnTypes; |
| 322 | for (Value result : switchOp.getResults()) { |
| 323 | Type convertedType = typeConverter.convertType(t: result.getType()); |
| 324 | if (!convertedType) |
| 325 | return rewriter.notifyMatchFailure( |
| 326 | arg&: loc, |
| 327 | msg: llvm::formatv(Fmt: "failed to convert type '{0}'" , Vals: result.getType())); |
| 328 | returnTypes.push_back(Elt: convertedType); |
| 329 | } |
| 330 | |
| 331 | // The selector must be a SPIR-V integer; spirv.Switch literals are |
| 332 | // interpreted with the selector's bit width. |
| 333 | Value selector = adaptor.getArg(); |
| 334 | auto selectorType = dyn_cast<IntegerType>(Val: selector.getType()); |
| 335 | if (!selectorType) |
| 336 | return rewriter.notifyMatchFailure(arg&: loc, |
| 337 | msg: "selector type is not an integer" ); |
| 338 | unsigned selectorWidth = selectorType.getWidth(); |
| 339 | |
| 340 | // Create the `spirv.mlir.selection` op, its header block, and merge block. |
| 341 | auto selectionControl = spirv::SelectionControl::None; |
| 342 | if (auto attr = |
| 343 | switchOp->getDiscardableAttrOfType<spirv::SelectionControlAttr>( |
| 344 | name: spirv::getSelectionControlAttrName())) |
| 345 | selectionControl = attr.getValue(); |
| 346 | auto selectionOp = |
| 347 | spirv::SelectionOp::create(builder&: rewriter, location: loc, selectionControl); |
| 348 | auto *mergeBlock = rewriter.createBlock(parent: &selectionOp.getBody(), |
| 349 | insertPt: selectionOp.getBody().end()); |
| 350 | spirv::MergeOp::create(builder&: rewriter, location: loc); |
| 351 | |
| 352 | OpBuilder::InsertionGuard guard(rewriter); |
| 353 | auto * = rewriter.createBlock(insertBefore: &selectionOp.getBody().front()); |
| 354 | |
| 355 | // Inline each case region before the merge block and branch to it. |
| 356 | SmallVector<APInt> caseLiterals; |
| 357 | SmallVector<Block *> caseBlocks; |
| 358 | ArrayRef<int64_t> cases = switchOp.getCases(); |
| 359 | for (auto [caseValue, caseRegion] : |
| 360 | llvm::zip_equal(t&: cases, u: switchOp.getCaseRegions())) { |
| 361 | Block *caseBlock = &caseRegion.front(); |
| 362 | rewriter.setInsertionPointToEnd(&caseRegion.back()); |
| 363 | spirv::BranchOp::create(builder&: rewriter, location: loc, successor: mergeBlock); |
| 364 | rewriter.inlineRegionBefore(region&: caseRegion, before: mergeBlock); |
| 365 | caseLiterals.push_back( |
| 366 | Elt: APInt(selectorWidth, caseValue, /*isSigned=*/true)); |
| 367 | caseBlocks.push_back(Elt: caseBlock); |
| 368 | } |
| 369 | |
| 370 | // Inline the default region before the merge block and branch to it. |
| 371 | Region &defaultRegion = switchOp.getDefaultRegion(); |
| 372 | Block *defaultBlock = &defaultRegion.front(); |
| 373 | rewriter.setInsertionPointToEnd(&defaultRegion.back()); |
| 374 | spirv::BranchOp::create(builder&: rewriter, location: loc, successor: mergeBlock); |
| 375 | rewriter.inlineRegionBefore(region&: defaultRegion, before: mergeBlock); |
| 376 | |
| 377 | // Create the `spirv.Switch` terminator for the header block. The case |
| 378 | // regions carry their results through variables, so the branches take no |
| 379 | // operands. |
| 380 | SmallVector<ValueRange> caseOperands(caseBlocks.size(), ValueRange()); |
| 381 | rewriter.setInsertionPointToEnd(headerBlock); |
| 382 | spirv::SwitchOp::create(builder&: rewriter, location: loc, selector, defaultTarget: defaultBlock, defaultOperands: ValueRange(), |
| 383 | literals: caseLiterals, targets: caseBlocks, targetOperands: caseOperands); |
| 384 | |
| 385 | replaceSCFOutputValue(scfOp: switchOp, newOp: selectionOp, rewriter, scfToSPIRVContext, |
| 386 | returnTypes); |
| 387 | return success(); |
| 388 | } |
| 389 | }; |
| 390 | |
| 391 | //===----------------------------------------------------------------------===// |
| 392 | // scf::YieldOp |
| 393 | //===----------------------------------------------------------------------===// |
| 394 | |
| 395 | struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> { |
| 396 | public: |
| 397 | using SCFToSPIRVPattern::SCFToSPIRVPattern; |
| 398 | |
| 399 | LogicalResult |
| 400 | matchAndRewrite(scf::YieldOp terminatorOp, OpAdaptor adaptor, |
| 401 | ConversionPatternRewriter &rewriter) const override { |
| 402 | ValueRange operands = adaptor.getOperands(); |
| 403 | |
| 404 | Operation *parent = terminatorOp->getParentOp(); |
| 405 | |
| 406 | // TODO: Implement conversion for the remaining `scf` ops. |
| 407 | if (parent->getDialect()->getNamespace() == |
| 408 | scf::SCFDialect::getDialectNamespace() && |
| 409 | !isa<scf::IfOp, scf::ForOp, scf::WhileOp, scf::IndexSwitchOp>(Val: parent)) |
| 410 | return rewriter.notifyMatchFailure( |
| 411 | arg&: terminatorOp, |
| 412 | msg: llvm::formatv(Fmt: "conversion not supported for parent op: '{0}'" , |
| 413 | Vals: parent->getName())); |
| 414 | |
| 415 | // If the region return values, store each value into the associated |
| 416 | // VariableOp created during lowering of the parent region. |
| 417 | if (!operands.empty()) { |
| 418 | auto &allocas = scfToSPIRVContext->outputVars[parent]; |
| 419 | if (allocas.size() != operands.size()) |
| 420 | return failure(); |
| 421 | |
| 422 | auto loc = terminatorOp.getLoc(); |
| 423 | for (unsigned i = 0, e = operands.size(); i < e; i++) |
| 424 | spirv::StoreOp::create(builder&: rewriter, location: loc, ptr: allocas[i], value: operands[i]); |
| 425 | if (isa<spirv::LoopOp>(Val: parent)) { |
| 426 | // For loops we also need to update the branch jumping back to the |
| 427 | // header. |
| 428 | auto br = cast<spirv::BranchOp>( |
| 429 | Val: rewriter.getInsertionBlock()->getTerminator()); |
| 430 | SmallVector<Value, 8> args(br.getBlockArguments()); |
| 431 | args.append(in_start: operands.begin(), in_end: operands.end()); |
| 432 | rewriter.setInsertionPoint(br); |
| 433 | spirv::BranchOp::create(builder&: rewriter, location: terminatorOp.getLoc(), successor: br.getTarget(), |
| 434 | arguments: args); |
| 435 | rewriter.eraseOp(op: br); |
| 436 | } |
| 437 | } |
| 438 | rewriter.eraseOp(op: terminatorOp); |
| 439 | return success(); |
| 440 | } |
| 441 | }; |
| 442 | |
| 443 | //===----------------------------------------------------------------------===// |
| 444 | // scf::WhileOp |
| 445 | //===----------------------------------------------------------------------===// |
| 446 | |
| 447 | struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> { |
| 448 | using SCFToSPIRVPattern::SCFToSPIRVPattern; |
| 449 | |
| 450 | LogicalResult |
| 451 | matchAndRewrite(scf::WhileOp whileOp, OpAdaptor adaptor, |
| 452 | ConversionPatternRewriter &rewriter) const override { |
| 453 | auto loc = whileOp.getLoc(); |
| 454 | auto loopControl = spirv::LoopControl::None; |
| 455 | if (auto attr = whileOp->getDiscardableAttrOfType<spirv::LoopControlAttr>( |
| 456 | name: spirv::getLoopControlAttrName())) |
| 457 | loopControl = attr.getValue(); |
| 458 | auto loopOp = spirv::LoopOp::create(builder&: rewriter, location: loc, loopControl); |
| 459 | loopOp.addEntryAndMergeBlock(builder&: rewriter); |
| 460 | |
| 461 | Region &beforeRegion = whileOp.getBefore(); |
| 462 | Region &afterRegion = whileOp.getAfter(); |
| 463 | |
| 464 | if (failed(Result: rewriter.convertRegionTypes(region: &beforeRegion, converter: typeConverter)) || |
| 465 | failed(Result: rewriter.convertRegionTypes(region: &afterRegion, converter: typeConverter))) |
| 466 | return rewriter.notifyMatchFailure(arg&: whileOp, |
| 467 | msg: "Failed to convert region types" ); |
| 468 | |
| 469 | OpBuilder::InsertionGuard guard(rewriter); |
| 470 | |
| 471 | Block &entryBlock = *loopOp.getEntryBlock(); |
| 472 | Block &beforeBlock = beforeRegion.front(); |
| 473 | Block &afterBlock = afterRegion.front(); |
| 474 | Block &mergeBlock = *loopOp.getMergeBlock(); |
| 475 | |
| 476 | auto cond = cast<scf::ConditionOp>(Val: beforeBlock.getTerminator()); |
| 477 | SmallVector<Value> condArgs; |
| 478 | if (failed(Result: rewriter.getRemappedValues(keys: cond.getArgs(), results&: condArgs))) |
| 479 | return failure(); |
| 480 | |
| 481 | Value conditionVal = rewriter.getRemappedValue(key: cond.getCondition()); |
| 482 | if (!conditionVal) |
| 483 | return failure(); |
| 484 | |
| 485 | auto yield = cast<scf::YieldOp>(Val: afterBlock.getTerminator()); |
| 486 | SmallVector<Value> yieldArgs; |
| 487 | if (failed(Result: rewriter.getRemappedValues(keys: yield.getResults(), results&: yieldArgs))) |
| 488 | return failure(); |
| 489 | |
| 490 | // Move the while before block as the initial loop header block. |
| 491 | rewriter.inlineRegionBefore(region&: beforeRegion, parent&: loopOp.getBody(), |
| 492 | before: getBlockIt(region&: loopOp.getBody(), index: 1)); |
| 493 | |
| 494 | // Move the while after block as the initial loop body block. |
| 495 | rewriter.inlineRegionBefore(region&: afterRegion, parent&: loopOp.getBody(), |
| 496 | before: getBlockIt(region&: loopOp.getBody(), index: 2)); |
| 497 | |
| 498 | // Jump from the loop entry block to the loop header block. |
| 499 | rewriter.setInsertionPointToEnd(&entryBlock); |
| 500 | spirv::BranchOp::create(builder&: rewriter, location: loc, successor: &beforeBlock, arguments: adaptor.getInits()); |
| 501 | |
| 502 | auto condLoc = cond.getLoc(); |
| 503 | |
| 504 | SmallVector<Value> resultValues(condArgs.size()); |
| 505 | |
| 506 | // For other SCF ops, the scf.yield op yields the value for the whole SCF |
| 507 | // op. So we use the scf.yield op as the anchor to create/load/store SPIR-V |
| 508 | // local variables. But for the scf.while op, the scf.yield op yields a |
| 509 | // value for the before region, which may not matching the whole op's |
| 510 | // result. Instead, the scf.condition op returns values matching the whole |
| 511 | // op's results. So we need to create/load/store variables according to |
| 512 | // that. |
| 513 | for (const auto &it : llvm::enumerate(First&: condArgs)) { |
| 514 | auto res = it.value(); |
| 515 | auto i = it.index(); |
| 516 | auto pointerType = |
| 517 | spirv::PointerType::get(pointeeType: res.getType(), storageClass: spirv::StorageClass::Function); |
| 518 | |
| 519 | // Create local variables before the scf.while op. |
| 520 | rewriter.setInsertionPoint(loopOp); |
| 521 | auto alloc = spirv::VariableOp::create(builder&: rewriter, location: condLoc, pointer: pointerType, |
| 522 | storage_class: spirv::StorageClass::Function, |
| 523 | /*initializer=*/nullptr); |
| 524 | |
| 525 | // Load the final result values after the scf.while op. |
| 526 | rewriter.setInsertionPointAfter(loopOp); |
| 527 | auto loadResult = spirv::LoadOp::create(builder&: rewriter, location: condLoc, basePtr: alloc); |
| 528 | resultValues[i] = loadResult; |
| 529 | |
| 530 | // Store the current iteration's result value. |
| 531 | rewriter.setInsertionPointToEnd(&beforeBlock); |
| 532 | spirv::StoreOp::create(builder&: rewriter, location: condLoc, ptr: alloc, value: res); |
| 533 | } |
| 534 | |
| 535 | rewriter.setInsertionPointToEnd(&beforeBlock); |
| 536 | rewriter.replaceOpWithNewOp<spirv::BranchConditionalOp>( |
| 537 | op: cond, args&: conditionVal, args: &afterBlock, args&: condArgs, args: &mergeBlock, args: ValueRange()); |
| 538 | |
| 539 | // Convert the scf.yield op to a branch back to the header block. |
| 540 | rewriter.setInsertionPointToEnd(&afterBlock); |
| 541 | rewriter.replaceOpWithNewOp<spirv::BranchOp>(op: yield, args: &beforeBlock, |
| 542 | args&: yieldArgs); |
| 543 | |
| 544 | rewriter.replaceOp(op: whileOp, newValues: resultValues); |
| 545 | return success(); |
| 546 | } |
| 547 | }; |
| 548 | } // namespace |
| 549 | |
| 550 | //===----------------------------------------------------------------------===// |
| 551 | // Public API |
| 552 | //===----------------------------------------------------------------------===// |
| 553 | |
| 554 | void mlir::populateSCFToSPIRVPatterns(const SPIRVTypeConverter &typeConverter, |
| 555 | ScfToSPIRVContext &scfToSPIRVContext, |
| 556 | RewritePatternSet &patterns) { |
| 557 | patterns.add<ForOpConversion, IfOpConversion, IndexSwitchOpConversion, |
| 558 | TerminatorOpConversion, WhileOpConversion>( |
| 559 | arg: patterns.getContext(), args: typeConverter, args: scfToSPIRVContext.getImpl()); |
| 560 | } |
| 561 | |