1//===- SCFToSPIRV.cpp - SCF to SPIR-V Patterns ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements patterns to convert SCF dialect to SPIR-V dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h"
14#include "mlir/Dialect/SCF/IR/SCF.h"
15#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
16#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
17#include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
18#include "mlir/Transforms/DialectConversion.h"
19#include "llvm/Support/FormatVariadic.h"
20
21using namespace mlir;
22
23//===----------------------------------------------------------------------===//
24// Context
25//===----------------------------------------------------------------------===//
26
27namespace mlir {
28struct ScfToSPIRVContextImpl {
29 // Map between the spirv region control flow operation (spirv.mlir.loop or
30 // spirv.mlir.selection) to the VariableOp created to store the region
31 // results. The order of the VariableOp matches the order of the results.
32 DenseMap<Operation *, SmallVector<spirv::VariableOp, 8>> outputVars;
33};
34} // namespace mlir
35
36/// We use ScfToSPIRVContext to store information about the lowering of the scf
37/// region that need to be used later on. When we lower scf.for/scf.if we create
38/// VariableOp to store the results. We need to keep track of the VariableOp
39/// created as we need to insert stores into them when lowering Yield. Those
40/// StoreOp cannot be created earlier as they may use a different type than
41/// yield operands.
42ScfToSPIRVContext::ScfToSPIRVContext() {
43 impl = std::make_unique<::ScfToSPIRVContextImpl>();
44}
45
46ScfToSPIRVContext::~ScfToSPIRVContext() = default;
47
48namespace {
49
50//===----------------------------------------------------------------------===//
51// Helper Functions
52//===----------------------------------------------------------------------===//
53
54/// Replaces SCF op outputs with SPIR-V variable loads.
55/// We create VariableOp to handle the results value of the control flow region.
56/// spirv.mlir.loop/spirv.mlir.selection currently don't yield value. Right
57/// after the loop we load the value from the allocation and use it as the SCF
58/// op result.
59template <typename ScfOp, typename OpTy>
60void replaceSCFOutputValue(ScfOp scfOp, OpTy newOp,
61 ConversionPatternRewriter &rewriter,
62 ScfToSPIRVContextImpl *scfToSPIRVContext,
63 ArrayRef<Type> returnTypes) {
64
65 Location loc = scfOp.getLoc();
66 auto &allocas = scfToSPIRVContext->outputVars[newOp];
67 // Clearing the allocas is necessary in case a dialect conversion path failed
68 // previously, and this is the second attempt of this conversion.
69 allocas.clear();
70 SmallVector<Value, 8> resultValue;
71 for (Type convertedType : returnTypes) {
72 auto pointerType =
73 spirv::PointerType::get(pointeeType: convertedType, storageClass: spirv::StorageClass::Function);
74 rewriter.setInsertionPoint(newOp);
75 auto alloc = spirv::VariableOp::create(builder&: rewriter, location: loc, pointer: pointerType,
76 storage_class: spirv::StorageClass::Function,
77 /*initializer=*/nullptr);
78 allocas.push_back(alloc);
79 rewriter.setInsertionPointAfter(newOp);
80 Value loadResult = spirv::LoadOp::create(builder&: rewriter, location: loc, basePtr: alloc);
81 resultValue.push_back(Elt: loadResult);
82 }
83 rewriter.replaceOp(scfOp, resultValue);
84}
85
86Region::iterator getBlockIt(Region &region, unsigned index) {
87 return std::next(x: region.begin(), n: index);
88}
89
90//===----------------------------------------------------------------------===//
91// Conversion Patterns
92//===----------------------------------------------------------------------===//
93
94/// Common class for all vector to GPU patterns.
95template <typename OpTy>
96class SCFToSPIRVPattern : public OpConversionPattern<OpTy> {
97public:
98 SCFToSPIRVPattern(MLIRContext *context, const SPIRVTypeConverter &converter,
99 ScfToSPIRVContextImpl *scfToSPIRVContext)
100 : OpConversionPattern<OpTy>::OpConversionPattern(converter, context),
101 scfToSPIRVContext(scfToSPIRVContext), typeConverter(converter) {}
102
103protected:
104 ScfToSPIRVContextImpl *scfToSPIRVContext;
105 // FIXME: We explicitly keep a reference of the type converter here instead of
106 // passing it to OpConversionPattern during construction. This effectively
107 // bypasses the conversion framework's automation on type conversion. This is
108 // needed right now because the conversion framework will unconditionally
109 // legalize all types used by SCF ops upon discovering them, for example, the
110 // types of loop carried values. We use SPIR-V variables for those loop
111 // carried values. Depending on the available capabilities, the SPIR-V
112 // variable can be different, for example, cooperative matrix or normal
113 // variable. We'd like to detach the conversion of the loop carried values
114 // from the SCF ops (which is mainly a region). So we need to "mark" types
115 // used by SCF ops as legal, if to use the conversion framework for type
116 // conversion. There isn't a straightforward way to do that yet, as when
117 // converting types, ops aren't taken into consideration. Therefore, we just
118 // bypass the framework's type conversion for now.
119 const SPIRVTypeConverter &typeConverter;
120};
121
122//===----------------------------------------------------------------------===//
123// scf::ForOp
124//===----------------------------------------------------------------------===//
125
126/// Pattern to convert a scf::ForOp within kernel functions into spirv::LoopOp.
127struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
128 using SCFToSPIRVPattern::SCFToSPIRVPattern;
129
130 LogicalResult
131 matchAndRewrite(scf::ForOp forOp, OpAdaptor adaptor,
132 ConversionPatternRewriter &rewriter) const override {
133 // scf::ForOp can be lowered to the structured control flow represented by
134 // spirv::LoopOp by making the continue block of the spirv::LoopOp the loop
135 // latch and the merge block the exit block. The resulting spirv::LoopOp has
136 // a single back edge from the continue to header block, and a single exit
137 // from header to merge.
138 auto loc = forOp.getLoc();
139 auto loopControl = spirv::LoopControl::None;
140 if (auto attr = forOp->getDiscardableAttrOfType<spirv::LoopControlAttr>(
141 name: spirv::getLoopControlAttrName()))
142 loopControl = attr.getValue();
143 auto loopOp = spirv::LoopOp::create(builder&: rewriter, location: loc, loopControl);
144 loopOp.addEntryAndMergeBlock(builder&: rewriter);
145
146 OpBuilder::InsertionGuard guard(rewriter);
147 // Create the block for the header.
148 Block *header = rewriter.createBlock(parent: &loopOp.getBody(),
149 insertPt: getBlockIt(region&: loopOp.getBody(), index: 1));
150 rewriter.setInsertionPointAfter(loopOp);
151
152 // Create the new induction variable to use.
153 Value adapLowerBound = adaptor.getLowerBound();
154 BlockArgument newIndVar =
155 header->addArgument(type: adapLowerBound.getType(), loc: adapLowerBound.getLoc());
156 for (Value arg : adaptor.getInitArgs())
157 header->addArgument(type: arg.getType(), loc: arg.getLoc());
158 Block *body = forOp.getBody();
159
160 // Apply signature conversion to the body of the forOp. It has a single
161 // block, with argument which is the induction variable. That has to be
162 // replaced with the new induction variable.
163 TypeConverter::SignatureConversion signatureConverter(
164 body->getNumArguments());
165 signatureConverter.remapInput(origInputNo: 0, replacements: newIndVar);
166 for (unsigned i = 1, e = body->getNumArguments(); i < e; i++)
167 signatureConverter.remapInput(origInputNo: i, replacements: header->getArgument(i));
168 body = rewriter.applySignatureConversion(block: &forOp.getRegion().front(),
169 conversion&: signatureConverter);
170
171 // Move the blocks from the forOp into the loopOp. This is the body of the
172 // loopOp.
173 rewriter.inlineRegionBefore(region&: forOp->getRegion(index: 0), parent&: loopOp.getBody(),
174 before: getBlockIt(region&: loopOp.getBody(), index: 2));
175
176 SmallVector<Value, 8> args(1, adaptor.getLowerBound());
177 args.append(in_start: adaptor.getInitArgs().begin(), in_end: adaptor.getInitArgs().end());
178 // Branch into it from the entry.
179 rewriter.setInsertionPointToEnd(&(loopOp.getBody().front()));
180 spirv::BranchOp::create(builder&: rewriter, location: loc, successor: header, arguments: args);
181
182 // Generate the rest of the loop header.
183 rewriter.setInsertionPointToEnd(header);
184 auto *mergeBlock = loopOp.getMergeBlock();
185 Value cmpOp;
186 if (forOp.getUnsignedCmp()) {
187 cmpOp = spirv::ULessThanOp::create(builder&: rewriter, location: loc, result: rewriter.getI1Type(),
188 operand1: newIndVar, operand2: adaptor.getUpperBound());
189 } else {
190 cmpOp = spirv::SLessThanOp::create(builder&: rewriter, location: loc, result: rewriter.getI1Type(),
191 operand1: newIndVar, operand2: adaptor.getUpperBound());
192 }
193
194 spirv::BranchConditionalOp::create(builder&: rewriter, location: loc, condition: cmpOp, trueBlock: body,
195 trueArguments: ArrayRef<Value>(), falseBlock: mergeBlock,
196 falseArguments: ArrayRef<Value>());
197
198 // Generate instructions to increment the step of the induction variable and
199 // branch to the header.
200 Block *continueBlock = loopOp.getContinueBlock();
201 rewriter.setInsertionPointToEnd(continueBlock);
202
203 // Add the step to the induction variable and branch to the header.
204 Value updatedIndVar = spirv::IAddOp::create(
205 builder&: rewriter, location: loc, result: newIndVar.getType(), operand1: newIndVar, operand2: adaptor.getStep());
206 spirv::BranchOp::create(builder&: rewriter, location: loc, successor: header, arguments: updatedIndVar);
207
208 // Infer the return types from the init operands. Vector type may get
209 // converted to CooperativeMatrix or to Vector type, to avoid having complex
210 // extra logic to figure out the right type we just infer it from the Init
211 // operands.
212 SmallVector<Type, 8> initTypes;
213 for (auto arg : adaptor.getInitArgs())
214 initTypes.push_back(Elt: arg.getType());
215 replaceSCFOutputValue(scfOp: forOp, newOp: loopOp, rewriter, scfToSPIRVContext,
216 returnTypes: initTypes);
217
218 // Store init values so a zero-trip loop returns them instead of undef.
219 // Skip the stores if the loop is known to always execute at least once.
220 std::optional<APInt> tripCount = forOp.getStaticTripCount();
221 if (!tripCount || tripCount->isZero()) {
222 auto &allocas = scfToSPIRVContext->outputVars[loopOp];
223 rewriter.setInsertionPoint(loopOp);
224 for (auto [alloca, init] : llvm::zip(t&: allocas, u: adaptor.getInitArgs()))
225 spirv::StoreOp::create(builder&: rewriter, location: loc, ptr: alloca, value: init);
226 }
227 return success();
228 }
229};
230
231//===----------------------------------------------------------------------===//
232// scf::IfOp
233//===----------------------------------------------------------------------===//
234
235/// Pattern to convert a scf::IfOp within kernel functions into
236/// spirv::SelectionOp.
237struct IfOpConversion : SCFToSPIRVPattern<scf::IfOp> {
238 using SCFToSPIRVPattern::SCFToSPIRVPattern;
239
240 LogicalResult
241 matchAndRewrite(scf::IfOp ifOp, OpAdaptor adaptor,
242 ConversionPatternRewriter &rewriter) const override {
243 // When lowering `scf::IfOp` we explicitly create a selection header block
244 // before the control flow diverges and a merge block where control flow
245 // subsequently converges.
246 auto loc = ifOp.getLoc();
247
248 // Compute return types.
249 SmallVector<Type, 8> returnTypes;
250 for (auto result : ifOp.getResults()) {
251 auto convertedType = typeConverter.convertType(t: result.getType());
252 if (!convertedType)
253 return rewriter.notifyMatchFailure(
254 arg&: loc,
255 msg: llvm::formatv(Fmt: "failed to convert type '{0}'", Vals: result.getType()));
256
257 returnTypes.push_back(Elt: convertedType);
258 }
259
260 // Create `spirv.selection` operation, selection header block and merge
261 // block.
262 auto selectionControl = spirv::SelectionControl::None;
263 if (auto attr = ifOp->getDiscardableAttrOfType<spirv::SelectionControlAttr>(
264 name: spirv::getSelectionControlAttrName()))
265 selectionControl = attr.getValue();
266 auto selectionOp =
267 spirv::SelectionOp::create(builder&: rewriter, location: loc, selectionControl);
268 auto *mergeBlock = rewriter.createBlock(parent: &selectionOp.getBody(),
269 insertPt: selectionOp.getBody().end());
270 spirv::MergeOp::create(builder&: rewriter, location: loc);
271
272 OpBuilder::InsertionGuard guard(rewriter);
273 auto *selectionHeaderBlock =
274 rewriter.createBlock(insertBefore: &selectionOp.getBody().front());
275
276 // Inline `then` region before the merge block and branch to it.
277 auto &thenRegion = ifOp.getThenRegion();
278 auto *thenBlock = &thenRegion.front();
279 rewriter.setInsertionPointToEnd(&thenRegion.back());
280 spirv::BranchOp::create(builder&: rewriter, location: loc, successor: mergeBlock);
281 rewriter.inlineRegionBefore(region&: thenRegion, before: mergeBlock);
282
283 auto *elseBlock = mergeBlock;
284 // If `else` region is not empty, inline that region before the merge block
285 // and branch to it.
286 if (!ifOp.getElseRegion().empty()) {
287 auto &elseRegion = ifOp.getElseRegion();
288 elseBlock = &elseRegion.front();
289 rewriter.setInsertionPointToEnd(&elseRegion.back());
290 spirv::BranchOp::create(builder&: rewriter, location: loc, successor: mergeBlock);
291 rewriter.inlineRegionBefore(region&: elseRegion, before: mergeBlock);
292 }
293
294 // Create a `spirv.BranchConditional` operation for selection header block.
295 rewriter.setInsertionPointToEnd(selectionHeaderBlock);
296 spirv::BranchConditionalOp::create(builder&: rewriter, location: loc, condition: adaptor.getCondition(),
297 trueBlock: thenBlock, trueArguments: ArrayRef<Value>(), falseBlock: elseBlock,
298 falseArguments: ArrayRef<Value>());
299
300 replaceSCFOutputValue(scfOp: ifOp, newOp: selectionOp, rewriter, scfToSPIRVContext,
301 returnTypes);
302 return success();
303 }
304};
305
306//===----------------------------------------------------------------------===//
307// scf::IndexSwitchOp
308//===----------------------------------------------------------------------===//
309
310/// Pattern to convert a scf::IndexSwitchOp within kernel functions into
311/// spirv::SelectionOp with a spirv::SwitchOp header.
312struct IndexSwitchOpConversion final : SCFToSPIRVPattern<scf::IndexSwitchOp> {
313 using SCFToSPIRVPattern::SCFToSPIRVPattern;
314
315 LogicalResult
316 matchAndRewrite(scf::IndexSwitchOp switchOp, OpAdaptor adaptor,
317 ConversionPatternRewriter &rewriter) const override {
318 Location loc = switchOp.getLoc();
319
320 // Compute return types.
321 SmallVector<Type, 8> returnTypes;
322 for (Value result : switchOp.getResults()) {
323 Type convertedType = typeConverter.convertType(t: result.getType());
324 if (!convertedType)
325 return rewriter.notifyMatchFailure(
326 arg&: loc,
327 msg: llvm::formatv(Fmt: "failed to convert type '{0}'", Vals: result.getType()));
328 returnTypes.push_back(Elt: convertedType);
329 }
330
331 // The selector must be a SPIR-V integer; spirv.Switch literals are
332 // interpreted with the selector's bit width.
333 Value selector = adaptor.getArg();
334 auto selectorType = dyn_cast<IntegerType>(Val: selector.getType());
335 if (!selectorType)
336 return rewriter.notifyMatchFailure(arg&: loc,
337 msg: "selector type is not an integer");
338 unsigned selectorWidth = selectorType.getWidth();
339
340 // Create the `spirv.mlir.selection` op, its header block, and merge block.
341 auto selectionControl = spirv::SelectionControl::None;
342 if (auto attr =
343 switchOp->getDiscardableAttrOfType<spirv::SelectionControlAttr>(
344 name: spirv::getSelectionControlAttrName()))
345 selectionControl = attr.getValue();
346 auto selectionOp =
347 spirv::SelectionOp::create(builder&: rewriter, location: loc, selectionControl);
348 auto *mergeBlock = rewriter.createBlock(parent: &selectionOp.getBody(),
349 insertPt: selectionOp.getBody().end());
350 spirv::MergeOp::create(builder&: rewriter, location: loc);
351
352 OpBuilder::InsertionGuard guard(rewriter);
353 auto *headerBlock = rewriter.createBlock(insertBefore: &selectionOp.getBody().front());
354
355 // Inline each case region before the merge block and branch to it.
356 SmallVector<APInt> caseLiterals;
357 SmallVector<Block *> caseBlocks;
358 ArrayRef<int64_t> cases = switchOp.getCases();
359 for (auto [caseValue, caseRegion] :
360 llvm::zip_equal(t&: cases, u: switchOp.getCaseRegions())) {
361 Block *caseBlock = &caseRegion.front();
362 rewriter.setInsertionPointToEnd(&caseRegion.back());
363 spirv::BranchOp::create(builder&: rewriter, location: loc, successor: mergeBlock);
364 rewriter.inlineRegionBefore(region&: caseRegion, before: mergeBlock);
365 caseLiterals.push_back(
366 Elt: APInt(selectorWidth, caseValue, /*isSigned=*/true));
367 caseBlocks.push_back(Elt: caseBlock);
368 }
369
370 // Inline the default region before the merge block and branch to it.
371 Region &defaultRegion = switchOp.getDefaultRegion();
372 Block *defaultBlock = &defaultRegion.front();
373 rewriter.setInsertionPointToEnd(&defaultRegion.back());
374 spirv::BranchOp::create(builder&: rewriter, location: loc, successor: mergeBlock);
375 rewriter.inlineRegionBefore(region&: defaultRegion, before: mergeBlock);
376
377 // Create the `spirv.Switch` terminator for the header block. The case
378 // regions carry their results through variables, so the branches take no
379 // operands.
380 SmallVector<ValueRange> caseOperands(caseBlocks.size(), ValueRange());
381 rewriter.setInsertionPointToEnd(headerBlock);
382 spirv::SwitchOp::create(builder&: rewriter, location: loc, selector, defaultTarget: defaultBlock, defaultOperands: ValueRange(),
383 literals: caseLiterals, targets: caseBlocks, targetOperands: caseOperands);
384
385 replaceSCFOutputValue(scfOp: switchOp, newOp: selectionOp, rewriter, scfToSPIRVContext,
386 returnTypes);
387 return success();
388 }
389};
390
391//===----------------------------------------------------------------------===//
392// scf::YieldOp
393//===----------------------------------------------------------------------===//
394
395struct TerminatorOpConversion final : SCFToSPIRVPattern<scf::YieldOp> {
396public:
397 using SCFToSPIRVPattern::SCFToSPIRVPattern;
398
399 LogicalResult
400 matchAndRewrite(scf::YieldOp terminatorOp, OpAdaptor adaptor,
401 ConversionPatternRewriter &rewriter) const override {
402 ValueRange operands = adaptor.getOperands();
403
404 Operation *parent = terminatorOp->getParentOp();
405
406 // TODO: Implement conversion for the remaining `scf` ops.
407 if (parent->getDialect()->getNamespace() ==
408 scf::SCFDialect::getDialectNamespace() &&
409 !isa<scf::IfOp, scf::ForOp, scf::WhileOp, scf::IndexSwitchOp>(Val: parent))
410 return rewriter.notifyMatchFailure(
411 arg&: terminatorOp,
412 msg: llvm::formatv(Fmt: "conversion not supported for parent op: '{0}'",
413 Vals: parent->getName()));
414
415 // If the region return values, store each value into the associated
416 // VariableOp created during lowering of the parent region.
417 if (!operands.empty()) {
418 auto &allocas = scfToSPIRVContext->outputVars[parent];
419 if (allocas.size() != operands.size())
420 return failure();
421
422 auto loc = terminatorOp.getLoc();
423 for (unsigned i = 0, e = operands.size(); i < e; i++)
424 spirv::StoreOp::create(builder&: rewriter, location: loc, ptr: allocas[i], value: operands[i]);
425 if (isa<spirv::LoopOp>(Val: parent)) {
426 // For loops we also need to update the branch jumping back to the
427 // header.
428 auto br = cast<spirv::BranchOp>(
429 Val: rewriter.getInsertionBlock()->getTerminator());
430 SmallVector<Value, 8> args(br.getBlockArguments());
431 args.append(in_start: operands.begin(), in_end: operands.end());
432 rewriter.setInsertionPoint(br);
433 spirv::BranchOp::create(builder&: rewriter, location: terminatorOp.getLoc(), successor: br.getTarget(),
434 arguments: args);
435 rewriter.eraseOp(op: br);
436 }
437 }
438 rewriter.eraseOp(op: terminatorOp);
439 return success();
440 }
441};
442
443//===----------------------------------------------------------------------===//
444// scf::WhileOp
445//===----------------------------------------------------------------------===//
446
447struct WhileOpConversion final : SCFToSPIRVPattern<scf::WhileOp> {
448 using SCFToSPIRVPattern::SCFToSPIRVPattern;
449
450 LogicalResult
451 matchAndRewrite(scf::WhileOp whileOp, OpAdaptor adaptor,
452 ConversionPatternRewriter &rewriter) const override {
453 auto loc = whileOp.getLoc();
454 auto loopControl = spirv::LoopControl::None;
455 if (auto attr = whileOp->getDiscardableAttrOfType<spirv::LoopControlAttr>(
456 name: spirv::getLoopControlAttrName()))
457 loopControl = attr.getValue();
458 auto loopOp = spirv::LoopOp::create(builder&: rewriter, location: loc, loopControl);
459 loopOp.addEntryAndMergeBlock(builder&: rewriter);
460
461 Region &beforeRegion = whileOp.getBefore();
462 Region &afterRegion = whileOp.getAfter();
463
464 if (failed(Result: rewriter.convertRegionTypes(region: &beforeRegion, converter: typeConverter)) ||
465 failed(Result: rewriter.convertRegionTypes(region: &afterRegion, converter: typeConverter)))
466 return rewriter.notifyMatchFailure(arg&: whileOp,
467 msg: "Failed to convert region types");
468
469 OpBuilder::InsertionGuard guard(rewriter);
470
471 Block &entryBlock = *loopOp.getEntryBlock();
472 Block &beforeBlock = beforeRegion.front();
473 Block &afterBlock = afterRegion.front();
474 Block &mergeBlock = *loopOp.getMergeBlock();
475
476 auto cond = cast<scf::ConditionOp>(Val: beforeBlock.getTerminator());
477 SmallVector<Value> condArgs;
478 if (failed(Result: rewriter.getRemappedValues(keys: cond.getArgs(), results&: condArgs)))
479 return failure();
480
481 Value conditionVal = rewriter.getRemappedValue(key: cond.getCondition());
482 if (!conditionVal)
483 return failure();
484
485 auto yield = cast<scf::YieldOp>(Val: afterBlock.getTerminator());
486 SmallVector<Value> yieldArgs;
487 if (failed(Result: rewriter.getRemappedValues(keys: yield.getResults(), results&: yieldArgs)))
488 return failure();
489
490 // Move the while before block as the initial loop header block.
491 rewriter.inlineRegionBefore(region&: beforeRegion, parent&: loopOp.getBody(),
492 before: getBlockIt(region&: loopOp.getBody(), index: 1));
493
494 // Move the while after block as the initial loop body block.
495 rewriter.inlineRegionBefore(region&: afterRegion, parent&: loopOp.getBody(),
496 before: getBlockIt(region&: loopOp.getBody(), index: 2));
497
498 // Jump from the loop entry block to the loop header block.
499 rewriter.setInsertionPointToEnd(&entryBlock);
500 spirv::BranchOp::create(builder&: rewriter, location: loc, successor: &beforeBlock, arguments: adaptor.getInits());
501
502 auto condLoc = cond.getLoc();
503
504 SmallVector<Value> resultValues(condArgs.size());
505
506 // For other SCF ops, the scf.yield op yields the value for the whole SCF
507 // op. So we use the scf.yield op as the anchor to create/load/store SPIR-V
508 // local variables. But for the scf.while op, the scf.yield op yields a
509 // value for the before region, which may not matching the whole op's
510 // result. Instead, the scf.condition op returns values matching the whole
511 // op's results. So we need to create/load/store variables according to
512 // that.
513 for (const auto &it : llvm::enumerate(First&: condArgs)) {
514 auto res = it.value();
515 auto i = it.index();
516 auto pointerType =
517 spirv::PointerType::get(pointeeType: res.getType(), storageClass: spirv::StorageClass::Function);
518
519 // Create local variables before the scf.while op.
520 rewriter.setInsertionPoint(loopOp);
521 auto alloc = spirv::VariableOp::create(builder&: rewriter, location: condLoc, pointer: pointerType,
522 storage_class: spirv::StorageClass::Function,
523 /*initializer=*/nullptr);
524
525 // Load the final result values after the scf.while op.
526 rewriter.setInsertionPointAfter(loopOp);
527 auto loadResult = spirv::LoadOp::create(builder&: rewriter, location: condLoc, basePtr: alloc);
528 resultValues[i] = loadResult;
529
530 // Store the current iteration's result value.
531 rewriter.setInsertionPointToEnd(&beforeBlock);
532 spirv::StoreOp::create(builder&: rewriter, location: condLoc, ptr: alloc, value: res);
533 }
534
535 rewriter.setInsertionPointToEnd(&beforeBlock);
536 rewriter.replaceOpWithNewOp<spirv::BranchConditionalOp>(
537 op: cond, args&: conditionVal, args: &afterBlock, args&: condArgs, args: &mergeBlock, args: ValueRange());
538
539 // Convert the scf.yield op to a branch back to the header block.
540 rewriter.setInsertionPointToEnd(&afterBlock);
541 rewriter.replaceOpWithNewOp<spirv::BranchOp>(op: yield, args: &beforeBlock,
542 args&: yieldArgs);
543
544 rewriter.replaceOp(op: whileOp, newValues: resultValues);
545 return success();
546 }
547};
548} // namespace
549
550//===----------------------------------------------------------------------===//
551// Public API
552//===----------------------------------------------------------------------===//
553
554void mlir::populateSCFToSPIRVPatterns(const SPIRVTypeConverter &typeConverter,
555 ScfToSPIRVContext &scfToSPIRVContext,
556 RewritePatternSet &patterns) {
557 patterns.add<ForOpConversion, IfOpConversion, IndexSwitchOpConversion,
558 TerminatorOpConversion, WhileOpConversion>(
559 arg: patterns.getContext(), args: typeConverter, args: scfToSPIRVContext.getImpl());
560}
561