| 1 | //===- ACCToLLVMUtils.cpp - OpenACC to LLVM helpers -------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/OpenACCToLLVM/ACCToLLVMUtils.h" |
| 10 | |
| 11 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 12 | #include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h" |
| 13 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h" |
| 18 | #include "mlir/Interfaces/FunctionInterfaces.h" |
| 19 | #include "llvm/ADT/Hashing.h" |
| 20 | #include "llvm/ADT/STLExtras.h" |
| 21 | #include "llvm/ADT/SmallString.h" |
| 22 | #include "llvm/ADT/StringExtras.h" |
| 23 | #include "llvm/ADT/Twine.h" |
| 24 | |
| 25 | #include <iterator> |
| 26 | |
| 27 | using namespace mlir; |
| 28 | using namespace mlir::acc; |
| 29 | |
| 30 | Location acc::unfuseLoc(Location loc) { |
| 31 | while (auto fusedLoc = dyn_cast<FusedLoc>(Val&: loc)) |
| 32 | loc = fusedLoc.getLocations().back(); |
| 33 | return loc; |
| 34 | } |
| 35 | |
| 36 | std::optional<FileLineColLoc> |
| 37 | acc::getFileLineColLoc(Location loc, bool errorOnInvalidLocation) { |
| 38 | Location unfusedLoc = unfuseLoc(loc); |
| 39 | |
| 40 | if (auto fileLoc = dyn_cast<FileLineColLoc>(Val&: unfusedLoc)) |
| 41 | return fileLoc; |
| 42 | |
| 43 | if (auto callSiteLoc = dyn_cast<CallSiteLoc>(Val&: unfusedLoc)) { |
| 44 | if (auto calleeFileLoc = getFileLineColLoc(loc: callSiteLoc.getCallee(), errorOnInvalidLocation: false)) |
| 45 | return calleeFileLoc; |
| 46 | if (auto callerFileLoc = |
| 47 | getFileLineColLoc(loc: callSiteLoc.getCaller(), errorOnInvalidLocation)) |
| 48 | return callerFileLoc; |
| 49 | } |
| 50 | |
| 51 | if (errorOnInvalidLocation) |
| 52 | llvm_unreachable( |
| 53 | "cannot get file:line information: invalid Location information" ); |
| 54 | return std::nullopt; |
| 55 | } |
| 56 | |
| 57 | StringRef acc::getParentFunctionName(Operation *op) { |
| 58 | if (!op) |
| 59 | return "" ; |
| 60 | auto funcOp = dyn_cast<FunctionOpInterface>(Val: op); |
| 61 | if (!funcOp) |
| 62 | funcOp = op->getParentOfType<FunctionOpInterface>(); |
| 63 | return funcOp ? funcOp.getName() : StringRef(); |
| 64 | } |
| 65 | |
| 66 | StringRef acc::getParentFunctionName(Value value) { |
| 67 | if (auto *op = value.getDefiningOp()) |
| 68 | return getParentFunctionName(op); |
| 69 | return "" ; |
| 70 | } |
| 71 | |
| 72 | StringRef acc::getParentFunctionName(ValueRange values) { |
| 73 | for (Value value : values) { |
| 74 | if (StringRef name = getParentFunctionName(value); !name.empty()) |
| 75 | return name; |
| 76 | } |
| 77 | return "" ; |
| 78 | } |
| 79 | |
| 80 | std::string acc::getInternalGlobalName(StringRef kind, StringRef detail) { |
| 81 | SmallString<64> name("acc." ); |
| 82 | name += kind; |
| 83 | if (detail.empty()) |
| 84 | return name.str().str(); |
| 85 | |
| 86 | name += '.'; |
| 87 | for (unsigned char c : detail) |
| 88 | name += (llvm::isAlnum(C: c) || c == '_' || c == '$' || c == '.') ? c : '_'; |
| 89 | return name.str().str(); |
| 90 | } |
| 91 | |
| 92 | /// Creates or reuses a module-internal null-terminated string global and |
| 93 | /// returns the GlobalOp. |
| 94 | static LLVM::GlobalOp getOrCreateGlobalStringOp(Location loc, |
| 95 | OpBuilder &builder, |
| 96 | StringRef name, StringRef value, |
| 97 | Region &globalSymbolRegion, |
| 98 | SymbolTable *symbolTable) { |
| 99 | SmallString<32> nullTermStr(value); |
| 100 | nullTermStr.push_back(Elt: '\0'); |
| 101 | StringAttr valueAttr = builder.getStringAttr(bytes: nullTermStr); |
| 102 | |
| 103 | // A name says what the global holds, so a global of that name is the one to |
| 104 | // reuse as long as it holds this very value. Names come out of detail the |
| 105 | // source spells and are not one to one with it, so a name can also lead to a |
| 106 | // global holding something else, which a suffix then keeps apart. |
| 107 | SmallString<64> uniqueName(name); |
| 108 | if (symbolTable) { |
| 109 | unsigned suffix = 0; |
| 110 | while (Operation *taken = symbolTable->lookup(name: uniqueName)) { |
| 111 | auto global = dyn_cast<LLVM::GlobalOp>(Val: taken); |
| 112 | if (global && global.getValueOrNull() == valueAttr) |
| 113 | return global; |
| 114 | uniqueName.clear(); |
| 115 | (name + "." + Twine(suffix++)).toVector(Out&: uniqueName); |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | // Materialize the global through the incoming builder so that it stays |
| 120 | // tracked when the caller is a dialect conversion rewriter. |
| 121 | OpBuilder::InsertionGuard guard(builder); |
| 122 | builder.setInsertionPointToStart(&globalSymbolRegion.front()); |
| 123 | auto arrayTy = LLVM::LLVMArrayType::get(elementType: builder.getI8Type(), |
| 124 | numElements: nullTermStr.size_in_bytes()); |
| 125 | auto global = |
| 126 | LLVM::GlobalOp::create(builder, location: loc, type: arrayTy, /*isConstant=*/true, |
| 127 | linkage: LLVM::Linkage::Internal, name: uniqueName, value: valueAttr, |
| 128 | /*alignment=*/0); |
| 129 | if (symbolTable) |
| 130 | symbolTable->insert(symbol: global); |
| 131 | return global; |
| 132 | } |
| 133 | |
| 134 | Value acc::getOrCreateGlobalString(Location loc, OpBuilder &builder, |
| 135 | StringRef name, StringRef value, |
| 136 | Region &globalSymbolRegion, |
| 137 | SymbolTable *symbolTable) { |
| 138 | Type i64Ty = builder.getI64Type(); |
| 139 | Type ptrTy = LLVM::LLVMPointerType::get(context: builder.getContext()); |
| 140 | LLVM::GlobalOp global = getOrCreateGlobalStringOp( |
| 141 | loc, builder, name, value, globalSymbolRegion, symbolTable); |
| 142 | |
| 143 | Value globalPtr = LLVM::AddressOfOp::create(builder, location: loc, global); |
| 144 | Value cst0 = LLVM::ConstantOp::create(builder, location: loc, res: i64Ty, |
| 145 | value: builder.getI64IntegerAttr(value: 0)); |
| 146 | return LLVM::GEPOp::create(builder, location: loc, resultType: ptrTy, elementType: global.getType(), basePtr: globalPtr, |
| 147 | indices: ArrayRef<Value>({cst0, cst0})); |
| 148 | } |
| 149 | |
| 150 | Value acc::createIdent(Location loc, StringRef functionName, OpBuilder &builder, |
| 151 | Region &globalSymbolRegion, |
| 152 | const ACCRuntimeCallConfig &config, |
| 153 | SymbolTable *symbolTable) { |
| 154 | MLIRContext *ctx = builder.getContext(); |
| 155 | Type i32Ty = builder.getI32Type(); |
| 156 | Type i64Ty = builder.getI64Type(); |
| 157 | Type ptrTy = LLVM::LLVMPointerType::get(context: ctx); |
| 158 | Type structTy = LLVM::LLVMStructType::getLiteral( |
| 159 | context: ctx, types: {i32Ty, i32Ty, i32Ty, i32Ty, ptrTy}); |
| 160 | |
| 161 | std::string source; |
| 162 | // The position the globals describe is what names them, so a global found |
| 163 | // under such a name holds this very source string or ident. |
| 164 | std::string position; |
| 165 | if (auto fileLineColLoc = |
| 166 | getFileLineColLoc(loc, /*errorOnInvalidLocation=*/false)) { |
| 167 | std::string filename = fileLineColLoc->getFilename().str(); |
| 168 | std::string line = std::to_string(val: fileLineColLoc->getLine()); |
| 169 | std::string column = std::to_string(val: fileLineColLoc->getColumn()); |
| 170 | std::string functionDisplayName = |
| 171 | functionName.empty() ? std::string() |
| 172 | : config.getFunctionDisplayName(mangledOrSymbol: functionName); |
| 173 | source = ";" ; |
| 174 | source += filename + ";" ; |
| 175 | source += functionDisplayName + ";" ; |
| 176 | source += line + ";" ; |
| 177 | source += column + ";" ; |
| 178 | source += ";" ; |
| 179 | position = line + "." + column + "." ; |
| 180 | position += std::to_string(val: static_cast<uint64_t>(llvm::hash_value(arg: source))); |
| 181 | } else { |
| 182 | source = ";unknown;unknown;0;0;;" ; |
| 183 | position = "unknown" ; |
| 184 | } |
| 185 | |
| 186 | std::string identGlobalName = getInternalGlobalName(kind: "ident" , detail: position); |
| 187 | LLVM::GlobalOp identGlobal = |
| 188 | symbolTable ? symbolTable->lookup<LLVM::GlobalOp>(name: identGlobalName) |
| 189 | : LLVM::GlobalOp(); |
| 190 | if (!identGlobal) { |
| 191 | LLVM::GlobalOp sourceGlobal = getOrCreateGlobalStringOp( |
| 192 | loc, builder, name: getInternalGlobalName(kind: "loc" , detail: position), value: source, |
| 193 | globalSymbolRegion, symbolTable); |
| 194 | |
| 195 | OpBuilder::InsertionGuard guard(builder); |
| 196 | builder.setInsertionPointAfter(sourceGlobal); |
| 197 | identGlobal = LLVM::GlobalOp::create( |
| 198 | builder, location: loc, type: structTy, /*isConstant=*/true, linkage: LLVM::Linkage::Internal, |
| 199 | name: identGlobalName, /*value=*/Attribute(), /*alignment=*/0); |
| 200 | if (symbolTable) |
| 201 | symbolTable->insert(symbol: identGlobal); |
| 202 | |
| 203 | Block *block = builder.createBlock(parent: &identGlobal.getInitializerRegion()); |
| 204 | builder.setInsertionPointToStart(block); |
| 205 | Value ident = LLVM::ZeroOp::create(builder, location: loc, res: structTy); |
| 206 | Value sourceBase = LLVM::AddressOfOp::create(builder, location: loc, global: sourceGlobal); |
| 207 | Value cst0 = LLVM::ConstantOp::create(builder, location: loc, res: i64Ty, |
| 208 | value: builder.getI64IntegerAttr(value: 0)); |
| 209 | Value sourcePtr = |
| 210 | LLVM::GEPOp::create(builder, location: loc, resultType: ptrTy, elementType: sourceGlobal.getType(), |
| 211 | basePtr: sourceBase, indices: ArrayRef<Value>({cst0, cst0})); |
| 212 | ident = LLVM::InsertValueOp::create(builder, location: loc, res: structTy, container: ident, |
| 213 | value: sourcePtr, position: ArrayRef<int64_t>{4}); |
| 214 | LLVM::ReturnOp::create(builder, location: loc, arg: ident); |
| 215 | } |
| 216 | |
| 217 | return LLVM::AddressOfOp::create(builder, location: loc, global: identGlobal); |
| 218 | } |
| 219 | |
| 220 | Value acc::castToI64(Location loc, Value value, OpBuilder &builder) { |
| 221 | Type i64Ty = builder.getI64Type(); |
| 222 | if (value.getType() == i64Ty) |
| 223 | return value; |
| 224 | if (isa<IndexType>(Val: value.getType())) |
| 225 | return arith::IndexCastOp::create(builder, location: loc, out: i64Ty, in: value); |
| 226 | unsigned bitwidth = value.getType().getIntOrFloatBitWidth(); |
| 227 | if (bitwidth > 64) |
| 228 | return arith::TruncIOp::create(builder, location: loc, out: i64Ty, in: value); |
| 229 | return arith::ExtSIOp::create(builder, location: loc, out: i64Ty, in: value); |
| 230 | } |
| 231 | |
| 232 | Value acc::constantI64(Location loc, int64_t value, OpBuilder &builder) { |
| 233 | return LLVM::ConstantOp::create(builder, location: loc, type: builder.getI64Type(), value); |
| 234 | } |
| 235 | |
| 236 | Value acc::remapOrSelf(Value value, ConversionPatternRewriter &rewriter) { |
| 237 | if (!value) |
| 238 | return {}; |
| 239 | if (Value converted = rewriter.getRemappedValue(key: value)) |
| 240 | return converted; |
| 241 | return value; |
| 242 | } |
| 243 | |
| 244 | Value acc::getAsyncQueue(Location loc, Value asyncOperand, bool asyncOnly, |
| 245 | OpBuilder &builder, |
| 246 | const ACCRuntimeCallConfig &config) { |
| 247 | Type i64Ty = builder.getI64Type(); |
| 248 | if (asyncOnly) |
| 249 | return LLVM::ConstantOp::create(builder, location: loc, type: i64Ty, |
| 250 | value: config.getAsyncNoValueRuntimeValue()); |
| 251 | if (asyncOperand) |
| 252 | return castToI64(loc, value: asyncOperand, builder); |
| 253 | return LLVM::ConstantOp::create(builder, location: loc, type: i64Ty, |
| 254 | value: config.getAsyncSyncRuntimeValue()); |
| 255 | } |
| 256 | |
| 257 | LogicalResult acc::emitWaitCall(Location loc, ValueRange waitOperands, |
| 258 | Value asyncQueue, OpBuilder &builder, |
| 259 | Region &globalSymbolRegion, |
| 260 | SymbolTable &symbolTable, |
| 261 | const ACCRuntimeCallConfig &config, |
| 262 | Value deviceNum) { |
| 263 | Type i32Ty = builder.getI32Type(); |
| 264 | Type i64Ty = builder.getI64Type(); |
| 265 | Type ptrTy = LLVM::LLVMPointerType::get(context: builder.getContext()); |
| 266 | |
| 267 | SmallVector<Value> queues; |
| 268 | queues.reserve(N: waitOperands.size()); |
| 269 | for (Value waitOperand : waitOperands) |
| 270 | queues.push_back(Elt: castToI64(loc, value: waitOperand, builder)); |
| 271 | |
| 272 | unsigned size = queues.size(); |
| 273 | Value waitNum = LLVM::ConstantOp::create(builder, location: loc, type: i32Ty, value: size); |
| 274 | Value waitList; |
| 275 | if (size == 0) { |
| 276 | waitList = LLVM::ZeroOp::create(builder, location: loc, res: ptrTy); |
| 277 | } else { |
| 278 | waitList = LLVM::AllocaOp::create(builder, location: loc, resultType: ptrTy, elementType: i64Ty, arraySize: waitNum); |
| 279 | for (auto [index, queue] : llvm::enumerate(First&: queues)) { |
| 280 | Value idx = LLVM::ConstantOp::create(builder, location: loc, type: i32Ty, |
| 281 | value: static_cast<int64_t>(index)); |
| 282 | Value elementPtr = LLVM::GEPOp::create(builder, location: loc, resultType: ptrTy, elementType: i64Ty, |
| 283 | basePtr: waitList, indices: ArrayRef<Value>{idx}); |
| 284 | LLVM::StoreOp::create(builder, location: loc, value: queue, addr: elementPtr); |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | StringRef functionName = getParentFunctionName(values: waitOperands); |
| 289 | if (functionName.empty()) |
| 290 | if (Block *block = builder.getInsertionBlock()) |
| 291 | functionName = getParentFunctionName(op: block->getParentOp()); |
| 292 | Value ident = createIdent(loc, functionName, builder, globalSymbolRegion, |
| 293 | config, symbolTable: &symbolTable); |
| 294 | Value flags = LLVM::ConstantOp::create(builder, location: loc, type: i64Ty, value: 0); |
| 295 | Value deviceType = LLVM::ConstantOp::create( |
| 296 | builder, location: loc, type: i64Ty, value: config.getDeviceTypeRuntimeValue(type: DeviceType::None)); |
| 297 | if (!deviceNum) |
| 298 | deviceNum = LLVM::ConstantOp::create(builder, location: loc, type: i32Ty, value: 0); |
| 299 | |
| 300 | return createRuntimeCall( |
| 301 | loc, builder, globalSymbolRegion, symbolTable, |
| 302 | fn: RuntimeFunction::ACCRTL_tgt_acc_wait, config, |
| 303 | arguments: {ident, flags, deviceType, deviceNum, waitNum, waitList, asyncQueue}); |
| 304 | } |
| 305 | |
| 306 | FailureOr<Value> acc::emitGetDevicePtrCall(Operation *clauseOp, Value hostPtr, |
| 307 | bool ifPresent, OpBuilder &builder, |
| 308 | Region &globalSymbolRegion, |
| 309 | SymbolTable &symbolTable, |
| 310 | const ACCRuntimeCallConfig &config) { |
| 311 | Location loc = clauseOp->getLoc(); |
| 312 | Value ident = createIdent(loc, functionName: getParentFunctionName(op: clauseOp), builder, |
| 313 | globalSymbolRegion, config, symbolTable: &symbolTable); |
| 314 | Value flags = LLVM::ConstantOp::create( |
| 315 | builder, location: loc, type: builder.getI64Type(), |
| 316 | value: config.getMapFlagsRuntimeValue(flags: ifPresent ? MapFlags::if_present |
| 317 | : MapFlags::none)); |
| 318 | // Both slots hold the address of the object, a clause naming a section of |
| 319 | // one included: the entry point looks up the mapping that address belongs |
| 320 | // to rather than being told which one to ask about. |
| 321 | FailureOr<LLVM::CallOp> call = createRuntimeCall( |
| 322 | loc, builder, globalSymbolRegion, symbolTable, |
| 323 | fn: RuntimeFunction::ACCRTL_tgt_acc_get_deviceptr, config, |
| 324 | arguments: {ident, /*basePtr=*/hostPtr, flags, /*hostPtr=*/hostPtr}); |
| 325 | if (failed(Result: call)) |
| 326 | return failure(); |
| 327 | return call->getResult(); |
| 328 | } |
| 329 | |
| 330 | LogicalResult acc::emitGuardedByIfCond(Location loc, Value ifCond, |
| 331 | RewriterBase &rewriter, |
| 332 | function_ref<LogicalResult()> emitFn) { |
| 333 | if (!ifCond) |
| 334 | return emitFn(); |
| 335 | |
| 336 | Block *parentBlock = rewriter.getInsertionBlock(); |
| 337 | Block *continueBlock = |
| 338 | rewriter.splitBlock(block: parentBlock, before: rewriter.getInsertionPoint()); |
| 339 | Block *thenBlock = rewriter.createBlock( |
| 340 | parent: parentBlock->getParent(), insertPt: std::next(x: Region::iterator(parentBlock))); |
| 341 | |
| 342 | rewriter.setInsertionPointToEnd(parentBlock); |
| 343 | LLVM::CondBrOp::create(builder&: rewriter, location: loc, condition: ifCond, trueDest: thenBlock, trueOperands: ValueRange{}, |
| 344 | falseDest: continueBlock, falseOperands: ValueRange{}); |
| 345 | |
| 346 | rewriter.setInsertionPointToStart(thenBlock); |
| 347 | LogicalResult result = emitFn(); |
| 348 | rewriter.setInsertionPointToEnd(thenBlock); |
| 349 | LLVM::BrOp::create(builder&: rewriter, location: loc, operands: ValueRange{}, dest: continueBlock); |
| 350 | rewriter.setInsertionPointToStart(continueBlock); |
| 351 | return result; |
| 352 | } |
| 353 | |
| 354 | FailureOr<Value> |
| 355 | acc::emitValueSelectedByIfCond(Location loc, Value ifCond, |
| 356 | RewriterBase &rewriter, |
| 357 | function_ref<FailureOr<Value>()> thenFn, |
| 358 | function_ref<FailureOr<Value>()> elseFn) { |
| 359 | if (!ifCond) |
| 360 | return thenFn(); |
| 361 | |
| 362 | Block *parentBlock = rewriter.getInsertionBlock(); |
| 363 | Block *continueBlock = |
| 364 | rewriter.splitBlock(block: parentBlock, before: rewriter.getInsertionPoint()); |
| 365 | Block *thenBlock = rewriter.createBlock( |
| 366 | parent: parentBlock->getParent(), insertPt: std::next(x: Region::iterator(parentBlock))); |
| 367 | Block *elseBlock = rewriter.createBlock( |
| 368 | parent: parentBlock->getParent(), insertPt: std::next(x: Region::iterator(thenBlock))); |
| 369 | |
| 370 | rewriter.setInsertionPointToEnd(parentBlock); |
| 371 | LLVM::CondBrOp::create(builder&: rewriter, location: loc, condition: ifCond, trueDest: thenBlock, trueOperands: ValueRange{}, |
| 372 | falseDest: elseBlock, falseOperands: ValueRange{}); |
| 373 | |
| 374 | rewriter.setInsertionPointToStart(thenBlock); |
| 375 | FailureOr<Value> thenValue = thenFn(); |
| 376 | if (failed(Result: thenValue)) |
| 377 | return failure(); |
| 378 | rewriter.setInsertionPointToEnd(thenBlock); |
| 379 | LLVM::BrOp::create(builder&: rewriter, location: loc, operands: ValueRange{*thenValue}, dest: continueBlock); |
| 380 | |
| 381 | rewriter.setInsertionPointToStart(elseBlock); |
| 382 | FailureOr<Value> elseValue = elseFn(); |
| 383 | if (failed(Result: elseValue)) |
| 384 | return failure(); |
| 385 | rewriter.setInsertionPointToEnd(elseBlock); |
| 386 | LLVM::BrOp::create(builder&: rewriter, location: loc, operands: ValueRange{*elseValue}, dest: continueBlock); |
| 387 | |
| 388 | assert(thenValue->getType() == elseValue->getType() && |
| 389 | "both paths of an if clause have to produce the same type" ); |
| 390 | BlockArgument selected = |
| 391 | continueBlock->addArgument(type: thenValue->getType(), loc); |
| 392 | rewriter.setInsertionPointToStart(continueBlock); |
| 393 | return Value(selected); |
| 394 | } |
| 395 | |
| 396 | void acc::spliceConstructRegion(Operation *op, Region ®ion, |
| 397 | RewriterBase &rewriter) { |
| 398 | Block *prev = op->getBlock(); |
| 399 | Block *succ = rewriter.splitBlock(block: prev, before: Block::iterator(op)); |
| 400 | rewriter.setInsertionPointToEnd(prev); |
| 401 | LLVM::BrOp::create(builder&: rewriter, location: op->getLoc(), dest: ®ion.getBlocks().front()); |
| 402 | for (Block &block : region.getBlocks()) { |
| 403 | // A region that ends without a terminator, as an `acc.kernel_environment` |
| 404 | // body does, continues after the construct from the end of its block. |
| 405 | if (block.empty() || !block.back().hasTrait<OpTrait::IsTerminator>()) { |
| 406 | rewriter.setInsertionPointToEnd(&block); |
| 407 | LLVM::BrOp::create(builder&: rewriter, location: op->getLoc(), dest: succ); |
| 408 | continue; |
| 409 | } |
| 410 | Operation *terminator = block.getTerminator(); |
| 411 | if (isa<acc::TerminatorOp>(Val: terminator)) { |
| 412 | rewriter.setInsertionPoint(terminator); |
| 413 | LLVM::BrOp::create(builder&: rewriter, location: op->getLoc(), dest: succ); |
| 414 | rewriter.eraseOp(op: terminator); |
| 415 | } |
| 416 | } |
| 417 | rewriter.inlineRegionBefore(region, before: succ); |
| 418 | } |
| 419 | |