| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Emit OpenACC Stmt nodes as CIR code. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "CIRGenBuilder.h" |
| 14 | #include "CIRGenFunction.h" |
| 15 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 16 | #include "clang/AST/OpenACCClause.h" |
| 17 | #include "clang/AST/StmtOpenACC.h" |
| 18 | |
| 19 | using namespace clang; |
| 20 | using namespace clang::CIRGen; |
| 21 | using namespace cir; |
| 22 | using namespace mlir::acc; |
| 23 | |
| 24 | template <typename Op, typename TermOp> |
| 25 | mlir::LogicalResult CIRGenFunction::emitOpenACCOpAssociatedStmt( |
| 26 | mlir::Location start, mlir::Location end, OpenACCDirectiveKind dirKind, |
| 27 | llvm::ArrayRef<const OpenACCClause *> clauses, const Stmt *associatedStmt) { |
| 28 | mlir::LogicalResult res = mlir::success(); |
| 29 | |
| 30 | llvm::SmallVector<mlir::Type> retTy; |
| 31 | llvm::SmallVector<mlir::Value> operands; |
| 32 | auto op = Op::create(builder, start, retTy, operands, |
| 33 | cir::getDefaultProperties<Op>(builder.getContext())); |
| 34 | |
| 35 | emitOpenACCClauses(op, dirKind, clauses); |
| 36 | |
| 37 | { |
| 38 | mlir::Block &block = op.getRegion().emplaceBlock(); |
| 39 | mlir::OpBuilder::InsertionGuard guardCase(builder); |
| 40 | builder.setInsertionPointToEnd(&block); |
| 41 | |
| 42 | LexicalScope ls{*this, start, builder.getInsertionBlock()}; |
| 43 | if (associatedStmt) |
| 44 | res = emitStmt(s: associatedStmt, /*useCurrentScope=*/true); |
| 45 | |
| 46 | TermOp::create(builder, end); |
| 47 | } |
| 48 | return res; |
| 49 | } |
| 50 | |
| 51 | namespace { |
| 52 | template <typename Op> struct CombinedType; |
| 53 | template <> struct CombinedType<ParallelOp> { |
| 54 | static constexpr mlir::acc::CombinedConstructsType value = |
| 55 | mlir::acc::CombinedConstructsType::ParallelLoop; |
| 56 | }; |
| 57 | template <> struct CombinedType<SerialOp> { |
| 58 | static constexpr mlir::acc::CombinedConstructsType value = |
| 59 | mlir::acc::CombinedConstructsType::SerialLoop; |
| 60 | }; |
| 61 | template <> struct CombinedType<KernelsOp> { |
| 62 | static constexpr mlir::acc::CombinedConstructsType value = |
| 63 | mlir::acc::CombinedConstructsType::KernelsLoop; |
| 64 | }; |
| 65 | } // namespace |
| 66 | |
| 67 | template <typename Op, typename TermOp> |
| 68 | mlir::LogicalResult CIRGenFunction::emitOpenACCOpCombinedConstruct( |
| 69 | mlir::Location start, mlir::Location end, OpenACCDirectiveKind dirKind, |
| 70 | llvm::ArrayRef<const OpenACCClause *> clauses, const Stmt *loopStmt) { |
| 71 | mlir::LogicalResult res = mlir::success(); |
| 72 | |
| 73 | llvm::SmallVector<mlir::Type> retTy; |
| 74 | llvm::SmallVector<mlir::Value> operands; |
| 75 | |
| 76 | auto computeOp = |
| 77 | Op::create(builder, start, retTy, operands, |
| 78 | cir::getDefaultProperties<Op>(builder.getContext())); |
| 79 | computeOp.setCombinedAttr(builder.getUnitAttr()); |
| 80 | mlir::acc::LoopOp loopOp; |
| 81 | |
| 82 | // First, emit the bodies of both operations, with the loop inside the body of |
| 83 | // the combined construct. |
| 84 | { |
| 85 | mlir::Block &block = computeOp.getRegion().emplaceBlock(); |
| 86 | mlir::OpBuilder::InsertionGuard guardCase(builder); |
| 87 | builder.setInsertionPointToEnd(&block); |
| 88 | |
| 89 | LexicalScope ls{*this, start, builder.getInsertionBlock()}; |
| 90 | auto loopOp = |
| 91 | LoopOp::create(builder, location: start, resultTypes: retTy, operands, |
| 92 | properties: cir::getDefaultProperties<LoopOp>(context: builder.getContext())); |
| 93 | loopOp.setCombinedAttr(mlir::acc::CombinedConstructsTypeAttr::get( |
| 94 | context: builder.getContext(), value: CombinedType<Op>::value)); |
| 95 | |
| 96 | { |
| 97 | mlir::Block &innerBlock = loopOp.getRegion().emplaceBlock(); |
| 98 | mlir::OpBuilder::InsertionGuard guardCase(builder); |
| 99 | builder.setInsertionPointToEnd(&innerBlock); |
| 100 | |
| 101 | LexicalScope ls{*this, start, builder.getInsertionBlock()}; |
| 102 | ActiveOpenACCLoopRAII activeLoop{*this, &loopOp}; |
| 103 | |
| 104 | if (loopStmt) |
| 105 | res = emitStmt(s: loopStmt, /*useCurrentScope=*/true); |
| 106 | |
| 107 | mlir::acc::YieldOp::create(builder, location: end); |
| 108 | } |
| 109 | |
| 110 | emitOpenACCClauses(computeOp, loopOp, dirKind, clauses); |
| 111 | |
| 112 | updateLoopOpParallelism(op&: loopOp, /*isOrphan=*/false, dk: dirKind); |
| 113 | |
| 114 | TermOp::create(builder, end); |
| 115 | } |
| 116 | |
| 117 | return res; |
| 118 | } |
| 119 | |
| 120 | template <typename Op> |
| 121 | Op CIRGenFunction::emitOpenACCOp( |
| 122 | mlir::Location start, OpenACCDirectiveKind dirKind, |
| 123 | llvm::ArrayRef<const OpenACCClause *> clauses) { |
| 124 | llvm::SmallVector<mlir::Type> retTy; |
| 125 | llvm::SmallVector<mlir::Value> operands; |
| 126 | auto op = Op::create(builder, start, retTy, operands, |
| 127 | cir::getDefaultProperties<Op>(builder.getContext())); |
| 128 | |
| 129 | emitOpenACCClauses(op, dirKind, clauses); |
| 130 | return op; |
| 131 | } |
| 132 | |
| 133 | mlir::LogicalResult |
| 134 | CIRGenFunction::emitOpenACCComputeConstruct(const OpenACCComputeConstruct &s) { |
| 135 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 136 | mlir::Location end = getLoc(srcLoc: s.getSourceRange().getEnd()); |
| 137 | |
| 138 | switch (s.getDirectiveKind()) { |
| 139 | case OpenACCDirectiveKind::Parallel: |
| 140 | return emitOpenACCOpAssociatedStmt<ParallelOp, mlir::acc::YieldOp>( |
| 141 | start, end, dirKind: s.getDirectiveKind(), clauses: s.clauses(), associatedStmt: s.getStructuredBlock()); |
| 142 | case OpenACCDirectiveKind::Serial: |
| 143 | return emitOpenACCOpAssociatedStmt<SerialOp, mlir::acc::YieldOp>( |
| 144 | start, end, dirKind: s.getDirectiveKind(), clauses: s.clauses(), associatedStmt: s.getStructuredBlock()); |
| 145 | case OpenACCDirectiveKind::Kernels: |
| 146 | return emitOpenACCOpAssociatedStmt<KernelsOp, mlir::acc::TerminatorOp>( |
| 147 | start, end, dirKind: s.getDirectiveKind(), clauses: s.clauses(), associatedStmt: s.getStructuredBlock()); |
| 148 | default: |
| 149 | llvm_unreachable("invalid compute construct kind" ); |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | mlir::LogicalResult |
| 154 | CIRGenFunction::emitOpenACCDataConstruct(const OpenACCDataConstruct &s) { |
| 155 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 156 | mlir::Location end = getLoc(srcLoc: s.getSourceRange().getEnd()); |
| 157 | |
| 158 | return emitOpenACCOpAssociatedStmt<DataOp, mlir::acc::TerminatorOp>( |
| 159 | start, end, dirKind: s.getDirectiveKind(), clauses: s.clauses(), associatedStmt: s.getStructuredBlock()); |
| 160 | } |
| 161 | |
| 162 | mlir::LogicalResult |
| 163 | CIRGenFunction::emitOpenACCInitConstruct(const OpenACCInitConstruct &s) { |
| 164 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 165 | emitOpenACCOp<InitOp>(start, dirKind: s.getDirectiveKind(), clauses: s.clauses()); |
| 166 | return mlir::success(); |
| 167 | } |
| 168 | |
| 169 | mlir::LogicalResult |
| 170 | CIRGenFunction::emitOpenACCSetConstruct(const OpenACCSetConstruct &s) { |
| 171 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 172 | emitOpenACCOp<SetOp>(start, dirKind: s.getDirectiveKind(), clauses: s.clauses()); |
| 173 | return mlir::success(); |
| 174 | } |
| 175 | |
| 176 | mlir::LogicalResult CIRGenFunction::emitOpenACCShutdownConstruct( |
| 177 | const OpenACCShutdownConstruct &s) { |
| 178 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 179 | emitOpenACCOp<ShutdownOp>(start, dirKind: s.getDirectiveKind(), clauses: s.clauses()); |
| 180 | return mlir::success(); |
| 181 | } |
| 182 | |
| 183 | mlir::LogicalResult |
| 184 | CIRGenFunction::emitOpenACCWaitConstruct(const OpenACCWaitConstruct &s) { |
| 185 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 186 | auto waitOp = emitOpenACCOp<WaitOp>(start, dirKind: s.getDirectiveKind(), clauses: s.clauses()); |
| 187 | |
| 188 | auto createIntExpr = [this](const Expr *intExpr) { |
| 189 | mlir::Value expr = emitScalarExpr(e: intExpr); |
| 190 | mlir::Location exprLoc = cgm.getLoc(cLoc: intExpr->getBeginLoc()); |
| 191 | |
| 192 | mlir::IntegerType targetType = mlir::IntegerType::get( |
| 193 | context: &getMLIRContext(), width: getContext().getIntWidth(T: intExpr->getType()), |
| 194 | signedness: intExpr->getType()->isSignedIntegerOrEnumerationType() |
| 195 | ? mlir::IntegerType::SignednessSemantics::Signed |
| 196 | : mlir::IntegerType::SignednessSemantics::Unsigned); |
| 197 | |
| 198 | return builder.createBuiltinIntCast(loc: exprLoc, src: expr, newTy: targetType); |
| 199 | }; |
| 200 | |
| 201 | // Emit the correct 'wait' clauses. |
| 202 | { |
| 203 | mlir::OpBuilder::InsertionGuard guardCase(builder); |
| 204 | builder.setInsertionPoint(waitOp); |
| 205 | |
| 206 | if (s.hasDevNumExpr()) |
| 207 | waitOp.getWaitDevnumMutable().append(values: createIntExpr(s.getDevNumExpr())); |
| 208 | |
| 209 | for (Expr *QueueExpr : s.getQueueIdExprs()) |
| 210 | waitOp.getWaitOperandsMutable().append(values: createIntExpr(QueueExpr)); |
| 211 | } |
| 212 | |
| 213 | return mlir::success(); |
| 214 | } |
| 215 | |
| 216 | mlir::LogicalResult CIRGenFunction::emitOpenACCCombinedConstruct( |
| 217 | const OpenACCCombinedConstruct &s) { |
| 218 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 219 | mlir::Location end = getLoc(srcLoc: s.getSourceRange().getEnd()); |
| 220 | |
| 221 | switch (s.getDirectiveKind()) { |
| 222 | case OpenACCDirectiveKind::ParallelLoop: |
| 223 | return emitOpenACCOpCombinedConstruct<ParallelOp, mlir::acc::YieldOp>( |
| 224 | start, end, dirKind: s.getDirectiveKind(), clauses: s.clauses(), loopStmt: s.getLoop()); |
| 225 | case OpenACCDirectiveKind::SerialLoop: |
| 226 | return emitOpenACCOpCombinedConstruct<SerialOp, mlir::acc::YieldOp>( |
| 227 | start, end, dirKind: s.getDirectiveKind(), clauses: s.clauses(), loopStmt: s.getLoop()); |
| 228 | case OpenACCDirectiveKind::KernelsLoop: |
| 229 | return emitOpenACCOpCombinedConstruct<KernelsOp, mlir::acc::TerminatorOp>( |
| 230 | start, end, dirKind: s.getDirectiveKind(), clauses: s.clauses(), loopStmt: s.getLoop()); |
| 231 | default: |
| 232 | llvm_unreachable("invalid compute construct kind" ); |
| 233 | } |
| 234 | } |
| 235 | |
| 236 | mlir::LogicalResult CIRGenFunction::emitOpenACCHostDataConstruct( |
| 237 | const OpenACCHostDataConstruct &s) { |
| 238 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 239 | mlir::Location end = getLoc(srcLoc: s.getSourceRange().getEnd()); |
| 240 | |
| 241 | return emitOpenACCOpAssociatedStmt<HostDataOp, mlir::acc::TerminatorOp>( |
| 242 | start, end, dirKind: s.getDirectiveKind(), clauses: s.clauses(), associatedStmt: s.getStructuredBlock()); |
| 243 | } |
| 244 | |
| 245 | mlir::LogicalResult CIRGenFunction::emitOpenACCEnterDataConstruct( |
| 246 | const OpenACCEnterDataConstruct &s) { |
| 247 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 248 | emitOpenACCOp<EnterDataOp>(start, dirKind: s.getDirectiveKind(), clauses: s.clauses()); |
| 249 | return mlir::success(); |
| 250 | } |
| 251 | |
| 252 | mlir::LogicalResult CIRGenFunction::emitOpenACCExitDataConstruct( |
| 253 | const OpenACCExitDataConstruct &s) { |
| 254 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 255 | emitOpenACCOp<ExitDataOp>(start, dirKind: s.getDirectiveKind(), clauses: s.clauses()); |
| 256 | return mlir::success(); |
| 257 | } |
| 258 | |
| 259 | mlir::LogicalResult |
| 260 | CIRGenFunction::emitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &s) { |
| 261 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 262 | emitOpenACCOp<UpdateOp>(start, dirKind: s.getDirectiveKind(), clauses: s.clauses()); |
| 263 | return mlir::success(); |
| 264 | } |
| 265 | |
| 266 | mlir::LogicalResult |
| 267 | CIRGenFunction::emitOpenACCCacheConstruct(const OpenACCCacheConstruct &s) { |
| 268 | // The 'cache' directive 'may' be at the top of a loop by standard, but |
| 269 | // doesn't have to be. Additionally, there is nothing that requires this be a |
| 270 | // loop affected by an OpenACC pragma. Sema doesn't do any level of |
| 271 | // enforcement here, since it isn't particularly valuable to do so thanks to |
| 272 | // that. Instead, we treat cache as a 'noop' if there is no acc.loop to apply |
| 273 | // it to. |
| 274 | if (!activeLoopOp) |
| 275 | return mlir::success(); |
| 276 | |
| 277 | mlir::acc::LoopOp loopOp = *activeLoopOp; |
| 278 | |
| 279 | mlir::OpBuilder::InsertionGuard guard(builder); |
| 280 | builder.setInsertionPoint(loopOp); |
| 281 | |
| 282 | for (const Expr *var : s.getVarList()) { |
| 283 | CIRGenFunction::OpenACCDataOperandInfo opInfo = |
| 284 | getOpenACCDataOperandInfo(e: var); |
| 285 | |
| 286 | auto cacheOp = CacheOp::create(builder, location: opInfo.beginLoc, var: opInfo.varValue, |
| 287 | /*structured=*/false, /*implicit=*/false, |
| 288 | name: opInfo.name, bounds: opInfo.bounds); |
| 289 | |
| 290 | loopOp.getCacheOperandsMutable().append(values: cacheOp.getResult()); |
| 291 | } |
| 292 | |
| 293 | return mlir::success(); |
| 294 | } |
| 295 | |
| 296 | const VarDecl *getLValueDecl(const Expr *e) { |
| 297 | // We are going to assume that after stripping implicit casts, that the LValue |
| 298 | // is just a DRE around the var-decl. |
| 299 | |
| 300 | e = e->IgnoreImpCasts(); |
| 301 | |
| 302 | const auto *dre = cast<DeclRefExpr>(Val: e); |
| 303 | return cast<VarDecl>(Val: dre->getDecl()); |
| 304 | } |
| 305 | |
| 306 | static mlir::acc::AtomicReadOp |
| 307 | emitAtomicRead(CIRGenFunction &cgf, CIRGenBuilderTy &builder, |
| 308 | mlir::Location start, |
| 309 | const OpenACCAtomicConstruct::SingleStmtInfo &inf) { |
| 310 | // Atomic 'read' only permits 'v = x', where v and x are both scalar L |
| 311 | // values. The getAssociatedStmtInfo strips off implicit casts, which |
| 312 | // includes implicit conversions and L-to-R-Value conversions, so we can |
| 313 | // just emit it as an L value. The Flang implementation has no problem with |
| 314 | // different types, so it appears that the dialect can handle the |
| 315 | // conversions. |
| 316 | mlir::Value v = cgf.emitLValue(e: inf.V).getPointer(); |
| 317 | mlir::Value x = cgf.emitLValue(e: inf.X).getPointer(); |
| 318 | mlir::Type resTy = cgf.convertType(t: inf.V->getType()); |
| 319 | return mlir::acc::AtomicReadOp::create(builder, location: start, x, v, element_type: resTy, |
| 320 | /*ifCond=*/{}); |
| 321 | } |
| 322 | |
| 323 | static mlir::acc::AtomicWriteOp |
| 324 | emitAtomicWrite(CIRGenFunction &cgf, CIRGenBuilderTy &builder, |
| 325 | mlir::Location start, |
| 326 | const OpenACCAtomicConstruct::SingleStmtInfo &inf) { |
| 327 | mlir::Value x = cgf.emitLValue(e: inf.X).getPointer(); |
| 328 | mlir::Value expr = cgf.emitAnyExpr(e: inf.RefExpr).getValue(); |
| 329 | return mlir::acc::AtomicWriteOp::create(builder, location: start, x, expr, |
| 330 | /*ifCond=*/{}); |
| 331 | } |
| 332 | |
| 333 | static std::pair<mlir::LogicalResult, mlir::acc::AtomicUpdateOp> |
| 334 | emitAtomicUpdate(CIRGenFunction &cgf, CIRGenBuilderTy &builder, |
| 335 | mlir::Location start, mlir::Location end, |
| 336 | const OpenACCAtomicConstruct::SingleStmtInfo &inf) { |
| 337 | mlir::Value x = cgf.emitLValue(e: inf.X).getPointer(); |
| 338 | auto op = mlir::acc::AtomicUpdateOp::create(builder, location: start, x, /*ifCond=*/{}); |
| 339 | |
| 340 | mlir::LogicalResult res = mlir::success(); |
| 341 | { |
| 342 | mlir::OpBuilder::InsertionGuard guardCase(builder); |
| 343 | mlir::Type argTy = cast<cir::PointerType>(Val: x.getType()).getPointee(); |
| 344 | std::array<mlir::Type, 1> recipeType{argTy}; |
| 345 | std::array<mlir::Location, 1> recipeLoc{start}; |
| 346 | auto *recipeBlock = builder.createBlock( |
| 347 | parent: &op.getRegion(), insertPt: op.getRegion().end(), argTypes: recipeType, locs: recipeLoc); |
| 348 | builder.setInsertionPointToEnd(recipeBlock); |
| 349 | // Since we have an initial value that we know is a scalar type, we can |
| 350 | // just emit the entire statement here after sneaking-in our 'alloca' in |
| 351 | // the right place, then loading out of it. Flang does a lot less work |
| 352 | // (probably does its own emitting!), but we have more complicated AST |
| 353 | // nodes to worry about, so we can just count on opt to remove the extra |
| 354 | // alloca/load/store set. |
| 355 | auto alloca = cir::AllocaOp::create( |
| 356 | builder, location: start, addr: x.getType(), name: "x_var" , |
| 357 | alignment: cgf.cgm.getSize( |
| 358 | size: cgf.getContext().getTypeAlignInChars(T: inf.X->getType()))); |
| 359 | |
| 360 | alloca.setInitAttr(builder.getUnitAttr()); |
| 361 | builder.CIRBaseBuilderTy::createStore(loc: start, val: recipeBlock->getArgument(i: 0), |
| 362 | dst: alloca); |
| 363 | |
| 364 | const VarDecl *xval = getLValueDecl(e: inf.X); |
| 365 | CIRGenFunction::DeclMapRevertingRAII declMapRAII{cgf, xval}; |
| 366 | cgf.replaceAddrOfLocalVar( |
| 367 | vd: xval, addr: Address{alloca, argTy, cgf.getContext().getDeclAlign(D: xval)}); |
| 368 | |
| 369 | if (inf.WholeExpr) |
| 370 | res = cgf.emitStmt(s: inf.WholeExpr, /*useCurrentScope=*/true); |
| 371 | |
| 372 | auto load = cir::LoadOp::create(builder, location: start, addr: alloca.getResult()); |
| 373 | mlir::acc::YieldOp::create(builder, location: end, operands: {load}); |
| 374 | } |
| 375 | |
| 376 | return {res, op}; |
| 377 | } |
| 378 | |
| 379 | mlir::LogicalResult |
| 380 | CIRGenFunction::emitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &s) { |
| 381 | // While Atomic is an 'associated statement' construct, it 'steals' the |
| 382 | // expression it is associated with rather than emitting it inside of it. So |
| 383 | // it has custom emit logic. |
| 384 | mlir::Location start = getLoc(srcLoc: s.getSourceRange().getBegin()); |
| 385 | mlir::Location end = getLoc(srcLoc: s.getSourceRange().getEnd()); |
| 386 | OpenACCAtomicConstruct::StmtInfo inf = s.getAssociatedStmtInfo(); |
| 387 | |
| 388 | switch (s.getAtomicKind()) { |
| 389 | case OpenACCAtomicKind::Read: { |
| 390 | assert(inf.Form == OpenACCAtomicConstruct::StmtInfo::StmtForm::Read); |
| 391 | mlir::acc::AtomicReadOp op = |
| 392 | emitAtomicRead(cgf&: *this, builder, start, inf: inf.First); |
| 393 | emitOpenACCClauses(op, dirKind: s.getDirectiveKind(), clauses: s.clauses()); |
| 394 | return mlir::success(); |
| 395 | } |
| 396 | case OpenACCAtomicKind::Write: { |
| 397 | assert(inf.Form == OpenACCAtomicConstruct::StmtInfo::StmtForm::Write); |
| 398 | auto op = emitAtomicWrite(cgf&: *this, builder, start, inf: inf.First); |
| 399 | emitOpenACCClauses(op, dirKind: s.getDirectiveKind(), clauses: s.clauses()); |
| 400 | return mlir::success(); |
| 401 | } |
| 402 | case OpenACCAtomicKind::None: |
| 403 | case OpenACCAtomicKind::Update: { |
| 404 | assert(inf.Form == OpenACCAtomicConstruct::StmtInfo::StmtForm::Update); |
| 405 | auto [res, op] = emitAtomicUpdate(cgf&: *this, builder, start, end, inf: inf.First); |
| 406 | emitOpenACCClauses(op, dirKind: s.getDirectiveKind(), clauses: s.clauses()); |
| 407 | return res; |
| 408 | } |
| 409 | case OpenACCAtomicKind::Capture: { |
| 410 | // Atomic-capture is made up of two statements, either an update = read, |
| 411 | // read + update, or read + write. As a result, the IR represents the |
| 412 | // capture region as having those two 'inside' of it. |
| 413 | auto op = mlir::acc::AtomicCaptureOp::create(builder, location: start, /*ifCond=*/{}); |
| 414 | emitOpenACCClauses(op, dirKind: s.getDirectiveKind(), clauses: s.clauses()); |
| 415 | mlir::LogicalResult res = mlir::success(); |
| 416 | { |
| 417 | mlir::OpBuilder::InsertionGuard guardCase(builder); |
| 418 | |
| 419 | mlir::Block *block = |
| 420 | builder.createBlock(parent: &op.getRegion(), insertPt: op.getRegion().end(), argTypes: {}, locs: {}); |
| 421 | |
| 422 | builder.setInsertionPointToStart(block); |
| 423 | |
| 424 | auto terminator = mlir::acc::TerminatorOp::create(builder, location: end); |
| 425 | |
| 426 | // The AtomicCaptureOp only permits the two acc.atomic.* operations inside |
| 427 | // of it, so all other parts of the expression need to be emitted before |
| 428 | // the AtomicCaptureOp, then moved into place. |
| 429 | builder.setInsertionPoint(op); |
| 430 | |
| 431 | switch (inf.Form) { |
| 432 | default: |
| 433 | llvm_unreachable("invalid form for Capture" ); |
| 434 | case OpenACCAtomicConstruct::StmtInfo::StmtForm::ReadWrite: { |
| 435 | mlir::acc::AtomicReadOp first = |
| 436 | emitAtomicRead(cgf&: *this, builder, start, inf: inf.First); |
| 437 | mlir::acc::AtomicWriteOp second = |
| 438 | emitAtomicWrite(cgf&: *this, builder, start, inf: inf.Second); |
| 439 | |
| 440 | first->moveBefore(existingOp: terminator); |
| 441 | second->moveBefore(existingOp: terminator); |
| 442 | break; |
| 443 | } |
| 444 | case OpenACCAtomicConstruct::StmtInfo::StmtForm::ReadUpdate: { |
| 445 | mlir::acc::AtomicReadOp first = |
| 446 | emitAtomicRead(cgf&: *this, builder, start, inf: inf.First); |
| 447 | auto [this_res, second] = |
| 448 | emitAtomicUpdate(cgf&: *this, builder, start, end, inf: inf.Second); |
| 449 | res = this_res; |
| 450 | |
| 451 | first->moveBefore(existingOp: terminator); |
| 452 | second->moveBefore(existingOp: terminator); |
| 453 | break; |
| 454 | } |
| 455 | case OpenACCAtomicConstruct::StmtInfo::StmtForm::UpdateRead: { |
| 456 | auto [this_res, first] = |
| 457 | emitAtomicUpdate(cgf&: *this, builder, start, end, inf: inf.First); |
| 458 | res = this_res; |
| 459 | mlir::acc::AtomicReadOp second = |
| 460 | emitAtomicRead(cgf&: *this, builder, start, inf: inf.Second); |
| 461 | |
| 462 | first->moveBefore(existingOp: terminator); |
| 463 | second->moveBefore(existingOp: terminator); |
| 464 | break; |
| 465 | } |
| 466 | } |
| 467 | } |
| 468 | return res; |
| 469 | } |
| 470 | } |
| 471 | |
| 472 | llvm_unreachable("unknown OpenACC atomic kind" ); |
| 473 | } |
| 474 | |