| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Helperes to emit OpenACC clause recipes as CIR code. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include <numeric> |
| 14 | |
| 15 | #include "CIRGenOpenACCRecipe.h" |
| 16 | |
| 17 | namespace clang::CIRGen { |
| 18 | mlir::Block *OpenACCRecipeBuilderBase::createRecipeBlock(mlir::Region ®ion, |
| 19 | mlir::Type opTy, |
| 20 | mlir::Location loc, |
| 21 | size_t numBounds, |
| 22 | bool isInit) { |
| 23 | llvm::SmallVector<mlir::Type> types; |
| 24 | types.reserve(N: numBounds + 2); |
| 25 | types.push_back(Elt: opTy); |
| 26 | // The init section is the only one that doesn't have TWO copies of the |
| 27 | // operation-type. Copy has a to/from, and destroy has a |
| 28 | // 'reference'/'privatized' copy version. |
| 29 | if (!isInit) |
| 30 | types.push_back(Elt: opTy); |
| 31 | |
| 32 | auto boundsTy = mlir::acc::DataBoundsType::get(ctx: &cgf.getMLIRContext()); |
| 33 | for (size_t i = 0; i < numBounds; ++i) |
| 34 | types.push_back(Elt: boundsTy); |
| 35 | |
| 36 | llvm::SmallVector<mlir::Location> locs{types.size(), loc}; |
| 37 | return builder.createBlock(parent: ®ion, insertPt: region.end(), argTypes: types, locs); |
| 38 | } |
| 39 | void OpenACCRecipeBuilderBase::makeAllocaCopy(mlir::Location loc, |
| 40 | mlir::Type copyType, |
| 41 | mlir::Value numEltsToCopy, |
| 42 | mlir::Value offsetPerSubarray, |
| 43 | mlir::Value destAlloca, |
| 44 | mlir::Value srcAlloca) { |
| 45 | mlir::OpBuilder::InsertionGuard guardCase(builder); |
| 46 | |
| 47 | mlir::Type itrTy = cgf.cgm.convertType(type: cgf.getContext().UnsignedLongLongTy); |
| 48 | auto itrPtrTy = cir::PointerType::get(pointee: itrTy); |
| 49 | mlir::IntegerAttr itrAlign = |
| 50 | cgf.cgm.getSize(size: cgf.getContext().getTypeAlignInChars( |
| 51 | T: cgf.getContext().UnsignedLongLongTy)); |
| 52 | |
| 53 | auto loopBuilder = [&]() { |
| 54 | auto itr = cir::AllocaOp::create(builder, location: loc, addr: itrPtrTy, name: "itr" , alignment: itrAlign); |
| 55 | cir::ConstantOp constZero = builder.getConstInt(loc, t: itrTy, c: 0); |
| 56 | builder.CIRBaseBuilderTy::createStore(loc, val: constZero, dst: itr); |
| 57 | builder.createFor( |
| 58 | loc, |
| 59 | /*condBuilder=*/ |
| 60 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 61 | // itr < numEltsToCopy |
| 62 | // Enforce a trip count of 1 if there wasn't any element count, this |
| 63 | // way we can just use this loop with a constant bounds instead of a |
| 64 | // separate code path. |
| 65 | if (!numEltsToCopy) |
| 66 | numEltsToCopy = builder.getConstInt(loc, t: itrTy, c: 1); |
| 67 | |
| 68 | auto loadCur = cir::LoadOp::create(builder, location: loc, addr: itr.getResult()); |
| 69 | auto cmp = builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: loadCur, |
| 70 | rhs: numEltsToCopy); |
| 71 | builder.createCondition(condition: cmp); |
| 72 | }, |
| 73 | /*bodyBuilder=*/ |
| 74 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 75 | // destAlloca[itr] = srcAlloca[offsetPerSubArray * itr]; |
| 76 | auto loadCur = cir::LoadOp::create(builder, location: loc, addr: itr.getResult()); |
| 77 | auto srcOffset = builder.createMul(loc, lhs: offsetPerSubarray, rhs: loadCur); |
| 78 | |
| 79 | auto ptrToOffsetIntoSrc = cir::PtrStrideOp::create( |
| 80 | builder, location: loc, result: copyType, base: srcAlloca, stride: srcOffset); |
| 81 | |
| 82 | auto offsetIntoDecayDest = cir::PtrStrideOp::create( |
| 83 | builder, location: loc, result: builder.getPointerTo(ty: copyType), base: destAlloca, |
| 84 | stride: loadCur); |
| 85 | |
| 86 | builder.CIRBaseBuilderTy::createStore(loc, val: ptrToOffsetIntoSrc, |
| 87 | dst: offsetIntoDecayDest); |
| 88 | builder.createYield(loc); |
| 89 | }, |
| 90 | /*stepBuilder=*/ |
| 91 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 92 | // Simple increment of the iterator. |
| 93 | auto load = cir::LoadOp::create(builder, location: loc, addr: itr.getResult()); |
| 94 | auto inc = builder.createInc(loc, input: load); |
| 95 | builder.CIRBaseBuilderTy::createStore(loc, val: inc, dst: itr); |
| 96 | builder.createYield(loc); |
| 97 | }); |
| 98 | }; |
| 99 | |
| 100 | cir::ScopeOp::create(builder, location: loc, |
| 101 | scopeBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 102 | loopBuilder(); |
| 103 | builder.createYield(loc); |
| 104 | }); |
| 105 | } |
| 106 | |
| 107 | mlir::Value OpenACCRecipeBuilderBase::makeBoundsAlloca( |
| 108 | mlir::Block *block, SourceRange exprRange, mlir::Location loc, |
| 109 | std::string_view allocaName, size_t numBounds, |
| 110 | llvm::ArrayRef<QualType> boundTypes) { |
| 111 | mlir::OpBuilder::InsertionGuard guardCase(builder); |
| 112 | |
| 113 | // Get the range of bounds arguments, which are all but the 1st arg. |
| 114 | llvm::ArrayRef<mlir::BlockArgument> boundsRange = |
| 115 | block->getArguments().drop_front(N: 1); |
| 116 | |
| 117 | // boundTypes contains the before and after of each bounds, so it ends up |
| 118 | // having 1 extra. Assert this is the case to ensure we don't call this in the |
| 119 | // wrong 'block'. |
| 120 | assert(boundsRange.size() + 1 == boundTypes.size()); |
| 121 | |
| 122 | mlir::Type itrTy = cgf.cgm.convertType(type: cgf.getContext().UnsignedLongLongTy); |
| 123 | auto idxType = mlir::IndexType::get(context: &cgf.getMLIRContext()); |
| 124 | |
| 125 | auto getUpperBound = [&](mlir::Value bound) { |
| 126 | auto upperBoundVal = |
| 127 | mlir::acc::GetUpperboundOp::create(builder, location: loc, result: idxType, bounds: bound); |
| 128 | return builder.createBuiltinIntCast(loc, src: upperBoundVal.getResult(), newTy: itrTy); |
| 129 | }; |
| 130 | |
| 131 | auto isArrayTy = [&](QualType ty) { |
| 132 | if (ty->isArrayType() && !ty->isConstantArrayType()) |
| 133 | cgf.cgm.errorNYI(exprRange, "OpenACC recipe init for VLAs" ); |
| 134 | return ty->isConstantArrayType(); |
| 135 | }; |
| 136 | |
| 137 | mlir::Type topLevelTy = cgf.convertType(t: boundTypes.back()); |
| 138 | cir::PointerType topLevelTyPtr = builder.getPointerTo(ty: topLevelTy); |
| 139 | // Do an alloca for the 'top' level type without bounds. |
| 140 | mlir::Value initialAlloca = builder.createAlloca( |
| 141 | loc, addrType: topLevelTyPtr, name: allocaName, |
| 142 | alignment: cgf.getContext().getTypeAlignInChars(T: boundTypes.back())); |
| 143 | |
| 144 | bool lastBoundWasArray = isArrayTy(boundTypes.back()); |
| 145 | |
| 146 | // Make sure we track a moving version of this so we can get our |
| 147 | // 'copying' back to correct. |
| 148 | mlir::Value lastAlloca = initialAlloca; |
| 149 | |
| 150 | // Since we're iterating the types in reverse, this sets up for each index |
| 151 | // corresponding to the boundsRange to be the 'after application of the |
| 152 | // bounds. |
| 153 | llvm::ArrayRef<QualType> boundResults = boundTypes.drop_back(N: 1); |
| 154 | |
| 155 | // Collect the 'do we have any allocas needed after this type' list. |
| 156 | llvm::SmallVector<bool> allocasLeftArr; |
| 157 | llvm::ArrayRef<QualType> resultTypes = boundTypes.drop_front(); |
| 158 | bool accumulator = false; |
| 159 | for (QualType ty : resultTypes) { |
| 160 | accumulator = accumulator || !ty->isConstantArrayType(); |
| 161 | allocasLeftArr.push_back(Elt: accumulator); |
| 162 | } |
| 163 | |
| 164 | // Keep track of the number of 'elements' that we're allocating. Individual |
| 165 | // allocas should multiply this by the size of its current allocation. |
| 166 | mlir::Value cumulativeElts; |
| 167 | for (auto [bound, resultType, allocasLeft] : llvm::reverse( |
| 168 | C: llvm::zip_equal(t&: boundsRange, u&: boundResults, args&: allocasLeftArr))) { |
| 169 | |
| 170 | // if there is no further 'alloca' operation we need to do, we can skip |
| 171 | // creating the UB/multiplications/etc. |
| 172 | if (!allocasLeft) |
| 173 | break; |
| 174 | |
| 175 | // First: figure out the number of elements in the current 'bound' list. |
| 176 | mlir::Value eltsPerSubArray = getUpperBound(bound); |
| 177 | mlir::Value eltsToAlloca; |
| 178 | |
| 179 | // IF we are in a sub-bounds, the total number of elements to alloca is |
| 180 | // the product of that one and the current 'bounds' size. That is, |
| 181 | // arr[5][5], we would need 25 elements, not just 5. Else it is just the |
| 182 | // current number of elements. |
| 183 | if (cumulativeElts) |
| 184 | eltsToAlloca = builder.createMul(loc, lhs: eltsPerSubArray, rhs: cumulativeElts); |
| 185 | else |
| 186 | eltsToAlloca = eltsPerSubArray; |
| 187 | |
| 188 | if (!lastBoundWasArray) { |
| 189 | // If we have to do an allocation, figure out the size of the |
| 190 | // allocation. alloca takes the number of bytes, not elements. |
| 191 | TypeInfoChars eltInfo = cgf.getContext().getTypeInfoInChars(T: resultType); |
| 192 | cir::ConstantOp eltSize = builder.getConstInt( |
| 193 | loc, t: itrTy, c: eltInfo.Width.alignTo(Align: eltInfo.Align).getQuantity()); |
| 194 | mlir::Value curSize = builder.createMul(loc, lhs: eltsToAlloca, rhs: eltSize); |
| 195 | |
| 196 | mlir::Type eltTy = cgf.convertType(t: resultType); |
| 197 | cir::PointerType ptrTy = builder.getPointerTo(ty: eltTy); |
| 198 | mlir::Value curAlloca = builder.createAlloca( |
| 199 | loc, addrType: ptrTy, type: eltTy, name: "openacc.init.bounds" , |
| 200 | alignment: cgf.getContext().getTypeAlignInChars(T: resultType), dynAllocSize: curSize); |
| 201 | |
| 202 | makeAllocaCopy(loc, copyType: ptrTy, numEltsToCopy: cumulativeElts, offsetPerSubarray: eltsPerSubArray, destAlloca: lastAlloca, |
| 203 | srcAlloca: curAlloca); |
| 204 | lastAlloca = curAlloca; |
| 205 | } else { |
| 206 | // In the case of an array, we just need to decay the pointer, so just do |
| 207 | // a zero-offset stride on the last alloca to decay it down an array |
| 208 | // level. |
| 209 | cir::ConstantOp constZero = builder.getConstInt(loc, t: itrTy, c: 0); |
| 210 | lastAlloca = builder.getArrayElement(arrayLocBegin: loc, arrayLocEnd: loc, arrayPtr: lastAlloca, |
| 211 | eltTy: cgf.convertType(t: resultType), |
| 212 | idx: constZero, /*shouldDecay=*/true); |
| 213 | } |
| 214 | |
| 215 | cumulativeElts = eltsToAlloca; |
| 216 | lastBoundWasArray = isArrayTy(resultType); |
| 217 | } |
| 218 | return initialAlloca; |
| 219 | } |
| 220 | |
| 221 | std::pair<mlir::Value, mlir::Value> OpenACCRecipeBuilderBase::createBoundsLoop( |
| 222 | mlir::Value subscriptedValue, mlir::Value subscriptedValue2, |
| 223 | mlir::Value bound, mlir::Location loc, bool inverse) { |
| 224 | mlir::Operation *bodyInsertLoc; |
| 225 | |
| 226 | mlir::Type itrTy = cgf.cgm.convertType(type: cgf.getContext().UnsignedLongLongTy); |
| 227 | auto itrPtrTy = cir::PointerType::get(pointee: itrTy); |
| 228 | mlir::IntegerAttr itrAlign = |
| 229 | cgf.cgm.getSize(size: cgf.getContext().getTypeAlignInChars( |
| 230 | T: cgf.getContext().UnsignedLongLongTy)); |
| 231 | auto idxType = mlir::IndexType::get(context: &cgf.getMLIRContext()); |
| 232 | |
| 233 | auto doSubscriptOp = [&](mlir::Value subVal, |
| 234 | cir::LoadOp idxLoad) -> mlir::Value { |
| 235 | auto eltTy = cast<cir::PointerType>(Val: subVal.getType()).getPointee(); |
| 236 | |
| 237 | if (auto arrayTy = dyn_cast<cir::ArrayType>(Val&: eltTy)) |
| 238 | return builder.getArrayElement(arrayLocBegin: loc, arrayLocEnd: loc, arrayPtr: subVal, eltTy: arrayTy.getElementType(), |
| 239 | idx: idxLoad, |
| 240 | /*shouldDecay=*/true); |
| 241 | |
| 242 | assert(isa<cir::PointerType>(eltTy)); |
| 243 | |
| 244 | auto eltLoad = cir::LoadOp::create(builder, location: loc, addr: subVal); |
| 245 | |
| 246 | return cir::PtrStrideOp::create(builder, location: loc, result: eltLoad.getType(), base: eltLoad, |
| 247 | stride: idxLoad); |
| 248 | }; |
| 249 | |
| 250 | auto forStmtBuilder = [&]() { |
| 251 | // get the lower and upper bound for iterating over. |
| 252 | auto lowerBoundVal = |
| 253 | mlir::acc::GetLowerboundOp::create(builder, location: loc, result: idxType, bounds: bound); |
| 254 | mlir::Value lbConversion = |
| 255 | builder.createBuiltinIntCast(loc, src: lowerBoundVal.getResult(), newTy: itrTy); |
| 256 | auto upperBoundVal = |
| 257 | mlir::acc::GetUpperboundOp::create(builder, location: loc, result: idxType, bounds: bound); |
| 258 | mlir::Value ubConversion = |
| 259 | builder.createBuiltinIntCast(loc, src: upperBoundVal.getResult(), newTy: itrTy); |
| 260 | |
| 261 | // Create a memory location for the iterator. |
| 262 | auto itr = cir::AllocaOp::create(builder, location: loc, addr: itrPtrTy, name: "iter" , alignment: itrAlign); |
| 263 | // Store to the iterator: either lower bound, or if inverse loop, upper |
| 264 | // bound. |
| 265 | if (inverse) { |
| 266 | cir::ConstantOp constOne = builder.getConstInt(loc, t: itrTy, c: 1); |
| 267 | |
| 268 | auto sub = cir::SubOp::create(builder, location: loc, lhs: ubConversion, rhs: constOne); |
| 269 | |
| 270 | // Upperbound is exclusive, so subtract 1. |
| 271 | builder.CIRBaseBuilderTy::createStore(loc, val: sub, dst: itr); |
| 272 | } else { |
| 273 | // Lowerbound is inclusive, so we can include it. |
| 274 | builder.CIRBaseBuilderTy::createStore(loc, val: lbConversion, dst: itr); |
| 275 | } |
| 276 | // Save the 'end' iterator based on whether we are inverted or not. This |
| 277 | // end iterator never changes, so we can just get it and convert it, so no |
| 278 | // need to store/load/etc. |
| 279 | mlir::Value endItr = inverse ? lbConversion : ubConversion; |
| 280 | |
| 281 | builder.createFor( |
| 282 | loc, |
| 283 | /*condBuilder=*/ |
| 284 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 285 | auto loadCur = cir::LoadOp::create(builder, location: loc, addr: itr.getResult()); |
| 286 | // Use 'not equal' since we are just doing an increment/decrement. |
| 287 | auto cmp = builder.createCompare( |
| 288 | loc, kind: inverse ? cir::CmpOpKind::ge : cir::CmpOpKind::lt, lhs: loadCur, |
| 289 | rhs: endItr); |
| 290 | builder.createCondition(condition: cmp); |
| 291 | }, |
| 292 | /*bodyBuilder=*/ |
| 293 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 294 | auto load = cir::LoadOp::create(builder, location: loc, addr: itr.getResult()); |
| 295 | |
| 296 | if (subscriptedValue) |
| 297 | subscriptedValue = doSubscriptOp(subscriptedValue, load); |
| 298 | if (subscriptedValue2) |
| 299 | subscriptedValue2 = doSubscriptOp(subscriptedValue2, load); |
| 300 | bodyInsertLoc = builder.createYield(loc); |
| 301 | }, |
| 302 | /*stepBuilder=*/ |
| 303 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 304 | auto load = cir::LoadOp::create(builder, location: loc, addr: itr.getResult()); |
| 305 | auto unary = inverse ? builder.createDec(loc, input: load) |
| 306 | : builder.createInc(loc, input: load); |
| 307 | builder.CIRBaseBuilderTy::createStore(loc, val: unary, dst: itr); |
| 308 | builder.createYield(loc); |
| 309 | }); |
| 310 | }; |
| 311 | |
| 312 | cir::ScopeOp::create(builder, location: loc, |
| 313 | scopeBuilder: [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 314 | forStmtBuilder(); |
| 315 | builder.createYield(loc); |
| 316 | }); |
| 317 | |
| 318 | // Leave the insertion point to be inside the body, so we can loop over |
| 319 | // these things. |
| 320 | builder.setInsertionPoint(bodyInsertLoc); |
| 321 | return {subscriptedValue, subscriptedValue2}; |
| 322 | } |
| 323 | |
| 324 | mlir::acc::ReductionOperator |
| 325 | OpenACCRecipeBuilderBase::convertReductionOp(OpenACCReductionOperator op) { |
| 326 | switch (op) { |
| 327 | case OpenACCReductionOperator::Addition: |
| 328 | return mlir::acc::ReductionOperator::AccAdd; |
| 329 | case OpenACCReductionOperator::Multiplication: |
| 330 | return mlir::acc::ReductionOperator::AccMul; |
| 331 | case OpenACCReductionOperator::Max: |
| 332 | return mlir::acc::ReductionOperator::AccMax; |
| 333 | case OpenACCReductionOperator::Min: |
| 334 | return mlir::acc::ReductionOperator::AccMin; |
| 335 | case OpenACCReductionOperator::BitwiseAnd: |
| 336 | return mlir::acc::ReductionOperator::AccIand; |
| 337 | case OpenACCReductionOperator::BitwiseOr: |
| 338 | return mlir::acc::ReductionOperator::AccIor; |
| 339 | case OpenACCReductionOperator::BitwiseXOr: |
| 340 | return mlir::acc::ReductionOperator::AccXor; |
| 341 | case OpenACCReductionOperator::And: |
| 342 | return mlir::acc::ReductionOperator::AccLand; |
| 343 | case OpenACCReductionOperator::Or: |
| 344 | return mlir::acc::ReductionOperator::AccLor; |
| 345 | case OpenACCReductionOperator::Invalid: |
| 346 | llvm_unreachable("invalid reduction operator" ); |
| 347 | } |
| 348 | |
| 349 | llvm_unreachable("invalid reduction operator" ); |
| 350 | } |
| 351 | |
| 352 | // This function generates the 'destroy' section for a recipe. Note |
| 353 | // that this function is not 'insertion point' clean, in that it alters the |
| 354 | // insertion point to be inside of the 'destroy' section of the recipe, but |
| 355 | // doesn't restore it aftewards. |
| 356 | void OpenACCRecipeBuilderBase::createRecipeDestroySection( |
| 357 | mlir::Location loc, mlir::Location locEnd, mlir::Value mainOp, |
| 358 | CharUnits alignment, QualType origType, size_t numBounds, QualType baseType, |
| 359 | mlir::Region &destroyRegion) { |
| 360 | mlir::Block *block = createRecipeBlock(region&: destroyRegion, opTy: mainOp.getType(), loc, |
| 361 | numBounds, /*isInit=*/false); |
| 362 | builder.setInsertionPointToEnd(&destroyRegion.back()); |
| 363 | CIRGenFunction::LexicalScope ls(cgf, loc, block); |
| 364 | |
| 365 | mlir::Type elementTy = |
| 366 | mlir::cast<cir::PointerType>(Val: mainOp.getType()).getPointee(); |
| 367 | auto emitDestroy = [&](mlir::Value var, mlir::Type ty) { |
| 368 | Address addr{var, ty, alignment}; |
| 369 | cgf.emitDestroy(addr, type: origType, |
| 370 | destroyer: cgf.getDestroyer(kind: QualType::DK_cxx_destructor)); |
| 371 | }; |
| 372 | |
| 373 | if (numBounds) { |
| 374 | mlir::OpBuilder::InsertionGuard guardCase(builder); |
| 375 | // Get the range of bounds arguments, which are all but the 1st 2. 1st is |
| 376 | // a 'reference', 2nd is the 'private' variant we need to destroy from. |
| 377 | llvm::MutableArrayRef<mlir::BlockArgument> boundsRange = |
| 378 | block->getArguments().drop_front(N: 2); |
| 379 | |
| 380 | mlir::Value subscriptedValue = block->getArgument(i: 1); |
| 381 | for (mlir::BlockArgument boundArg : llvm::reverse(C&: boundsRange)) |
| 382 | subscriptedValue = createBoundsLoop(subscriptedValue, bound: boundArg, loc, |
| 383 | /*inverse=*/true); |
| 384 | |
| 385 | emitDestroy(subscriptedValue, cgf.cgm.convertType(type: origType)); |
| 386 | } else { |
| 387 | // If we don't have any bounds, we can just destroy the variable directly. |
| 388 | // The destroy region has a signature of "original item, privatized item". |
| 389 | // So the 2nd item is the one that needs destroying, the former is just |
| 390 | // for reference and we don't really have a need for it at the moment. |
| 391 | emitDestroy(block->getArgument(i: 1), elementTy); |
| 392 | } |
| 393 | |
| 394 | ls.forceCleanup(); |
| 395 | mlir::acc::YieldOp::create(builder, location: locEnd); |
| 396 | } |
| 397 | void OpenACCRecipeBuilderBase::makeBoundsInit( |
| 398 | mlir::Value alloca, mlir::Location loc, mlir::Block *block, |
| 399 | const VarDecl *allocaDecl, QualType origType, bool isInitSection) { |
| 400 | mlir::OpBuilder::InsertionGuard guardCase(builder); |
| 401 | builder.setInsertionPointToEnd(block); |
| 402 | CIRGenFunction::LexicalScope ls(cgf, loc, block); |
| 403 | |
| 404 | CIRGenFunction::AutoVarEmission tempDeclEmission{*allocaDecl}; |
| 405 | tempDeclEmission.emittedAsOffload = true; |
| 406 | |
| 407 | // The init section is the only one of the handful that only has a single |
| 408 | // argument for the 'type', so we have to drop 1 for init, and future calls |
| 409 | // to this will need to drop 2. |
| 410 | llvm::MutableArrayRef<mlir::BlockArgument> boundsRange = |
| 411 | block->getArguments().drop_front(N: isInitSection ? 1 : 2); |
| 412 | |
| 413 | mlir::Value subscriptedValue = alloca; |
| 414 | for (mlir::BlockArgument boundArg : llvm::reverse(C&: boundsRange)) |
| 415 | subscriptedValue = createBoundsLoop(subscriptedValue, bound: boundArg, loc, |
| 416 | /*inverse=*/false); |
| 417 | |
| 418 | tempDeclEmission.setAllocatedAddress( |
| 419 | Address{subscriptedValue, cgf.convertType(t: origType), |
| 420 | cgf.getContext().getDeclAlign(D: allocaDecl)}); |
| 421 | cgf.emitAutoVarInit(emission: tempDeclEmission); |
| 422 | } |
| 423 | |
| 424 | // TODO: OpenACC: when we start doing firstprivate for array/vlas/etc, we |
| 425 | // probably need to do a little work about the 'init' calls to put it in 'copy' |
| 426 | // region instead. |
| 427 | void OpenACCRecipeBuilderBase::createInitRecipe( |
| 428 | mlir::Location loc, mlir::Location locEnd, SourceRange exprRange, |
| 429 | mlir::Value mainOp, mlir::Region &recipeInitRegion, size_t numBounds, |
| 430 | llvm::ArrayRef<QualType> boundTypes, const VarDecl *allocaDecl, |
| 431 | QualType origType, bool emitInitExpr) { |
| 432 | assert(allocaDecl && "Required recipe variable not set?" ); |
| 433 | CIRGenFunction::DeclMapRevertingRAII declMapRAII{cgf, allocaDecl}; |
| 434 | |
| 435 | mlir::Block *block = createRecipeBlock(region&: recipeInitRegion, opTy: mainOp.getType(), |
| 436 | loc, numBounds, /*isInit=*/true); |
| 437 | builder.setInsertionPointToEnd(&recipeInitRegion.back()); |
| 438 | CIRGenFunction::LexicalScope ls(cgf, loc, block); |
| 439 | |
| 440 | const Type *allocaPointeeType = |
| 441 | allocaDecl->getType()->getPointeeOrArrayElementType(); |
| 442 | // We are OK with no init for builtins, arrays of builtins, or pointers, |
| 443 | // else we should NYI so we know to go look for these. |
| 444 | if (cgf.getContext().getLangOpts().CPlusPlus && !allocaDecl->getInit() && |
| 445 | !allocaDecl->getType()->isPointerType() && |
| 446 | !allocaPointeeType->isBuiltinType() && |
| 447 | !allocaPointeeType->isPointerType()) { |
| 448 | // If we don't have any initialization recipe, we failed during Sema to |
| 449 | // initialize this correctly. If we disable the |
| 450 | // Sema::TentativeAnalysisScopes in SemaOpenACC::CreateInitRecipe, it'll |
| 451 | // emit an error to tell us. However, emitting those errors during |
| 452 | // production is a violation of the standard, so we cannot do them. |
| 453 | cgf.cgm.errorNYI(exprRange, "private/reduction default-init recipe" ); |
| 454 | } |
| 455 | |
| 456 | if (!numBounds) { |
| 457 | // This is an 'easy' case, we just have to use the builtin init stuff to |
| 458 | // initialize this variable correctly. |
| 459 | CIRGenFunction::AutoVarEmission tempDeclEmission = |
| 460 | cgf.emitAutoVarAlloca(d: *allocaDecl, ip: builder.saveInsertionPoint()); |
| 461 | if (emitInitExpr) |
| 462 | cgf.emitAutoVarInit(emission: tempDeclEmission); |
| 463 | } else { |
| 464 | mlir::Value alloca = makeBoundsAlloca( |
| 465 | block, exprRange, loc, allocaName: allocaDecl->getName(), numBounds, boundTypes); |
| 466 | |
| 467 | // If the initializer is trivial, there is nothing to do here, so save |
| 468 | // ourselves some effort. |
| 469 | if (emitInitExpr && allocaDecl->getInit() && |
| 470 | (!cgf.isTrivialInitializer(init: allocaDecl->getInit()) || |
| 471 | cgf.getContext().getLangOpts().getTrivialAutoVarInit() != |
| 472 | LangOptions::TrivialAutoVarInitKind::Uninitialized)) |
| 473 | makeBoundsInit(alloca, loc, block, allocaDecl, origType, |
| 474 | /*isInitSection=*/true); |
| 475 | } |
| 476 | |
| 477 | ls.forceCleanup(); |
| 478 | mlir::acc::YieldOp::create(builder, location: locEnd); |
| 479 | } |
| 480 | |
| 481 | void OpenACCRecipeBuilderBase::createFirstprivateRecipeCopy( |
| 482 | mlir::Location loc, mlir::Location locEnd, mlir::Value mainOp, |
| 483 | const VarDecl *allocaDecl, const VarDecl *temporary, |
| 484 | mlir::Region ©Region, size_t numBounds) { |
| 485 | mlir::Block *block = createRecipeBlock(region&: copyRegion, opTy: mainOp.getType(), loc, |
| 486 | numBounds, /*isInit=*/false); |
| 487 | builder.setInsertionPointToEnd(©Region.back()); |
| 488 | CIRGenFunction::LexicalScope ls(cgf, loc, block); |
| 489 | |
| 490 | mlir::Value fromArg = block->getArgument(i: 0); |
| 491 | mlir::Value toArg = block->getArgument(i: 1); |
| 492 | |
| 493 | llvm::MutableArrayRef<mlir::BlockArgument> boundsRange = |
| 494 | block->getArguments().drop_front(N: 2); |
| 495 | |
| 496 | for (mlir::BlockArgument boundArg : llvm::reverse(C&: boundsRange)) |
| 497 | std::tie(args&: fromArg, args&: toArg) = |
| 498 | createBoundsLoop(subscriptedValue: fromArg, subscriptedValue2: toArg, bound: boundArg, loc, /*inverse=*/false); |
| 499 | |
| 500 | // Set up the 'to' address. |
| 501 | mlir::Type elementTy = |
| 502 | mlir::cast<cir::PointerType>(Val: toArg.getType()).getPointee(); |
| 503 | CIRGenFunction::AutoVarEmission tempDeclEmission(*allocaDecl); |
| 504 | tempDeclEmission.emittedAsOffload = true; |
| 505 | tempDeclEmission.setAllocatedAddress( |
| 506 | Address{toArg, elementTy, cgf.getContext().getDeclAlign(D: allocaDecl)}); |
| 507 | |
| 508 | // Set up the 'from' address from the temporary. |
| 509 | CIRGenFunction::DeclMapRevertingRAII declMapRAII{cgf, temporary}; |
| 510 | cgf.setAddrOfLocalVar( |
| 511 | vd: temporary, |
| 512 | addr: Address{fromArg, elementTy, cgf.getContext().getDeclAlign(D: allocaDecl)}); |
| 513 | cgf.emitAutoVarInit(emission: tempDeclEmission); |
| 514 | |
| 515 | builder.setInsertionPointToEnd(©Region.back()); |
| 516 | ls.forceCleanup(); |
| 517 | mlir::acc::YieldOp::create(builder, location: locEnd); |
| 518 | } |
| 519 | |
| 520 | // This function generates the 'combiner' section for a reduction recipe. Note |
| 521 | // that this function is not 'insertion point' clean, in that it alters the |
| 522 | // insertion point to be inside of the 'combiner' section of the recipe, but |
| 523 | // doesn't restore it aftewards. |
| 524 | void OpenACCRecipeBuilderBase::createReductionRecipeCombiner( |
| 525 | mlir::Location loc, mlir::Location locEnd, mlir::Value mainOp, |
| 526 | mlir::acc::ReductionRecipeOp recipe, size_t numBounds, QualType origType, |
| 527 | llvm::ArrayRef<OpenACCReductionRecipe::CombinerRecipe> combinerRecipes) { |
| 528 | mlir::Block *block = |
| 529 | createRecipeBlock(region&: recipe.getCombinerRegion(), opTy: mainOp.getType(), loc, |
| 530 | numBounds, /*isInit=*/false); |
| 531 | builder.setInsertionPointToEnd(&recipe.getCombinerRegion().back()); |
| 532 | CIRGenFunction::LexicalScope ls(cgf, loc, block); |
| 533 | |
| 534 | mlir::Value lhsArg = block->getArgument(i: 0); |
| 535 | mlir::Value rhsArg = block->getArgument(i: 1); |
| 536 | llvm::MutableArrayRef<mlir::BlockArgument> boundsRange = |
| 537 | block->getArguments().drop_front(N: 2); |
| 538 | |
| 539 | if (llvm::any_of(Range&: combinerRecipes, P: [](auto &r) { return r.Op == nullptr; })) { |
| 540 | cgf.cgm.errorNYI(loc, feature: "OpenACC Reduction combiner not generated" ); |
| 541 | mlir::acc::YieldOp::create(builder, location: locEnd, operands: block->getArgument(i: 0)); |
| 542 | return; |
| 543 | } |
| 544 | |
| 545 | // apply the bounds so that we can get our bounds emitted correctly. |
| 546 | for (mlir::BlockArgument boundArg : llvm::reverse(C&: boundsRange)) |
| 547 | std::tie(args&: lhsArg, args&: rhsArg) = |
| 548 | createBoundsLoop(subscriptedValue: lhsArg, subscriptedValue2: rhsArg, bound: boundArg, loc, /*inverse=*/false); |
| 549 | |
| 550 | // Emitter for when we know this isn't a struct or array we have to loop |
| 551 | // through. This should work for the 'field' once the get-element call has |
| 552 | // been made. |
| 553 | auto emitSingleCombiner = |
| 554 | [&](mlir::Value lhsArg, mlir::Value rhsArg, |
| 555 | const OpenACCReductionRecipe::CombinerRecipe &combiner) { |
| 556 | mlir::Type elementTy = |
| 557 | mlir::cast<cir::PointerType>(Val: lhsArg.getType()).getPointee(); |
| 558 | CIRGenFunction::DeclMapRevertingRAII declMapRAIILhs{cgf, combiner.LHS}; |
| 559 | cgf.setAddrOfLocalVar( |
| 560 | vd: combiner.LHS, addr: Address{lhsArg, elementTy, |
| 561 | cgf.getContext().getDeclAlign(D: combiner.LHS)}); |
| 562 | CIRGenFunction::DeclMapRevertingRAII declMapRAIIRhs{cgf, combiner.RHS}; |
| 563 | cgf.setAddrOfLocalVar( |
| 564 | vd: combiner.RHS, addr: Address{rhsArg, elementTy, |
| 565 | cgf.getContext().getDeclAlign(D: combiner.RHS)}); |
| 566 | |
| 567 | [[maybe_unused]] mlir::LogicalResult stmtRes = |
| 568 | cgf.emitStmt(s: combiner.Op, /*useCurrentScope=*/true); |
| 569 | }; |
| 570 | |
| 571 | // Emitter for when we know this is either a non-array or element of an array |
| 572 | // (which also shouldn't be an array type?). This function should generate the |
| 573 | // initialization code for an entire 'array-element'/non-array, including |
| 574 | // diving into each element of a struct (if necessary). |
| 575 | auto emitCombiner = [&](mlir::Value lhsArg, mlir::Value rhsArg, QualType ty) { |
| 576 | assert(!ty->isArrayType() && "Array type shouldn't get here" ); |
| 577 | if (const auto *rd = ty->getAsRecordDecl()) { |
| 578 | if (combinerRecipes.size() == 1 && |
| 579 | cgf.getContext().hasSameType(T1: ty, T2: combinerRecipes[0].LHS->getType())) { |
| 580 | // If this is a 'top level' operator on the type we can just emit this |
| 581 | // as a simple one. |
| 582 | emitSingleCombiner(lhsArg, rhsArg, combinerRecipes[0]); |
| 583 | } else { |
| 584 | // else we have to handle each individual field after after a |
| 585 | // get-element. |
| 586 | const CIRGenRecordLayout &layout = |
| 587 | cgf.cgm.getTypes().getCIRGenRecordLayout(rd); |
| 588 | for (const auto &[field, combiner] : |
| 589 | llvm::zip_equal(t: rd->fields(), u&: combinerRecipes)) { |
| 590 | mlir::Type fieldType = cgf.convertType(t: field->getType()); |
| 591 | auto fieldPtr = cir::PointerType::get(pointee: fieldType); |
| 592 | unsigned fieldIndex = layout.getCIRFieldNo(fd: field); |
| 593 | |
| 594 | mlir::Value lhsField = builder.createGetMember( |
| 595 | loc, resultTy: fieldPtr, base: lhsArg, name: field->getName(), index: fieldIndex); |
| 596 | mlir::Value rhsField = builder.createGetMember( |
| 597 | loc, resultTy: fieldPtr, base: rhsArg, name: field->getName(), index: fieldIndex); |
| 598 | |
| 599 | emitSingleCombiner(lhsField, rhsField, combiner); |
| 600 | } |
| 601 | } |
| 602 | |
| 603 | } else { |
| 604 | // if this is a single-thing (because we should know this isn't an array, |
| 605 | // as Sema wouldn't let us get here), we can just do a normal emit call. |
| 606 | emitSingleCombiner(lhsArg, rhsArg, combinerRecipes[0]); |
| 607 | } |
| 608 | }; |
| 609 | |
| 610 | if (const auto *cat = cgf.getContext().getAsConstantArrayType(T: origType)) { |
| 611 | // If we're in an array, we have to emit the combiner for each element of |
| 612 | // the array. |
| 613 | auto itrTy = mlir::cast<cir::IntType>(Val&: cgf.ptrDiffTy); |
| 614 | auto itrPtrTy = cir::PointerType::get(pointee: itrTy); |
| 615 | |
| 616 | mlir::Value zero = |
| 617 | builder.getConstInt(loc, t: mlir::cast<cir::IntType>(Val&: cgf.ptrDiffTy), c: 0); |
| 618 | mlir::Value itr = cir::AllocaOp::create( |
| 619 | builder, location: loc, addr: itrPtrTy, name: "itr" , alignment: cgf.cgm.getSize(size: cgf.getPointerAlign())); |
| 620 | builder.CIRBaseBuilderTy::createStore(loc, val: zero, dst: itr); |
| 621 | |
| 622 | builder.setInsertionPointAfter(builder.createFor( |
| 623 | loc, |
| 624 | /*condBuilder=*/ |
| 625 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 626 | auto loadItr = cir::LoadOp::create(builder, location: loc, addr: itr); |
| 627 | mlir::Value arraySize = builder.getConstInt( |
| 628 | loc, t: mlir::cast<cir::IntType>(Val&: cgf.ptrDiffTy), c: cat->getZExtSize()); |
| 629 | auto cmp = builder.createCompare(loc, kind: cir::CmpOpKind::lt, lhs: loadItr, |
| 630 | rhs: arraySize); |
| 631 | builder.createCondition(condition: cmp); |
| 632 | }, |
| 633 | /*bodyBuilder=*/ |
| 634 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 635 | auto loadItr = cir::LoadOp::create(builder, location: loc, addr: itr); |
| 636 | auto lhsElt = builder.getArrayElement( |
| 637 | arrayLocBegin: loc, arrayLocEnd: loc, arrayPtr: lhsArg, eltTy: cgf.convertType(t: cat->getElementType()), idx: loadItr, |
| 638 | /*shouldDecay=*/true); |
| 639 | auto rhsElt = builder.getArrayElement( |
| 640 | arrayLocBegin: loc, arrayLocEnd: loc, arrayPtr: rhsArg, eltTy: cgf.convertType(t: cat->getElementType()), idx: loadItr, |
| 641 | /*shouldDecay=*/true); |
| 642 | |
| 643 | emitCombiner(lhsElt, rhsElt, cat->getElementType()); |
| 644 | builder.createYield(loc); |
| 645 | }, |
| 646 | /*stepBuilder=*/ |
| 647 | [&](mlir::OpBuilder &b, mlir::Location loc) { |
| 648 | auto loadItr = cir::LoadOp::create(builder, location: loc, addr: itr); |
| 649 | auto inc = builder.createInc(loc, input: loadItr); |
| 650 | builder.CIRBaseBuilderTy::createStore(loc, val: inc, dst: itr); |
| 651 | builder.createYield(loc); |
| 652 | })); |
| 653 | |
| 654 | } else if (origType->isArrayType()) { |
| 655 | cgf.cgm.errorNYI(loc, |
| 656 | feature: "OpenACC Reduction combiner non-constant array recipe" ); |
| 657 | } else { |
| 658 | emitCombiner(lhsArg, rhsArg, origType); |
| 659 | } |
| 660 | |
| 661 | builder.setInsertionPointToEnd(&recipe.getCombinerRegion().back()); |
| 662 | ls.forceCleanup(); |
| 663 | mlir::acc::YieldOp::create(builder, location: locEnd, operands: block->getArgument(i: 0)); |
| 664 | } |
| 665 | |
| 666 | } // namespace clang::CIRGen |
| 667 | |