| 1 | //===- ACCRecipeMaterialization.cpp - Materialize ACC recipes -------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Overview: |
| 10 | // --------- |
| 11 | // OpenACC compute constructs (acc.parallel, acc.serial, acc.kernels) and |
| 12 | // acc.loop can carry data clauses (acc.private, acc.firstprivate, |
| 13 | // acc.reduction) that refer to recipes (acc.private.recipe, |
| 14 | // acc.firstprivate.recipe, acc.reduction.recipe). Recipes define how to |
| 15 | // initialize, copy, combine, or destroy a particular variable. This pass clones |
| 16 | // those regions into the construct and ensures the materialized SSA values are |
| 17 | // used instead. |
| 18 | // |
| 19 | // Transforms: |
| 20 | // ----------- |
| 21 | // 1. Firstprivate: Inserts acc.firstprivate_map so the initial value is |
| 22 | // available on the device, then clones the recipe init and copy regions |
| 23 | // into the construct and replaces uses with the materialized alloca. |
| 24 | // Optional destroy region is cloned before the region terminator. |
| 25 | // |
| 26 | // 2. Private: Clones the recipe init region into the construct (at the |
| 27 | // region entry or at the loop op for acc.loop private). Replaces uses |
| 28 | // of the recipe result with the materialized alloca. Optional destroy |
| 29 | // region is cloned before the region terminator. |
| 30 | // |
| 31 | // 3. Reduction: Creates acc.reduction_init (init region inlined) and |
| 32 | // acc.reduction_combine_region (combiner region inlined). Uses within |
| 33 | // the region are updated to the reduction init result. |
| 34 | // In addition, creates appropriate acc.copyin/copyout around |
| 35 | // the compute region; the reduction's initial value is taken |
| 36 | // from the copied in variable, and the final reduction value |
| 37 | // is copied out to the variable. If there are existing |
| 38 | // data operations for the reduction variable, no new data |
| 39 | // operations are added. |
| 40 | // |
| 41 | // Requirements: |
| 42 | // ------------- |
| 43 | // 1. OpenACCSupport: The pass uses the `acc::OpenACCSupport` analysis |
| 44 | // including emitNYI for unsupported cases. |
| 45 | // |
| 46 | //===----------------------------------------------------------------------===// |
| 47 | |
| 48 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 49 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 50 | #include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" |
| 51 | #include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h" |
| 52 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 53 | #include "mlir/Dialect/OpenACC/OpenACCParMapping.h" |
| 54 | #include "mlir/Dialect/OpenACC/OpenACCUtils.h" |
| 55 | #include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h" |
| 56 | #include "mlir/Dialect/OpenACC/OpenACCUtilsLoop.h" |
| 57 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h" |
| 58 | #include "mlir/IR/Block.h" |
| 59 | #include "mlir/IR/Builders.h" |
| 60 | #include "mlir/IR/IRMapping.h" |
| 61 | #include "mlir/IR/SymbolTable.h" |
| 62 | #include "mlir/IR/Value.h" |
| 63 | #include "mlir/IR/ValueRange.h" |
| 64 | #include "mlir/Interfaces/LoopLikeInterface.h" |
| 65 | #include "mlir/Support/LLVM.h" |
| 66 | #include "mlir/Transforms/RegionUtils.h" |
| 67 | #include "llvm/ADT/DenseSet.h" |
| 68 | #include "llvm/ADT/STLExtras.h" |
| 69 | #include "llvm/ADT/TypeSwitch.h" |
| 70 | #include "llvm/Support/Debug.h" |
| 71 | #include "llvm/Support/ErrorHandling.h" |
| 72 | |
| 73 | namespace mlir { |
| 74 | namespace acc { |
| 75 | #define GEN_PASS_DEF_ACCRECIPEMATERIALIZATION |
| 76 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" |
| 77 | } // namespace acc |
| 78 | } // namespace mlir |
| 79 | |
| 80 | #define DEBUG_TYPE "acc-recipe-materialization" |
| 81 | |
| 82 | namespace { |
| 83 | using namespace mlir; |
| 84 | |
| 85 | static void setLocation(Region ®ion, Location loc) { |
| 86 | // Since recipes are generated per type and not per variable, the location |
| 87 | // of the recipe operations which get inlined will not necessarily be the |
| 88 | // same as the location of the op that is being materialized. Force an update |
| 89 | // of the location of the recipe operations to the location of the op that is |
| 90 | // being materialized. |
| 91 | region.walk(callback: [&](Operation *op) { op->setLoc(loc); }); |
| 92 | } |
| 93 | |
| 94 | // Complex multiplication mixes the real and imaginary parts, so a reduction |
| 95 | // of that operator cannot be lowered as two independent component reductions. |
| 96 | // Keep unwrapping pointer-like types so a reference to a reference still |
| 97 | // reports the element type. |
| 98 | static bool isComplexReductionType(Type type) { |
| 99 | while (auto ptrTy = dyn_cast<acc::PointerLikeType>(Val&: type)) { |
| 100 | Type elementType = ptrTy.getElementType(); |
| 101 | if (!elementType || elementType == type) |
| 102 | return false; |
| 103 | type = elementType; |
| 104 | } |
| 105 | return isa<ComplexType>(Val: type); |
| 106 | } |
| 107 | |
| 108 | static void saveVarName(StringRef name, Value dst) { |
| 109 | if (name.empty()) |
| 110 | return; |
| 111 | if (Operation *dstOp = dst.getDefiningOp()) { |
| 112 | if (dstOp->getDiscardableAttrOfType<acc::VarNameAttr>( |
| 113 | name: acc::getVarNameAttrName())) |
| 114 | return; |
| 115 | if (isa<ACC_DATA_ENTRY_OPS>(Val: dstOp)) |
| 116 | return; |
| 117 | dstOp->setDiscardableAttr(name: acc::getVarNameAttrName(), |
| 118 | value: acc::VarNameAttr::get(context: dstOp->getContext(), name)); |
| 119 | return; |
| 120 | } |
| 121 | auto blockArg = dyn_cast<BlockArgument>(Val&: dst); |
| 122 | if (!blockArg) |
| 123 | return; |
| 124 | Block *block = blockArg.getOwner(); |
| 125 | Region *region = block ? block->getParent() : nullptr; |
| 126 | if (!region || !block->isEntryBlock()) |
| 127 | return; |
| 128 | Operation *parent = region->getParentOp(); |
| 129 | if (!parent) |
| 130 | return; |
| 131 | auto funcOp = dyn_cast<FunctionOpInterface>(Val: parent); |
| 132 | if (!funcOp) |
| 133 | return; |
| 134 | unsigned argIdx = blockArg.getArgNumber(); |
| 135 | if (argIdx >= funcOp.getNumArguments()) |
| 136 | return; |
| 137 | if (funcOp.getArgAttr(index: argIdx, name: acc::getVarNameAttrName())) |
| 138 | return; |
| 139 | funcOp.setArgAttr(index: argIdx, name: acc::getVarNameAttrName(), |
| 140 | value: acc::VarNameAttr::get(context: parent->getContext(), name)); |
| 141 | } |
| 142 | |
| 143 | static void saveVarName(Value src, Value dst) { |
| 144 | saveVarName(name: acc::getVariableName(v: src), dst); |
| 145 | } |
| 146 | |
| 147 | static void resolveVarNamePlaceholders(Block *block, Block::iterator ip, |
| 148 | StringRef name) { |
| 149 | StringRef placeholder = acc::getVarNamePlaceholder(); |
| 150 | for (auto it = block->begin(); it != std::next(x: ip); ++it) { |
| 151 | it->walk(callback: [&](Operation *op) { |
| 152 | auto attr = op->getDiscardableAttrOfType<acc::VarNameAttr>( |
| 153 | name: acc::getVarNameAttrName()); |
| 154 | if (!attr || attr.getName() != placeholder) |
| 155 | return; |
| 156 | if (name.empty()) |
| 157 | op->removeDiscardableAttr(name: acc::getVarNameAttrName()); |
| 158 | else |
| 159 | op->setDiscardableAttr(name: acc::getVarNameAttrName(), |
| 160 | value: acc::VarNameAttr::get(context: op->getContext(), name)); |
| 161 | }); |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | // Clone the destroy region of the recipe before the terminator of the provided |
| 166 | // block. Values must be provided for the destroy region block arguments |
| 167 | // according to the recipe specifications. |
| 168 | template <typename RecipeOpTy> |
| 169 | static void cloneDestroy(Location loc, RecipeOpTy recipe, mlir::Block *block, |
| 170 | Block::iterator ip, |
| 171 | const llvm::SmallVector<mlir::Value> &arguments) { |
| 172 | IRMapping mapping{}; |
| 173 | Region &destroyRegion = recipe.getDestroyRegion(); |
| 174 | assert(destroyRegion.getBlocks().front().getNumArguments() == |
| 175 | arguments.size() && |
| 176 | "unexpected acc recipe destroy block arguments" ); |
| 177 | |
| 178 | setLocation(region&: destroyRegion, loc); |
| 179 | |
| 180 | mapping.map(from: destroyRegion.getBlocks().front().getArguments(), to: arguments); |
| 181 | acc::cloneACCRegionInto(src: &destroyRegion, dest: block, inlinePoint: ip, mapping, |
| 182 | /*resultsToReplace=*/{}); |
| 183 | } |
| 184 | |
| 185 | class ACCRecipeMaterialization |
| 186 | : public acc::impl::ACCRecipeMaterializationBase<ACCRecipeMaterialization> { |
| 187 | public: |
| 188 | using acc::impl::ACCRecipeMaterializationBase< |
| 189 | ACCRecipeMaterialization>::ACCRecipeMaterializationBase; |
| 190 | |
| 191 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 192 | acc::impl::ACCRecipeMaterializationBase< |
| 193 | ACCRecipeMaterialization>::getDependentDialects(registry); |
| 194 | acc::registerAllExtensions(registry); |
| 195 | } |
| 196 | void runOnOperation() override; |
| 197 | |
| 198 | private: |
| 199 | // When the recipe reads the original variable, its initial value needs to be |
| 200 | // available on the GPU. One way to get that value there is to map the |
| 201 | // variable through global memory. |
| 202 | // Thus, when we materialize a firstprivate, we materialize it into |
| 203 | // a mapping action first. This function ends up with doing the following: |
| 204 | // %dev = acc.firstprivate var(%var) |
| 205 | // => |
| 206 | // %copy = acc.firstprivate_map var(%var) |
| 207 | // %dev = acc.firstprivate var(%copy) |
| 208 | // When the recipe materialization happens, the `acc.firstprivate` ends up |
| 209 | // being removed. But because of the way we chain it to the |
| 210 | // `acc.firstprivate_map`, then its result becomes live-in to the |
| 211 | // compute region and used as the variable the initial value is loaded from. |
| 212 | template <typename OpTy> |
| 213 | void handleInitialValueMapping(OpTy op) const; |
| 214 | template <typename OpTy> |
| 215 | void removeRecipe( |
| 216 | OpTy op, ModuleOp moduleOp, |
| 217 | const std::optional<llvm::DenseSet<StringAttr>> &usedSymbols) const; |
| 218 | template <typename OpTy, typename RecipeOpTy, typename AccOpTy> |
| 219 | LogicalResult materialize(OpTy op, RecipeOpTy recipe, AccOpTy accOp, |
| 220 | acc::OpenACCSupport &accSupport, |
| 221 | acc::ACCToGPUMappingPolicy &policy, |
| 222 | Value materializationVar = {}) const; |
| 223 | template <typename OpTy> |
| 224 | LogicalResult materializeForACCOp(OpTy accOp, acc::OpenACCSupport &accSupport, |
| 225 | acc::ACCToGPUMappingPolicy &policy) const; |
| 226 | }; |
| 227 | |
| 228 | template <typename OpTy> |
| 229 | void ACCRecipeMaterialization::handleInitialValueMapping(OpTy op) const { |
| 230 | OpBuilder builder(op); |
| 231 | auto mapInitialOp = acc::FirstprivateMapInitialOp::create( |
| 232 | builder, op.getLoc(), op.getVar(), op.getStructured(), op.getImplicit(), |
| 233 | op.getBounds()); |
| 234 | mapInitialOp.setName(op.getName()); |
| 235 | op.getVarMutable().assign(mapInitialOp.getAccVar()); |
| 236 | } |
| 237 | |
| 238 | // Whether a recipe region reads the variable it privatizes - a descriptor |
| 239 | // recipe loads it for the bounds, while a scalar one ignores it. Both init and |
| 240 | // destroy receive it as their first argument. |
| 241 | static bool readsVar(Region ®ion) { |
| 242 | if (region.empty() || region.getNumArguments() == 0) |
| 243 | return false; |
| 244 | return !region.getArgument(i: 0).use_empty(); |
| 245 | } |
| 246 | |
| 247 | template <typename OpTy> |
| 248 | void ACCRecipeMaterialization::removeRecipe( |
| 249 | OpTy op, ModuleOp moduleOp, |
| 250 | const std::optional<llvm::DenseSet<StringAttr>> &usedSymbols) const { |
| 251 | auto recipeName = op.getNameAttr(); |
| 252 | // Fall back to scanning the module when the symbol uses could not be |
| 253 | // gathered up front. |
| 254 | bool useEmpty = usedSymbols |
| 255 | ? !usedSymbols->contains(V: recipeName) |
| 256 | : SymbolTable::symbolKnownUseEmpty(recipeName, moduleOp); |
| 257 | if (useEmpty) { |
| 258 | LLVM_DEBUG(llvm::dbgs() << "erasing recipe: " << recipeName << "\n" ); |
| 259 | op.erase(); |
| 260 | } else { |
| 261 | LLVM_DEBUG({ |
| 262 | std::optional<SymbolTable::UseRange> symbolUses = |
| 263 | op.getSymbolUses(moduleOp); |
| 264 | if (symbolUses.has_value()) { |
| 265 | for (SymbolTable::SymbolUse symbolUse : *symbolUses) { |
| 266 | llvm::dbgs() << "symbol use: " ; |
| 267 | symbolUse.getUser()->dump(); |
| 268 | } |
| 269 | } |
| 270 | }); |
| 271 | llvm_unreachable("expected no use of recipe symbol" ); |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | template <typename OpTy, typename RecipeOpTy, typename AccOpTy> |
| 276 | LogicalResult ACCRecipeMaterialization::materialize( |
| 277 | OpTy op, RecipeOpTy recipe, AccOpTy accOp, acc::OpenACCSupport &accSupport, |
| 278 | acc::ACCToGPUMappingPolicy &policy, Value materializationVar) const { |
| 279 | Region ®ion = accOp.getRegion(); |
| 280 | Value origPtr = materializationVar ? materializationVar : op.getVar(); |
| 281 | Value accPtr = op.getAccVar(); |
| 282 | assert(accPtr && "invalid op: null acc var" ); |
| 283 | |
| 284 | OpBuilder b(op); |
| 285 | SmallVector<Value> triples; |
| 286 | |
| 287 | // Clone init block into the region at the insertion point specified. |
| 288 | Region &initRegion = recipe.getInitRegion(); |
| 289 | unsigned initNumArguments = |
| 290 | initRegion.getBlocks().front().getArguments().size(); |
| 291 | if (initNumArguments > 1) { |
| 292 | // Code from C/C++ will most likely only provide extent arguments to the |
| 293 | // recipe arguments. |
| 294 | if ((initNumArguments - 1) % 3 != 0) { |
| 295 | (void)accSupport.emitNYI(loc: recipe.getLoc(), |
| 296 | message: "privatization of array section with extents" ); |
| 297 | return failure(); |
| 298 | } |
| 299 | // The remaining arguments must be the bounds triples |
| 300 | // (lower-bound, upper-bound, step), ... |
| 301 | unsigned argIdx = 1; |
| 302 | // Cast the given value to the type of the combiner region's argument |
| 303 | // at position argIdx, and increment argIdx. |
| 304 | auto castValueToArgType = [&](Location loc, Value v) { |
| 305 | return convertScalarToDtype( |
| 306 | b, loc, operand: v, |
| 307 | toType: initRegion.getBlocks().front().getArgument(i: argIdx++).getType(), |
| 308 | /*isUnsignedCast=*/false); |
| 309 | }; |
| 310 | for (Value bound : acc::getBounds(accDataClauseOp: op)) { |
| 311 | auto dataBound = bound.getDefiningOp<acc::DataBoundsOp>(); |
| 312 | assert(dataBound && |
| 313 | "acc.reduction's bound must be defined by acc.bounds" ); |
| 314 | // NOTE: we should probably generate get_lowerbound, get_upperbound |
| 315 | // and get_stride here, so that we can stop looking for the acc.bounds |
| 316 | // operation above, and just use the `bound` value. |
| 317 | Value lb = |
| 318 | castValueToArgType(dataBound.getLoc(), dataBound.getLowerbound()); |
| 319 | Value ub = |
| 320 | castValueToArgType(dataBound.getLoc(), dataBound.getUpperbound()); |
| 321 | Value step = |
| 322 | castValueToArgType(dataBound.getLoc(), dataBound.getStride()); |
| 323 | triples.append(IL: {lb, ub, step}); |
| 324 | } |
| 325 | assert(triples.size() + 1 == initNumArguments && |
| 326 | "mismatch between number bounds and number of recipe init block " |
| 327 | "arguments" ); |
| 328 | } |
| 329 | |
| 330 | IRMapping mapping; |
| 331 | SmallVector<Value> initArgs{origPtr}; |
| 332 | initArgs.append(RHS: triples); |
| 333 | mapping.map(from: initRegion.getBlocks().front().getArguments(), to&: initArgs); |
| 334 | |
| 335 | Location loc = op.getLoc(); |
| 336 | setLocation(region&: initRegion, loc); |
| 337 | |
| 338 | if constexpr (std::is_same_v<OpTy, acc::PrivateOp>) { |
| 339 | // Clone the init region for a private. |
| 340 | Block *block = ®ion.front(); |
| 341 | auto [results, ip] = acc::cloneACCRegionInto( |
| 342 | src: &initRegion, dest: block, inlinePoint: block->begin(), mapping, resultsToReplace: {accPtr}); |
| 343 | assert(!results.empty() && "expected a result from init region" ); |
| 344 | saveVarName(op.getAccVar(), results[0]); |
| 345 | resolveVarNamePlaceholders(block, ip, acc::getVariableName(v: op.getAccVar())); |
| 346 | // Clone the destroy region for a private, if it exists. |
| 347 | if (!recipe.getDestroyRegion().empty()) { |
| 348 | results.insert(I: results.begin(), Elt: origPtr); |
| 349 | results.append(RHS: triples); |
| 350 | cloneDestroy(loc, recipe, block, std::prev(x: block->end()), results); |
| 351 | } |
| 352 | } else if constexpr (std::is_same_v<OpTy, acc::FirstprivateOp>) { |
| 353 | // Clone the init region for a firstprivate. |
| 354 | Block *block = ®ion.front(); |
| 355 | auto [results, ip] = acc::cloneACCRegionInto( |
| 356 | src: &initRegion, dest: block, inlinePoint: block->begin(), mapping, resultsToReplace: {accPtr}); |
| 357 | assert(!results.empty() && "expected a result from init region" ); |
| 358 | saveVarName(op.getAccVar(), results[0]); |
| 359 | resolveVarNamePlaceholders(block, ip, acc::getVariableName(v: op.getAccVar())); |
| 360 | // The copy only consumes the original and user-visible private value. |
| 361 | SmallVector<Value> copyArgs{origPtr, results.front()}; |
| 362 | copyArgs.append(RHS: triples); |
| 363 | // Destruction also consumes any cleanup values yielded by init. |
| 364 | SmallVector<Value> destroyArgs{origPtr}; |
| 365 | destroyArgs.append(RHS: results); |
| 366 | destroyArgs.append(RHS: triples); |
| 367 | |
| 368 | // Clone the copy region for a firstprivate |
| 369 | mapping.clear(); |
| 370 | mapping.map(recipe.getCopyRegion().front().getArguments(), copyArgs); |
| 371 | // Clone the copy region for a firstprivate. |
| 372 | Region ©Region = recipe.getCopyRegion(); |
| 373 | setLocation(region&: copyRegion, loc); |
| 374 | acc::cloneACCRegionInto(src: ©Region, dest: block, inlinePoint: std::next(x: ip), mapping, resultsToReplace: {}); |
| 375 | if (!recipe.getDestroyRegion().empty()) { |
| 376 | cloneDestroy(loc, recipe, block, std::prev(x: block->end()), destroyArgs); |
| 377 | } |
| 378 | } else if constexpr (std::is_same_v<OpTy, acc::ReductionOp>) { |
| 379 | if (recipe.getReductionOperator() == acc::ReductionOperator::AccMul && |
| 380 | isComplexReductionType(recipe.getType())) { |
| 381 | (void)accSupport.emitNYI(loc, message: "reduction: complex multiply reduction" ); |
| 382 | return failure(); |
| 383 | } |
| 384 | |
| 385 | auto cloneRegionIntoAccRegion = [&](Region *src, Region *dest, |
| 386 | bool hasResult) { |
| 387 | src->cloneInto(dest, mapper&: mapping); |
| 388 | Block *block = &dest->front(); |
| 389 | Operation *terminator = block->getTerminator(); |
| 390 | b.setInsertionPoint(terminator); |
| 391 | if (hasResult) |
| 392 | acc::YieldOp::create(b, op.getLoc(), terminator->getOperands()); |
| 393 | else |
| 394 | acc::YieldOp::create(b, op.getLoc(), ValueRange{}); |
| 395 | terminator->erase(); |
| 396 | }; |
| 397 | |
| 398 | // Clone the init region into acc.reduction_init. |
| 399 | if constexpr (std::is_same_v<AccOpTy, acc::ParallelOp>) |
| 400 | b.setInsertionPointToStart(®ion.front()); |
| 401 | else if constexpr (std::is_same_v<AccOpTy, acc::LoopOp>) |
| 402 | b.setInsertionPoint(op); |
| 403 | else |
| 404 | llvm_unreachable("unexpected acc op with reduction recipe" ); |
| 405 | |
| 406 | SmallVector<Value> reductionBounds(acc::getBounds(accDataClauseOp: op)); |
| 407 | auto reductionOp = |
| 408 | acc::ReductionInitOp::create(b, op.getLoc(), origPtr, reductionBounds, |
| 409 | recipe.getReductionOperatorAttr()); |
| 410 | saveVarName(op.getAccVar(), reductionOp.getResult()); |
| 411 | cloneRegionIntoAccRegion(&initRegion, &reductionOp.getRegion(), |
| 412 | /*hasResult=*/true); |
| 413 | Block *initBlock = &reductionOp.getRegion().front(); |
| 414 | resolveVarNamePlaceholders(initBlock, std::prev(x: initBlock->end()), |
| 415 | acc::getVariableName(v: op.getAccVar())); |
| 416 | |
| 417 | // Update the uses within the loop to use the reduction op result. |
| 418 | replaceAllUsesInRegionWith(accPtr, reductionOp.getResult(), region); |
| 419 | |
| 420 | // Clone the combiner region into acc.reduction_combine_region. |
| 421 | Region &combinerRegion = recipe.getCombinerRegion(); |
| 422 | setLocation(region&: combinerRegion, loc); |
| 423 | |
| 424 | Block *entryBlock = &combinerRegion.front(); |
| 425 | |
| 426 | if constexpr (std::is_same_v<AccOpTy, acc::ParallelOp>) |
| 427 | b.setInsertionPoint(region.back().getTerminator()); |
| 428 | else if constexpr (std::is_same_v<AccOpTy, acc::LoopOp>) |
| 429 | b.setInsertionPointAfter(accOp); |
| 430 | else |
| 431 | llvm_unreachable("unexpected acc op with reduction recipe" ); |
| 432 | |
| 433 | // Map the first two block arguments to the original and private |
| 434 | // reduction variables. If the recipe's combiner region has the bounds |
| 435 | // arguments, we have to map them to the corresponding operands of |
| 436 | // acc.reduction operation. |
| 437 | mapping.clear(); |
| 438 | SmallVector<Value, 2> argsRemapping{origPtr, reductionOp.getResult()}; |
| 439 | argsRemapping.append(RHS: triples); |
| 440 | mapping.map(from: entryBlock->getArguments(), to&: argsRemapping); |
| 441 | |
| 442 | auto combineRegionOp = acc::ReductionCombineRegionOp::create( |
| 443 | b, op.getLoc(), origPtr, reductionOp.getResult()); |
| 444 | cloneRegionIntoAccRegion(&combinerRegion, &combineRegionOp.getRegion(), |
| 445 | /*hasResult=*/false); |
| 446 | |
| 447 | auto *ctx = b.getContext(); |
| 448 | |
| 449 | // For reductions that come from parallel constructs, explicitly set the |
| 450 | // GPU parallel dimensions attribute to blockXDim since they will always be |
| 451 | // gang private. GPU parallel dimensions cannot be determined for acc.loop |
| 452 | // at this point. |
| 453 | if constexpr (std::is_same_v<AccOpTy, acc::ParallelOp>) { |
| 454 | acc::GPUParallelDimsAttr parDimsAttr; |
| 455 | if (accOp.isEffectivelySerial()) { |
| 456 | // If acc.serial has been lowered to a parallel op that is effectively |
| 457 | // sequential |
| 458 | parDimsAttr = acc::getSeqParDimsAttr(ctx, policy); |
| 459 | } else { |
| 460 | parDimsAttr = acc::getGangDim1ParDimsAttr(ctx, policy); |
| 461 | } |
| 462 | acc::setParDimsAttr(op: reductionOp, attr: parDimsAttr); |
| 463 | acc::setParDimsAttr(op: combineRegionOp, attr: parDimsAttr); |
| 464 | } |
| 465 | |
| 466 | // Set sequential parallel dimensions attribute for loops in the recipe. |
| 467 | auto setSeqParDimsForRecipeLoops = [&](Region *r) { |
| 468 | r->walk([&](LoopLikeOpInterface loopLike) { |
| 469 | acc::setParDimsAttr(op: loopLike, attr: acc::getSeqParDimsAttr(ctx, policy)); |
| 470 | }); |
| 471 | }; |
| 472 | setSeqParDimsForRecipeLoops(&reductionOp.getRegion()); |
| 473 | setSeqParDimsForRecipeLoops(&combineRegionOp.getRegion()); |
| 474 | |
| 475 | if (!recipe.getDestroyRegion().empty()) { |
| 476 | SmallVector<Value> results{origPtr, reductionOp.getResult()}; |
| 477 | results.append(RHS: triples); |
| 478 | Block::iterator ip = std::next(x: Block::iterator(combineRegionOp)); |
| 479 | cloneDestroy(loc, recipe, combineRegionOp->getBlock(), ip, results); |
| 480 | } |
| 481 | } else { |
| 482 | llvm_unreachable("unexpected op type" ); |
| 483 | } |
| 484 | |
| 485 | op.erase(); |
| 486 | return success(); |
| 487 | } |
| 488 | |
| 489 | template <typename OpTy> |
| 490 | LogicalResult ACCRecipeMaterialization::materializeForACCOp( |
| 491 | OpTy accOp, acc::OpenACCSupport &accSupport, |
| 492 | acc::ACCToGPUMappingPolicy &policy) const { |
| 493 | assert(isa<ACC_COMPUTE_CONSTRUCT_AND_LOOP_OPS>(accOp)); |
| 494 | |
| 495 | // Reduction recipes use the original variable both to initialize the |
| 496 | // private reduction value and to combine the result. Preserve copy semantics |
| 497 | // when materializing reductions on compute constructs, before the |
| 498 | // acc.reduction operation carrying that intent is erased. Loop reductions |
| 499 | // are excluded: their original variable can be an outer private reduction |
| 500 | // value rather than a host variable. Keep acc.reduction referring to its |
| 501 | // original host variable and pass the mapped value to materialization |
| 502 | // separately. |
| 503 | struct ReductionMapping { |
| 504 | Value originalVar; |
| 505 | Value mappedVar; |
| 506 | }; |
| 507 | SmallVector<ReductionMapping> mappedReductionVars; |
| 508 | if constexpr (!std::is_same_v<OpTy, acc::LoopOp>) { |
| 509 | for (Value dataOperand : accOp.getDataClauseOperands()) { |
| 510 | Operation *dataOp = dataOperand.getDefiningOp(); |
| 511 | if (dataOp && isa<ACC_DATA_ENTRY_OPS>(Val: dataOp)) |
| 512 | mappedReductionVars.push_back({acc::getVar(accDataClauseOp: dataOp), dataOperand}); |
| 513 | } |
| 514 | } |
| 515 | |
| 516 | auto getMappedReductionVar = [&](acc::ReductionOp reductionOp) -> Value { |
| 517 | if constexpr (std::is_same_v<OpTy, acc::LoopOp>) { |
| 518 | return reductionOp.getVar(); |
| 519 | } else { |
| 520 | Value originalVar = reductionOp.getVar(); |
| 521 | if (isa_and_nonnull<ACC_DATA_ENTRY_OPS>(Val: originalVar.getDefiningOp())) |
| 522 | return originalVar; |
| 523 | |
| 524 | // Note that we do not require matching bounds here. |
| 525 | // Bounds may be represented by different SSA values while evaluating to |
| 526 | // the same values at runtime. Data clauses for the same variable are |
| 527 | // expected to specify matching bounds. |
| 528 | auto existing = llvm::find_if(mappedReductionVars, |
| 529 | [&](const ReductionMapping &mapping) { |
| 530 | return mapping.originalVar == originalVar; |
| 531 | }); |
| 532 | if (existing != mappedReductionVars.end()) |
| 533 | return existing->mappedVar; |
| 534 | |
| 535 | OpBuilder builder(reductionOp); |
| 536 | acc::CopyinOp copyinOp; |
| 537 | if (std::optional<StringRef> name = reductionOp.getName()) |
| 538 | copyinOp = |
| 539 | acc::CopyinOp::create(builder, location: reductionOp.getLoc(), var: originalVar, |
| 540 | /*structured=*/true, /*implicit=*/true, name: *name, |
| 541 | bounds: reductionOp.getBounds()); |
| 542 | else |
| 543 | copyinOp = acc::CopyinOp::create( |
| 544 | builder, location: reductionOp.getLoc(), var: originalVar, |
| 545 | /*structured=*/true, /*implicit=*/true, bounds: reductionOp.getBounds()); |
| 546 | copyinOp.setDataClause(acc::DataClause::acc_reduction); |
| 547 | accOp.getDataClauseOperandsMutable().append(copyinOp.getAccVar()); |
| 548 | |
| 549 | builder.setInsertionPointAfter(accOp); |
| 550 | acc::CopyoutOp copyoutOp; |
| 551 | if (std::optional<StringRef> name = reductionOp.getName()) |
| 552 | copyoutOp = acc::CopyoutOp::create( |
| 553 | builder, location: reductionOp.getLoc(), accVar: copyinOp.getAccVar(), var: originalVar, |
| 554 | /*structured=*/true, /*implicit=*/true, name: *name, |
| 555 | bounds: reductionOp.getBounds()); |
| 556 | else |
| 557 | copyoutOp = acc::CopyoutOp::create( |
| 558 | builder, location: reductionOp.getLoc(), accVar: copyinOp.getAccVar(), var: originalVar, |
| 559 | /*structured=*/true, /*implicit=*/true, bounds: reductionOp.getBounds()); |
| 560 | copyoutOp.setDataClause(acc::DataClause::acc_reduction); |
| 561 | |
| 562 | mappedReductionVars.push_back({originalVar, copyinOp.getAccVar()}); |
| 563 | return copyinOp.getAccVar(); |
| 564 | } |
| 565 | }; |
| 566 | |
| 567 | if (!accOp.getFirstprivateOperands().empty()) { |
| 568 | // Clear the firstprivate operands list so there will be no uses after |
| 569 | // the recipe is materialized. |
| 570 | SmallVector<Value> operands(accOp.getFirstprivateOperands()); |
| 571 | accOp.getFirstprivateOperandsMutable().clear(); |
| 572 | for (Value operand : operands) { |
| 573 | auto firstprivateOp = cast<acc::FirstprivateOp>(Val: operand.getDefiningOp()); |
| 574 | auto symbolRef = cast<SymbolRefAttr>(Val: firstprivateOp.getRecipeAttr()); |
| 575 | auto decl = SymbolTable::lookupNearestSymbolFrom(accOp, symbolRef); |
| 576 | auto recipeOp = cast<acc::FirstprivateRecipeOp>(decl); |
| 577 | LLVM_DEBUG(llvm::dbgs() << "materializing: " << firstprivateOp << "\n" |
| 578 | << symbolRef << "\n" ); |
| 579 | handleInitialValueMapping(op: firstprivateOp); |
| 580 | if (failed( |
| 581 | materialize(firstprivateOp, recipeOp, accOp, accSupport, policy))) |
| 582 | return failure(); |
| 583 | } |
| 584 | } |
| 585 | |
| 586 | if (!accOp.getPrivateOperands().empty()) { |
| 587 | // Clear the private operands list so there will be no uses after |
| 588 | // the recipe is materialized. |
| 589 | SmallVector<Value> operands(accOp.getPrivateOperands()); |
| 590 | accOp.getPrivateOperandsMutable().clear(); |
| 591 | for (Value operand : operands) { |
| 592 | auto privateOp = cast<acc::PrivateOp>(Val: operand.getDefiningOp()); |
| 593 | auto symbolRef = cast<SymbolRefAttr>(Val: privateOp.getRecipeAttr()); |
| 594 | auto decl = SymbolTable::lookupNearestSymbolFrom(accOp, symbolRef); |
| 595 | auto recipeOp = cast<acc::PrivateRecipeOp>(decl); |
| 596 | LLVM_DEBUG(llvm::dbgs() << "materializing: " << privateOp << "\n" |
| 597 | << symbolRef << "\n" ); |
| 598 | if (readsVar(recipeOp.getInitRegion()) || |
| 599 | readsVar(recipeOp.getDestroyRegion())) |
| 600 | handleInitialValueMapping(op: privateOp); |
| 601 | if (failed(materialize(privateOp, recipeOp, accOp, accSupport, policy))) |
| 602 | return failure(); |
| 603 | } |
| 604 | } |
| 605 | |
| 606 | if (!accOp.getReductionOperands().empty()) { |
| 607 | // Clear the reduction operands list so there will be no uses after |
| 608 | // the recipe is materialized. |
| 609 | SmallVector<Value> operands(accOp.getReductionOperands()); |
| 610 | accOp.getReductionOperandsMutable().clear(); |
| 611 | for (Value operand : operands) { |
| 612 | auto reductionOp = cast<acc::ReductionOp>(Val: operand.getDefiningOp()); |
| 613 | auto symbolRef = cast<SymbolRefAttr>(Val: reductionOp.getRecipeAttr()); |
| 614 | auto decl = SymbolTable::lookupNearestSymbolFrom(accOp, symbolRef); |
| 615 | auto recipeOp = cast<acc::ReductionRecipeOp>(decl); |
| 616 | LLVM_DEBUG(llvm::dbgs() << "materializing: " << reductionOp << "\n" |
| 617 | << symbolRef << "\n" ); |
| 618 | Value mappedVar = getMappedReductionVar(reductionOp); |
| 619 | if (failed(materialize(reductionOp, recipeOp, accOp, accSupport, policy, |
| 620 | mappedVar))) |
| 621 | return failure(); |
| 622 | } |
| 623 | } |
| 624 | return success(); |
| 625 | } |
| 626 | |
| 627 | void ACCRecipeMaterialization::runOnOperation() { |
| 628 | ModuleOp moduleOp = getOperation(); |
| 629 | acc::OpenACCSupport &accSupport = getAnalysis<acc::OpenACCSupport>(); |
| 630 | |
| 631 | acc::DefaultACCToGPUMappingPolicy policy; |
| 632 | |
| 633 | // Materialize all recipes for all compute constructs and loop constructs. |
| 634 | bool anyFailed = false; |
| 635 | moduleOp.walk(callback: [&](Operation *op) { |
| 636 | if (anyFailed) |
| 637 | return; |
| 638 | TypeSwitch<Operation *>(op).Case<ACC_COMPUTE_CONSTRUCT_AND_LOOP_OPS>( |
| 639 | caseFn: [&](auto constructOp) { |
| 640 | if (failed(materializeForACCOp(constructOp, accSupport, policy))) |
| 641 | anyFailed = true; |
| 642 | }); |
| 643 | }); |
| 644 | if (anyFailed) { |
| 645 | signalPassFailure(); |
| 646 | return; |
| 647 | } |
| 648 | |
| 649 | // Remove all recipes. Gather the symbol uses that are left with a single |
| 650 | // walk: asking whether each recipe is still referenced walks the whole module |
| 651 | // again for every recipe, and recipes are generated per type, so there can be |
| 652 | // many of them. |
| 653 | std::optional<llvm::DenseSet<StringAttr>> usedSymbols; |
| 654 | // The module region, not the module op, is the symbol table scope: asking |
| 655 | // for the uses on the op itself would not walk into the body. |
| 656 | if (std::optional<SymbolTable::UseRange> uses = |
| 657 | SymbolTable::getSymbolUses(from: &moduleOp.getBodyRegion())) { |
| 658 | usedSymbols.emplace(); |
| 659 | for (const SymbolTable::SymbolUse &use : *uses) |
| 660 | usedSymbols->insert(V: use.getSymbolRef().getLeafReference()); |
| 661 | } |
| 662 | |
| 663 | moduleOp.walk(callback: [&](Operation *op) { |
| 664 | if (auto recipe = dyn_cast<acc::ReductionRecipeOp>(Val: op)) |
| 665 | removeRecipe(op: recipe, moduleOp, usedSymbols); |
| 666 | else if (auto recipe = dyn_cast<acc::PrivateRecipeOp>(Val: op)) |
| 667 | removeRecipe(op: recipe, moduleOp, usedSymbols); |
| 668 | else if (auto recipe = dyn_cast<acc::FirstprivateRecipeOp>(Val: op)) |
| 669 | removeRecipe(op: recipe, moduleOp, usedSymbols); |
| 670 | }); |
| 671 | } |
| 672 | } // namespace |
| 673 | |