| 1 | //===- OpenACCUtilsCG.cpp - OpenACC Code Generation Utilities -------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements utility functions for OpenACC code generation. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h" |
| 14 | |
| 15 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 16 | #include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" |
| 17 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 18 | #include "mlir/Dialect/OpenACC/OpenACCUtilsLoop.h" |
| 19 | #include "mlir/Dialect/OpenACC/OpenACCUtilsReduction.h" |
| 20 | #include "mlir/Dialect/OpenACC/OpenACCUtilsType.h" |
| 21 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 22 | #include "mlir/IR/BuiltinOps.h" |
| 23 | #include "mlir/IR/IRMapping.h" |
| 24 | #include "mlir/Interfaces/FunctionInterfaces.h" |
| 25 | #include "mlir/Interfaces/ValueBoundsOpInterface.h" |
| 26 | #include "llvm/ADT/STLExtras.h" |
| 27 | #include "llvm/ADT/SmallVector.h" |
| 28 | #include "llvm/ADT/TypeSwitch.h" |
| 29 | #include "llvm/Support/MathExtras.h" |
| 30 | |
| 31 | namespace mlir { |
| 32 | namespace acc { |
| 33 | |
| 34 | std::optional<DataLayout> getDataLayout(Operation *op, bool allowDefault) { |
| 35 | if (!op) |
| 36 | return std::nullopt; |
| 37 | |
| 38 | // Walk up the parent chain to find the nearest operation with an explicit |
| 39 | // data layout spec. Check ModuleOp explicitly since it does not actually |
| 40 | // implement DataLayoutOpInterface as a trait (it just has the same methods). |
| 41 | Operation *current = op; |
| 42 | while (current) { |
| 43 | // Check for ModuleOp with explicit data layout spec |
| 44 | if (auto mod = llvm::dyn_cast<ModuleOp>(Val: current)) { |
| 45 | if (mod.getDataLayoutSpec()) |
| 46 | return DataLayout(mod); |
| 47 | } else if (auto dataLayoutOp = |
| 48 | llvm::dyn_cast<DataLayoutOpInterface>(Val: current)) { |
| 49 | // Check other DataLayoutOpInterface implementations |
| 50 | if (dataLayoutOp.getDataLayoutSpec()) |
| 51 | return DataLayout(dataLayoutOp); |
| 52 | } |
| 53 | current = current->getParentOp(); |
| 54 | } |
| 55 | |
| 56 | // No explicit data layout found; return default if allowed |
| 57 | if (allowDefault) { |
| 58 | // Check if op itself is a ModuleOp |
| 59 | if (auto mod = llvm::dyn_cast<ModuleOp>(Val: op)) |
| 60 | return DataLayout(mod); |
| 61 | // Otherwise check parents |
| 62 | if (auto mod = op->getParentOfType<ModuleOp>()) |
| 63 | return DataLayout(mod); |
| 64 | } |
| 65 | |
| 66 | return std::nullopt; |
| 67 | } |
| 68 | |
| 69 | ComputeRegionOp buildComputeRegion(Location loc, ValueRange launchArgs, |
| 70 | ValueRange inputArgs, llvm::StringRef origin, |
| 71 | Region ®ionToClone, |
| 72 | RewriterBase &rewriter, IRMapping &mapping, |
| 73 | ValueRange output, |
| 74 | FlatSymbolRefAttr kernelFuncName, |
| 75 | FlatSymbolRefAttr kernelModuleName, |
| 76 | Value stream, ValueRange inputArgsToMap) { |
| 77 | SmallVector<Type> resultTypes; |
| 78 | for (auto val : output) |
| 79 | resultTypes.push_back(Elt: val.getType()); |
| 80 | auto computeRegion = |
| 81 | ComputeRegionOp::create(builder&: rewriter, location: loc, results: resultTypes, launchArgs, inputArgs, |
| 82 | stream, origin, kernel_func_name: kernelFuncName, kernel_module_name: kernelModuleName); |
| 83 | |
| 84 | assert(!regionToClone.getBlocks().empty() && |
| 85 | "empty region for acc.compute_region" ); |
| 86 | OpBuilder::InsertionGuard guard(rewriter); |
| 87 | |
| 88 | ValueRange mapKeys = inputArgsToMap.empty() ? inputArgs : inputArgsToMap; |
| 89 | assert(mapKeys.size() == inputArgs.size() && |
| 90 | "inputArgsToMap must have same size as inputArgs when provided" ); |
| 91 | |
| 92 | Type indexType = rewriter.getIndexType(); |
| 93 | Block *entryBlock = rewriter.createBlock(parent: &computeRegion.getRegion()); |
| 94 | for (size_t i = 0; i < launchArgs.size(); ++i) |
| 95 | entryBlock->addArgument(type: indexType, loc); |
| 96 | for (Value input : inputArgs) |
| 97 | entryBlock->addArgument(type: input.getType(), loc); |
| 98 | for (size_t i = 0; i < inputArgs.size(); ++i) |
| 99 | mapping.map(from: mapKeys[i], to: entryBlock->getArgument(i: launchArgs.size() + i)); |
| 100 | rewriter.setInsertionPointToStart(entryBlock); |
| 101 | if (regionToClone.getBlocks().size() == 1) { |
| 102 | for (auto &op : regionToClone.front().getOperations()) { |
| 103 | if (op.hasTrait<OpTrait::IsTerminator>()) |
| 104 | break; |
| 105 | rewriter.clone(op, mapper&: mapping); |
| 106 | } |
| 107 | SmallVector<Value> yieldOperands; |
| 108 | for (auto val : output) |
| 109 | yieldOperands.push_back(Elt: mapping.lookup(from: val)); |
| 110 | rewriter.setInsertionPointToEnd(entryBlock); |
| 111 | YieldOp::create(builder&: rewriter, location: loc, operands: yieldOperands); |
| 112 | } else { |
| 113 | auto exeRegion = mlir::acc::wrapMultiBlockRegionWithSCFExecuteRegion( |
| 114 | region&: regionToClone, mapping, loc, rewriter); |
| 115 | if (!exeRegion) { |
| 116 | rewriter.eraseOp(op: computeRegion); |
| 117 | return nullptr; |
| 118 | } |
| 119 | SmallVector<scf::YieldOp> yieldOps( |
| 120 | llvm::to_vector(Range: exeRegion.getOps<scf::YieldOp>())); |
| 121 | assert(!yieldOps.empty() && |
| 122 | "multi-block region must contain at least one scf.yield" ); |
| 123 | assert(llvm::all_of(yieldOps, |
| 124 | [&output](scf::YieldOp yieldOp) { |
| 125 | return yieldOp.getNumOperands() == |
| 126 | static_cast<int64_t>(output.size()) && |
| 127 | llvm::all_of( |
| 128 | llvm::zip(yieldOp.getOperands(), output), |
| 129 | [](auto pair) { |
| 130 | return std::get<0>(pair).getType() == |
| 131 | std::get<1>(pair).getType(); |
| 132 | }); |
| 133 | }) && |
| 134 | "each scf.yield operand count and types must match output" ); |
| 135 | rewriter.setInsertionPointToEnd(entryBlock); |
| 136 | YieldOp::create(builder&: rewriter, location: loc, operands: exeRegion.getResults()); |
| 137 | } |
| 138 | |
| 139 | return computeRegion; |
| 140 | } |
| 141 | |
| 142 | static SmallVector<GPUParallelDimAttr>::iterator |
| 143 | findParDim(SmallVector<GPUParallelDimAttr> &parDims, |
| 144 | GPUParallelDimAttr parDim) { |
| 145 | return llvm::lower_bound( |
| 146 | Range&: parDims, Value&: parDim, |
| 147 | C: [](const GPUParallelDimAttr &lhs, const GPUParallelDimAttr &rhs) { |
| 148 | return lhs.getOrder() > rhs.getOrder(); |
| 149 | }); |
| 150 | } |
| 151 | |
| 152 | void insertParDim(SmallVector<GPUParallelDimAttr> &parDims, |
| 153 | GPUParallelDimAttr parDim) { |
| 154 | SmallVector<GPUParallelDimAttr>::iterator lb = findParDim(parDims, parDim); |
| 155 | if (lb == parDims.end() || *lb != parDim) |
| 156 | parDims.insert(I: lb, Elt: parDim); |
| 157 | } |
| 158 | |
| 159 | void removeParDim(SmallVector<GPUParallelDimAttr> &parDims, |
| 160 | GPUParallelDimAttr parDim) { |
| 161 | SmallVector<GPUParallelDimAttr>::iterator lb = findParDim(parDims, parDim); |
| 162 | if (lb != parDims.end() && *lb == parDim) |
| 163 | parDims.erase(CI: lb); |
| 164 | } |
| 165 | |
| 166 | #define ACC_OP_WITH_PAR_DIMS_LIST \ |
| 167 | PrivatizeOp, ReductionAccumulateOp, ReductionAccumulateArrayOp, \ |
| 168 | ReductionCombineOp |
| 169 | |
| 170 | GPUParallelDimsAttr getParDimsAttr(Operation *op) { |
| 171 | return llvm::TypeSwitch<Operation *, GPUParallelDimsAttr>(op) |
| 172 | .Case<ACC_OP_WITH_PAR_DIMS_LIST>( |
| 173 | caseFn: [](auto parOp) { return parOp.getParDimsAttr(); }) |
| 174 | .Default(defaultFn: [](Operation *op) -> GPUParallelDimsAttr { |
| 175 | if (Attribute attr = |
| 176 | op->getDiscardableAttr(name: GPUParallelDimsAttr::name)) { |
| 177 | GPUParallelDimsAttr parDimsAttr = dyn_cast<GPUParallelDimsAttr>(Val&: attr); |
| 178 | assert(parDimsAttr && "acc.par_dims must be a GPUParallelDimsAttr" ); |
| 179 | return parDimsAttr; |
| 180 | } |
| 181 | return nullptr; |
| 182 | }); |
| 183 | } |
| 184 | |
| 185 | bool hasParDimsAttr(Operation *op) { return getParDimsAttr(op) != nullptr; } |
| 186 | |
| 187 | bool hasSeqParDims(Operation *op) { |
| 188 | if (GPUParallelDimsAttr parDimsAttr = getParDimsAttr(op)) |
| 189 | return parDimsAttr.isSeq(); |
| 190 | return false; |
| 191 | } |
| 192 | |
| 193 | void setParDimsAttr(Operation *op, GPUParallelDimsAttr attr) { |
| 194 | assert(!hasParDimsAttr(op) && "parallel dimensions attribute is already set" ); |
| 195 | llvm::TypeSwitch<Operation *>(op) |
| 196 | .Case<ACC_OP_WITH_PAR_DIMS_LIST>( |
| 197 | caseFn: [&](auto parOp) { parOp.setParDimsAttr(attr); }) |
| 198 | .Default(defaultFn: [&](Operation *op) { |
| 199 | op->setDiscardableAttr(name: GPUParallelDimsAttr::name, value: attr); |
| 200 | }); |
| 201 | } |
| 202 | |
| 203 | void updateParDimsAttr(Operation *op, GPUParallelDimsAttr attr) { |
| 204 | assert(hasParDimsAttr(op) && |
| 205 | "expected parallel dimensions attribute to already be set" ); |
| 206 | llvm::TypeSwitch<Operation *>(op) |
| 207 | .Case<ACC_OP_WITH_PAR_DIMS_LIST>( |
| 208 | caseFn: [&](auto parOp) { parOp.setParDimsAttr(attr); }) |
| 209 | .Default(defaultFn: [&](Operation *op) { |
| 210 | op->setDiscardableAttr(name: GPUParallelDimsAttr::name, value: attr); |
| 211 | }); |
| 212 | } |
| 213 | |
| 214 | #undef ACC_OP_WITH_PAR_DIMS_LIST |
| 215 | |
| 216 | bool hasGPUBlockRedundantAttr(Operation *op) { |
| 217 | return op->hasDiscardableAttrOfType<GPUBlockRedundantAttr>( |
| 218 | name: GPUBlockRedundantAttr::name); |
| 219 | } |
| 220 | |
| 221 | void setGPUBlockRedundantAttr(Operation *op) { |
| 222 | op->setDiscardableAttr(name: GPUBlockRedundantAttr::name, |
| 223 | value: GPUBlockRedundantAttr::get(ctx: op->getContext())); |
| 224 | } |
| 225 | |
| 226 | ChunkSizeAttr getChunkSizeAttr(Operation *op) { |
| 227 | return op->getDiscardableAttrOfType<ChunkSizeAttr>(name: ChunkSizeAttr::name); |
| 228 | } |
| 229 | |
| 230 | bool hasChunkSizeAttr(Operation *op) { return getChunkSizeAttr(op) != nullptr; } |
| 231 | |
| 232 | void setChunkSizeAttr(Operation *op, ChunkSizeAttr attr) { |
| 233 | op->setDiscardableAttr(name: ChunkSizeAttr::name, value: attr); |
| 234 | } |
| 235 | |
| 236 | void setChunkSizeAttr(Operation *op, int64_t chunkSize) { |
| 237 | setChunkSizeAttr(op, attr: ChunkSizeAttr::get(context: op->getContext(), chunk_size: chunkSize)); |
| 238 | } |
| 239 | |
| 240 | std::optional<int64_t> getChunkSize(Operation *op) { |
| 241 | if (ChunkSizeAttr attr = getChunkSizeAttr(op)) |
| 242 | return attr.getChunkSize(); |
| 243 | return std::nullopt; |
| 244 | } |
| 245 | |
| 246 | void copyParDimsAttr(Operation *from, Operation *to) { |
| 247 | assert(hasParDimsAttr(from) && |
| 248 | "expected parallel dimensions attribute to already be set" ); |
| 249 | setParDimsAttr(op: to, attr: getParDimsAttr(op: from)); |
| 250 | } |
| 251 | |
| 252 | ActiveParDimsAttr getActiveParDimsAttr(Operation *op) { |
| 253 | return op->getDiscardableAttrOfType<ActiveParDimsAttr>( |
| 254 | name: ActiveParDimsAttr::name); |
| 255 | } |
| 256 | |
| 257 | bool hasActiveParDimsAttr(Operation *op) { |
| 258 | return getActiveParDimsAttr(op) != nullptr; |
| 259 | } |
| 260 | |
| 261 | void setActiveParDimsAttr(Operation *op, ActiveParDimsAttr attr) { |
| 262 | op->setDiscardableAttr(name: ActiveParDimsAttr::name, value: attr); |
| 263 | } |
| 264 | |
| 265 | void setActiveParDimsAttr(Operation *op, ArrayRef<GPUParallelDimAttr> dims) { |
| 266 | setActiveParDimsAttr(op, attr: ActiveParDimsAttr::get(context: op->getContext(), array: dims)); |
| 267 | } |
| 268 | |
| 269 | int64_t SharedMemoryBudget::alignOffset(int64_t offset, int64_t alignment) { |
| 270 | assert(alignment > 0 && llvm::isPowerOf2_64(alignment) && |
| 271 | "alignment must be a power of two" ); |
| 272 | return (offset + alignment - 1) & ~(alignment - 1); |
| 273 | } |
| 274 | |
| 275 | bool SharedMemoryBudget::tryAllocate(int64_t bytes, int64_t alignment) { |
| 276 | int64_t aligned = alignOffset(offset: bytesUsed_, alignment); |
| 277 | if (aligned + bytes > maxTotalBytes_) { |
| 278 | return false; |
| 279 | } |
| 280 | bytesUsed_ = aligned + bytes; |
| 281 | return true; |
| 282 | } |
| 283 | |
| 284 | int64_t sumExistingSharedMemoryBytes(Region ®ion) { |
| 285 | int64_t total = 0; |
| 286 | region.walk(callback: [&](GPUSharedMemoryOp op) { |
| 287 | int64_t upperBound = op.getStaticUpperBoundBytes(); |
| 288 | total = SharedMemoryBudget::alignOffset(offset: total) + upperBound; |
| 289 | }); |
| 290 | return total; |
| 291 | } |
| 292 | |
| 293 | PrivatizeOp getPrivatizeOp(PrivateLocalOp privateLocal, |
| 294 | ComputeRegionOp computeRegion) { |
| 295 | Value value = privateLocal.getPrivatized(); |
| 296 | if (BlockArgument blockArg = dyn_cast<BlockArgument>(Val&: value)) { |
| 297 | auto owner = dyn_cast<ComputeRegionOp>(Val: blockArg.getOwner()->getParentOp()); |
| 298 | value = (owner ? owner : computeRegion).getOperand(blockArg); |
| 299 | } |
| 300 | PrivatizeOp privatizeOp = value.getDefiningOp<PrivatizeOp>(); |
| 301 | assert(privatizeOp && "expected privatize op to be the defining op" ); |
| 302 | return privatizeOp; |
| 303 | } |
| 304 | |
| 305 | static bool isThreadXPrivatize(PrivatizeOp privatize) { |
| 306 | if (GPUParallelDimsAttr parDimsAttr = privatize.getParDimsAttr()) |
| 307 | return llvm::any_of(Range: parDimsAttr.getArray(), |
| 308 | P: [](GPUParallelDimAttr d) { return d.isThreadX(); }); |
| 309 | return false; |
| 310 | } |
| 311 | |
| 312 | MemRefType getPrivateBaseMemRefType(Type baseTy, ModuleOp module) { |
| 313 | auto memrefTy = cast<PointerLikeType>(Val&: baseTy).getAsMemRefType(module); |
| 314 | assert(memrefTy && "private base type must be convertible to memref" ); |
| 315 | return memrefTy; |
| 316 | } |
| 317 | |
| 318 | SmallVector<GPUParallelDimAttr> |
| 319 | collectPrivateLocalParDims(PrivateLocalOp privateLocal, |
| 320 | ComputeRegionOp computeRegion) { |
| 321 | SmallVector<GPUParallelDimAttr> parDims; |
| 322 | // Walk the enclosing scf.parallel loops, but stop at the compute region |
| 323 | // boundary: loops outside the compute region do not contribute parallel |
| 324 | // dimensions to this privatization. |
| 325 | auto parentLoop = privateLocal->getParentOfType<scf::ParallelOp>(); |
| 326 | while (parentLoop && computeRegion->isProperAncestor(other: parentLoop)) { |
| 327 | if (GPUParallelDimsAttr parDimsAttr = getParDimsAttr(op: parentLoop)) |
| 328 | for (GPUParallelDimAttr parDim : parDimsAttr.getArray()) |
| 329 | insertParDim(parDims, parDim); |
| 330 | parentLoop = parentLoop->getParentOfType<scf::ParallelOp>(); |
| 331 | } |
| 332 | if (GPUParallelDimsAttr parDimsAttr = getParDimsAttr(op: computeRegion)) |
| 333 | for (GPUParallelDimAttr parDim : parDimsAttr.getArray()) |
| 334 | insertParDim(parDims, parDim); |
| 335 | if (parDims.empty()) { |
| 336 | for (GPUParallelDimAttr parDim : computeRegion.getLaunchParDims()) { |
| 337 | if (parDim.isAnyBlock()) |
| 338 | insertParDim(parDims, parDim); |
| 339 | } |
| 340 | } |
| 341 | |
| 342 | for (Operation *user : privateLocal.getResult().getUsers()) { |
| 343 | if (auto accumulateOp = dyn_cast<ReductionAccumulateOp>(Val: user)) { |
| 344 | if (accumulateOp.getMemref() == privateLocal.getResult()) |
| 345 | for (GPUParallelDimAttr parDim : accumulateOp.getParDims().getArray()) |
| 346 | insertParDim(parDims, parDim); |
| 347 | } |
| 348 | if (auto combineOp = dyn_cast<ReductionCombineOp>(Val: user)) { |
| 349 | if (combineOp.getSrcMemref() == privateLocal.getResult()) |
| 350 | for (GPUParallelDimAttr parDim : getReductionCombineParDims(op: combineOp)) |
| 351 | insertParDim(parDims, parDim); |
| 352 | } |
| 353 | if (auto combineRegionOp = dyn_cast<ReductionCombineRegionOp>(Val: user)) { |
| 354 | if (combineRegionOp.getSrcVar() == privateLocal.getResult()) |
| 355 | for (GPUParallelDimAttr parDim : |
| 356 | getReductionCombineParDims(op: combineRegionOp)) |
| 357 | insertParDim(parDims, parDim); |
| 358 | } |
| 359 | } |
| 360 | return parDims; |
| 361 | } |
| 362 | |
| 363 | static FailureOr<std::optional<int64_t>> getWorkerPrivateSharedMemoryNumCopies( |
| 364 | PrivateLocalOp privateLocal, ComputeRegionOp computeRegion, |
| 365 | bool isWorkerPrivate, OpenACCSupport *support) { |
| 366 | if (!isWorkerPrivate) |
| 367 | return std::optional<int64_t>(1); |
| 368 | |
| 369 | GPUParallelDimAttr threadY = |
| 370 | GPUParallelDimAttr::threadYDim(privateLocal.getContext()); |
| 371 | std::optional<Value> workerArg = computeRegion.getKnownLaunchArg(parDim: threadY); |
| 372 | if (!workerArg) |
| 373 | return std::optional<int64_t>(); |
| 374 | |
| 375 | auto workerArgConst = workerArg->getDefiningOp<arith::ConstantIndexOp>(); |
| 376 | if (workerArgConst) |
| 377 | return std::optional<int64_t>(workerArgConst.value()); |
| 378 | |
| 379 | FailureOr<int64_t> workerArgBound = |
| 380 | ValueBoundsConstraintSet::computeConstantBound(type: presburger::BoundType::EQ, |
| 381 | var: *workerArg); |
| 382 | if (succeeded(Result: workerArgBound)) |
| 383 | return std::optional<int64_t>(*workerArgBound); |
| 384 | |
| 385 | if (support) { |
| 386 | (void)support->emitNYI(loc: privateLocal.getLoc(), |
| 387 | message: "worker-private variables in shared memory " |
| 388 | "require compile-time constant num_workers" ); |
| 389 | return failure(); |
| 390 | } |
| 391 | return std::optional<int64_t>(); |
| 392 | } |
| 393 | |
| 394 | static bool isInsideACCSpecializedRoutine(Operation *op) { |
| 395 | auto funcOp = op->getParentOfType<FunctionOpInterface>(); |
| 396 | return funcOp && isSpecializedAccRoutine(op: funcOp); |
| 397 | } |
| 398 | |
| 399 | FailureOr<bool> isPrivateLocalSharedMemoryCandidate( |
| 400 | PrivateLocalOp privateLocal, ComputeRegionOp computeRegion, ModuleOp module, |
| 401 | const ACCToGPUMappingPolicy &policy, OpenACCSupport *support) { |
| 402 | if (isInsideACCSpecializedRoutine(op: computeRegion)) |
| 403 | return false; |
| 404 | |
| 405 | if (isThreadXPrivatize(privatize: getPrivatizeOp(privateLocal, computeRegion))) |
| 406 | return false; |
| 407 | |
| 408 | bool isReductionAccumulator = |
| 409 | llvm::any_of(Range: privateLocal.getResult().getUsers(), P: [](Operation *user) { |
| 410 | return isa<ReductionAccumulateOp>(Val: user); |
| 411 | }); |
| 412 | |
| 413 | SmallVector<GPUParallelDimAttr> parDims = |
| 414 | collectPrivateLocalParDims(privateLocal, computeRegion); |
| 415 | bool isGangPrivate = |
| 416 | llvm::any_of(Range&: parDims, P: [&](auto parDim) { return policy.isGang(attr: parDim); }); |
| 417 | bool isWorkerPrivate = llvm::any_of( |
| 418 | Range&: parDims, P: [&](auto parDim) { return policy.isWorker(attr: parDim); }); |
| 419 | bool isVectorPrivate = llvm::any_of( |
| 420 | Range&: parDims, P: [&](auto parDim) { return policy.isVector(attr: parDim); }); |
| 421 | |
| 422 | auto baseTy = getPrivateBaseMemRefType( |
| 423 | baseTy: cast<PrivateType>(Val: privateLocal.getPrivatized().getType()).getBaseTy(), |
| 424 | module); |
| 425 | |
| 426 | bool isBlockLevelPrivate = |
| 427 | !isVectorPrivate && |
| 428 | (isGangPrivate || |
| 429 | (isWorkerPrivate && baseTy.getRank() > 0 && !isReductionAccumulator)); |
| 430 | if (!isBlockLevelPrivate) |
| 431 | return false; |
| 432 | |
| 433 | for (int64_t dim : baseTy.getShape()) |
| 434 | if (dim == ShapedType::kDynamic) |
| 435 | return false; |
| 436 | |
| 437 | auto resultMemRefTy = dyn_cast<MemRefType>(Val: privateLocal.getType()); |
| 438 | if (!resultMemRefTy || !resultMemRefTy.getLayout().isIdentity() || |
| 439 | resultMemRefTy.getMemorySpace()) |
| 440 | return false; |
| 441 | |
| 442 | if (isGangPrivate && isWorkerPrivate && !isReductionAccumulator) |
| 443 | return false; |
| 444 | |
| 445 | FailureOr<std::optional<int64_t>> numCopies = |
| 446 | getWorkerPrivateSharedMemoryNumCopies(privateLocal, computeRegion, |
| 447 | isWorkerPrivate, support); |
| 448 | if (failed(Result: numCopies)) |
| 449 | return failure(); |
| 450 | return numCopies->has_value(); |
| 451 | } |
| 452 | |
| 453 | std::optional<int64_t> getPrivateLocalSharedMemoryUpperBoundBytes( |
| 454 | PrivateLocalOp privateLocal, ComputeRegionOp computeRegion, ModuleOp module, |
| 455 | const ACCToGPUMappingPolicy &policy, OpenACCSupport *support) { |
| 456 | FailureOr<bool> isCandidate = isPrivateLocalSharedMemoryCandidate( |
| 457 | privateLocal, computeRegion, module, policy); |
| 458 | if (failed(Result: isCandidate) || !*isCandidate) |
| 459 | return std::nullopt; |
| 460 | |
| 461 | SmallVector<GPUParallelDimAttr> parDims = |
| 462 | collectPrivateLocalParDims(privateLocal, computeRegion); |
| 463 | bool isWorkerPrivate = llvm::any_of( |
| 464 | Range&: parDims, P: [&](auto parDim) { return policy.isWorker(attr: parDim); }); |
| 465 | |
| 466 | FailureOr<std::optional<int64_t>> numCopies = |
| 467 | getWorkerPrivateSharedMemoryNumCopies( |
| 468 | privateLocal, computeRegion, isWorkerPrivate, /*support=*/nullptr); |
| 469 | if (failed(Result: numCopies) || !numCopies->has_value()) |
| 470 | return std::nullopt; |
| 471 | |
| 472 | auto baseTy = getPrivateBaseMemRefType( |
| 473 | baseTy: cast<PrivateType>(Val: privateLocal.getPrivatized().getType()).getBaseTy(), |
| 474 | module); |
| 475 | std::optional<TypeSizeAndAlignment> elementSizeAndAlignment = |
| 476 | getTypeSizeAndAlignment(ty: baseTy.getElementType(), module, support); |
| 477 | if (!elementSizeAndAlignment) |
| 478 | return std::nullopt; |
| 479 | |
| 480 | int64_t numElements = 1; |
| 481 | for (int64_t dim : baseTy.getShape()) |
| 482 | numElements *= dim; |
| 483 | return elementSizeAndAlignment->first.getFixedValue() * numElements * |
| 484 | numCopies->value(); |
| 485 | } |
| 486 | |
| 487 | bool hasAttachPoint(Operation *mapEntryOp) { |
| 488 | if (!mapEntryOp) |
| 489 | return false; |
| 490 | if (auto mapInfo = dyn_cast<MapInfoOp>(Val: mapEntryOp)) |
| 491 | return mapInfo.getVarPtrPtr() != nullptr; |
| 492 | if (isa<AttachOp>(Val: mapEntryOp)) |
| 493 | return true; |
| 494 | if (std::optional<DataClause> clause = getDataClause(accDataEntryOp: mapEntryOp)) { |
| 495 | if (*clause == DataClause::acc_attach || *clause == DataClause::acc_detach) |
| 496 | return true; |
| 497 | } |
| 498 | return getVarPtrPtr(accDataClauseOp: mapEntryOp) != nullptr; |
| 499 | } |
| 500 | |
| 501 | DataDescKind getDataDescKind(Operation *mapEntryOp) { |
| 502 | if (auto mapInfo = dyn_cast<MapInfoOp>(Val: mapEntryOp)) |
| 503 | return mapInfo.getDescKind(); |
| 504 | return DataDescKind::none; |
| 505 | } |
| 506 | |
| 507 | Value getDesc(Operation *mapEntryOp) { |
| 508 | auto mapInfo = dyn_cast<MapInfoOp>(Val: mapEntryOp); |
| 509 | if (!mapInfo) |
| 510 | return {}; |
| 511 | if (Value desc = mapInfo.getDesc()) |
| 512 | return desc; |
| 513 | // When the mapped var is itself the descriptor, map_info omits a redundant |
| 514 | // `desc` operand; recover it from `var` whenever a descriptor kind is set. |
| 515 | if (mapInfo.getDescKind() != DataDescKind::none) |
| 516 | return mapInfo.getVar(); |
| 517 | return {}; |
| 518 | } |
| 519 | |
| 520 | std::optional<int64_t> getMapElementSize(Operation *mapEntryOp) { |
| 521 | if (auto mapInfo = dyn_cast<MapInfoOp>(Val: mapEntryOp)) |
| 522 | if (auto attr = mapInfo.getElementSizeAttr()) |
| 523 | return attr.getInt(); |
| 524 | return std::nullopt; |
| 525 | } |
| 526 | |
| 527 | Value getMapSize(Operation *mapEntryOp) { |
| 528 | if (auto mapInfo = dyn_cast<MapInfoOp>(Val: mapEntryOp)) |
| 529 | return mapInfo.getSize(); |
| 530 | return {}; |
| 531 | } |
| 532 | |
| 533 | std::optional<MapFlags> getMapFlags(Operation *mapEntryOp) { |
| 534 | if (auto mapInfo = dyn_cast<MapInfoOp>(Val: mapEntryOp)) |
| 535 | return mapInfo.getMapFlags(); |
| 536 | return std::nullopt; |
| 537 | } |
| 538 | |
| 539 | SmallVector<Operation *> getPairedDataExitOps(Value entryResult) { |
| 540 | SmallVector<Operation *> exitOps; |
| 541 | for (OpOperand &use : entryResult.getUses()) { |
| 542 | Operation *op = use.getOwner(); |
| 543 | if (!isa<ACC_DATA_EXIT_OPS>(Val: op)) |
| 544 | continue; |
| 545 | Value accVar; |
| 546 | llvm::TypeSwitch<Operation *>(op).Case<ACC_DATA_EXIT_OPS>( |
| 547 | caseFn: [&](auto exit) { accVar = exit.getAccVar(); }); |
| 548 | // The entry result can also be used as another operand, such as async. |
| 549 | if (accVar == entryResult) |
| 550 | exitOps.push_back(Elt: op); |
| 551 | } |
| 552 | return exitOps; |
| 553 | } |
| 554 | |
| 555 | static Operation *findCorrespondingDataExit(Value entryResult) { |
| 556 | SmallVector<Operation *> exitOps = getPairedDataExitOps(entryResult); |
| 557 | return exitOps.empty() ? nullptr : exitOps.front(); |
| 558 | } |
| 559 | |
| 560 | static std::optional<Location> getMappingExitLoc(Value entryResult) { |
| 561 | if (auto mapInfo = entryResult.getDefiningOp<MapInfoOp>()) |
| 562 | if (std::optional<Location> exitLoc = mapInfo.getExitLoc()) |
| 563 | return exitLoc; |
| 564 | if (Operation *exitOp = findCorrespondingDataExit(entryResult)) |
| 565 | return exitOp->getLoc(); |
| 566 | return std::nullopt; |
| 567 | } |
| 568 | |
| 569 | std::optional<Location> getMappingExitLoc(ValueRange dataClauseOperands) { |
| 570 | for (Value operand : dataClauseOperands) |
| 571 | if (std::optional<Location> exitLoc = getMappingExitLoc(entryResult: operand)) |
| 572 | return exitLoc; |
| 573 | return std::nullopt; |
| 574 | } |
| 575 | |
| 576 | static std::optional<DataClause> getExitDataClause(Operation *exitOp) { |
| 577 | return llvm::TypeSwitch<Operation *, std::optional<DataClause>>(exitOp) |
| 578 | .Case<ACC_DATA_EXIT_OPS>(caseFn: [&](auto exit) { return exit.getDataClause(); }) |
| 579 | .Default(defaultFn: [&](Operation *) { return std::nullopt; }); |
| 580 | } |
| 581 | |
| 582 | bool hasCopyOutSibling(Operation *entryOp) { |
| 583 | auto getMappedVar = [](Operation *op) { |
| 584 | Value var = getVar(accDataClauseOp: op); |
| 585 | return var ? var : getVarPtr(accDataClauseOp: op); |
| 586 | }; |
| 587 | Value var = getMappedVar(entryOp); |
| 588 | if (!var) |
| 589 | return false; |
| 590 | |
| 591 | auto copiesOutOnly = [&](Value sibling) { |
| 592 | Operation *siblingOp = sibling.getDefiningOp(); |
| 593 | if (!siblingOp || getMappedVar(siblingOp) != var) |
| 594 | return false; |
| 595 | if (std::optional<MapFlags> siblingFlags = getMapFlags(mapEntryOp: siblingOp)) |
| 596 | return bitEnumContainsAny(bits: *siblingFlags, bit: MapFlags::from) && |
| 597 | !bitEnumContainsAny(bits: *siblingFlags, bit: MapFlags::to); |
| 598 | std::optional<DataClause> siblingClause = getDataClause(accDataEntryOp: siblingOp); |
| 599 | return siblingClause && (*siblingClause == DataClause::acc_copyout || |
| 600 | *siblingClause == DataClause::acc_copyout_zero); |
| 601 | }; |
| 602 | |
| 603 | Value entryResult = entryOp->getResult(idx: 0); |
| 604 | auto mapsSameVarOnConstruct = [&](auto construct) { |
| 605 | return llvm::any_of(construct.getDataClauseOperands(), [&](Value sibling) { |
| 606 | return sibling != entryResult && copiesOutOnly(sibling); |
| 607 | }); |
| 608 | }; |
| 609 | return llvm::any_of(Range: entryResult.getUsers(), P: [&](Operation *user) { |
| 610 | return llvm::TypeSwitch<Operation *, bool>(user) |
| 611 | .Case<KernelEnvironmentOp, KernelsOp, ParallelOp, SerialOp, DataOp>( |
| 612 | caseFn&: mapsSameVarOnConstruct) |
| 613 | .Default(defaultResult: false); |
| 614 | }); |
| 615 | } |
| 616 | |
| 617 | static DataClauseModifier getEntryModifiers(Operation *entryOp) { |
| 618 | return llvm::TypeSwitch<Operation *, DataClauseModifier>(entryOp) |
| 619 | .Case<ACC_DATA_ENTRY_OPS>( |
| 620 | caseFn: [&](auto entry) { return entry.getModifiers(); }) |
| 621 | .Default(defaultFn: [&](Operation *) { return DataClauseModifier::none; }); |
| 622 | } |
| 623 | |
| 624 | MapFlags computePrivatizeMapFlags(PrivatizeOp privatizeOp, |
| 625 | const ACCToGPUMappingPolicy &policy) { |
| 626 | MapFlags flags = MapFlags::private_; |
| 627 | |
| 628 | // Storage is private without being replicated per parallel level when the |
| 629 | // privatization does not name any parallel dimension. |
| 630 | GPUParallelDimsAttr parDims = privatizeOp.getParDimsAttr(); |
| 631 | if (!parDims) |
| 632 | return flags; |
| 633 | |
| 634 | auto addLevelFlags = [&](ArrayRef<GPUParallelDimAttr> dims) { |
| 635 | for (GPUParallelDimAttr parDim : dims) { |
| 636 | if (policy.isGang(attr: parDim)) |
| 637 | flags = flags | MapFlags::gang_private; |
| 638 | else if (policy.isWorker(attr: parDim)) |
| 639 | flags = flags | MapFlags::worker_private; |
| 640 | else if (policy.isVector(attr: parDim)) |
| 641 | flags = flags | MapFlags::vector_private; |
| 642 | } |
| 643 | }; |
| 644 | addLevelFlags(parDims.getArray()); |
| 645 | // Ownership par_dims may be narrower than the dimensions the kernel indexes |
| 646 | // the storage by, so replicate across the active dimensions as well. |
| 647 | if (ActiveParDimsAttr activeParDims = getActiveParDimsAttr(op: privatizeOp)) { |
| 648 | addLevelFlags(activeParDims.getArray()); |
| 649 | } |
| 650 | return flags; |
| 651 | } |
| 652 | |
| 653 | MapFlags computeDataClauseMapFlags(Operation *entryOp, bool ptrAndObj) { |
| 654 | MapFlags flags = MapFlags::none; |
| 655 | std::optional<DataClause> enterClause = getDataClause(accDataEntryOp: entryOp); |
| 656 | if (!enterClause) |
| 657 | return flags; |
| 658 | |
| 659 | switch (*enterClause) { |
| 660 | case DataClause::acc_create: |
| 661 | case DataClause::acc_copyout: |
| 662 | case DataClause::acc_present: |
| 663 | case DataClause::acc_private: |
| 664 | case DataClause::acc_firstprivate: |
| 665 | case DataClause::acc_delete: |
| 666 | case DataClause::acc_update_host: |
| 667 | case DataClause::acc_update_self: |
| 668 | case DataClause::acc_declare_device_resident: |
| 669 | if (*enterClause == DataClause::acc_declare_device_resident) |
| 670 | flags = flags | MapFlags::device_resident; |
| 671 | if (*enterClause == DataClause::acc_present) |
| 672 | flags = flags | MapFlags::present; |
| 673 | if (*enterClause == DataClause::acc_private || |
| 674 | *enterClause == DataClause::acc_firstprivate) |
| 675 | flags = flags | MapFlags::private_; |
| 676 | if (*enterClause == DataClause::acc_firstprivate) |
| 677 | flags = flags | MapFlags::to; |
| 678 | break; |
| 679 | case DataClause::acc_deviceptr: |
| 680 | flags = flags | MapFlags::devptr; |
| 681 | break; |
| 682 | case DataClause::acc_create_zero: |
| 683 | case DataClause::acc_copyout_zero: |
| 684 | flags = flags | MapFlags::init_zero; |
| 685 | break; |
| 686 | case DataClause::acc_copy: |
| 687 | case DataClause::acc_copyin: |
| 688 | case DataClause::acc_copyin_readonly: |
| 689 | case DataClause::acc_reduction: |
| 690 | case DataClause::acc_update_device: |
| 691 | flags = flags | MapFlags::to; |
| 692 | break; |
| 693 | case DataClause::acc_no_create: |
| 694 | flags = flags | MapFlags::no_create; |
| 695 | break; |
| 696 | case DataClause::acc_attach: |
| 697 | flags = flags | MapFlags::attach; |
| 698 | break; |
| 699 | case DataClause::acc_detach: |
| 700 | flags = flags | MapFlags::detach; |
| 701 | break; |
| 702 | case DataClause::acc_declare_link: |
| 703 | flags = flags | MapFlags::link; |
| 704 | break; |
| 705 | default: |
| 706 | break; |
| 707 | } |
| 708 | if (*enterClause == DataClause::acc_reduction) |
| 709 | flags = flags | MapFlags::reduction; |
| 710 | |
| 711 | std::optional<DataClause> exitClause; |
| 712 | if (Operation *exitOp = findCorrespondingDataExit(entryResult: entryOp->getResult(idx: 0))) |
| 713 | exitClause = getExitDataClause(exitOp); |
| 714 | if (exitClause) { |
| 715 | switch (*exitClause) { |
| 716 | case DataClause::acc_copy: |
| 717 | case DataClause::acc_reduction: |
| 718 | case DataClause::acc_copyout: |
| 719 | case DataClause::acc_copyout_zero: |
| 720 | case DataClause::acc_update_host: |
| 721 | case DataClause::acc_update_self: |
| 722 | flags = flags | MapFlags::from; |
| 723 | break; |
| 724 | case DataClause::acc_declare_device_resident: |
| 725 | flags = flags | MapFlags::device_resident; |
| 726 | break; |
| 727 | case DataClause::acc_present: |
| 728 | flags = flags | MapFlags::present; |
| 729 | break; |
| 730 | // `delete` only decrements the dynamic reference counter, so it must not |
| 731 | // request a forced unmap: the device copy has to survive while an |
| 732 | // enclosing region still references it. Only `finalize` zeroes the counter, |
| 733 | // and that is handled from the exit_data op below. |
| 734 | case DataClause::acc_delete: |
| 735 | break; |
| 736 | // An exit that repeats its entry clause only releases the device copy. |
| 737 | case DataClause::acc_create: |
| 738 | case DataClause::acc_create_zero: |
| 739 | case DataClause::acc_copyin: |
| 740 | case DataClause::acc_copyin_readonly: |
| 741 | if (hasCopyOutSibling(entryOp)) |
| 742 | flags = flags | MapFlags::from; |
| 743 | break; |
| 744 | default: |
| 745 | break; |
| 746 | } |
| 747 | if (*exitClause == DataClause::acc_reduction) |
| 748 | flags = flags | MapFlags::reduction; |
| 749 | } |
| 750 | |
| 751 | if (ptrAndObj) |
| 752 | flags = flags | MapFlags::ptr_and_obj; |
| 753 | if (getImplicitFlag(accDataEntryOp: entryOp)) |
| 754 | flags = flags | MapFlags::implicit; |
| 755 | if (entryOp->hasDiscardableAttr(name: getNoDeviceDescriptorAttrName())) |
| 756 | flags = flags | MapFlags::no_device_descriptor; |
| 757 | if (bitEnumContainsAny(bits: getEntryModifiers(entryOp), bit: DataClauseModifier::zero)) |
| 758 | flags = flags | MapFlags::init_zero; |
| 759 | |
| 760 | for (OpOperand &use : entryOp->getResult(idx: 0).getUses()) { |
| 761 | if (auto exitDataOp = dyn_cast<ExitDataOp>(Val: use.getOwner())) { |
| 762 | if (exitDataOp.getFinalize()) |
| 763 | flags = flags | MapFlags::delete_; |
| 764 | } |
| 765 | if (auto updateOp = dyn_cast<UpdateOp>(Val: use.getOwner())) { |
| 766 | if (updateOp.getIfPresent()) |
| 767 | flags = flags | MapFlags::if_present; |
| 768 | } |
| 769 | } |
| 770 | |
| 771 | return flags; |
| 772 | } |
| 773 | |
| 774 | /// Returns the module \p var lives in. |
| 775 | static ModuleOp getEnclosingModule(Value var) { |
| 776 | if (Operation *def = var.getDefiningOp()) |
| 777 | return def->getParentOfType<ModuleOp>(); |
| 778 | if (Region *region = var.getParentRegion()) |
| 779 | if (Operation *parent = region->getParentOp()) |
| 780 | return parent->getParentOfType<ModuleOp>(); |
| 781 | return {}; |
| 782 | } |
| 783 | |
| 784 | int64_t computeMapInfoSizeBytes(Value var, Type varType, DataDescKind descKind, |
| 785 | ValueRange bounds, const DataLayout &dataLayout, |
| 786 | OpenACCSupport *support) { |
| 787 | // Bounds-driven and descriptor-driven maps report size 0: the extents and |
| 788 | // element size already state the size, and restating it here could disagree. |
| 789 | if (!bounds.empty() || descKind != DataDescKind::none) |
| 790 | return 0; |
| 791 | |
| 792 | ModuleOp module = getEnclosingModule(var); |
| 793 | if (!module) |
| 794 | return -1; |
| 795 | |
| 796 | auto tryUtilsSize = [&](Type ty) -> std::optional<int64_t> { |
| 797 | std::optional<TypeSizeAndAlignment> sizeAndAlign = |
| 798 | getTypeSizeAndAlignment(ty, module, dl: dataLayout, support, var); |
| 799 | if (!sizeAndAlign || sizeAndAlign->first.isScalable()) |
| 800 | return std::nullopt; |
| 801 | return static_cast<int64_t>(sizeAndAlign->first.getFixedValue()); |
| 802 | }; |
| 803 | if (std::optional<int64_t> size = tryUtilsSize(varType)) |
| 804 | return *size; |
| 805 | if (std::optional<int64_t> size = tryUtilsSize(var.getType())) |
| 806 | return *size; |
| 807 | |
| 808 | return -1; |
| 809 | } |
| 810 | |
| 811 | int64_t computeMapInfoSizeBytes(Value var, Type varType, DataDescKind descKind, |
| 812 | ValueRange bounds, OpenACCSupport *support) { |
| 813 | ModuleOp module = getEnclosingModule(var); |
| 814 | if (!module) |
| 815 | return -1; |
| 816 | std::optional<DataLayout> dataLayout = getDataLayout(op: module); |
| 817 | if (!dataLayout) |
| 818 | return -1; |
| 819 | return computeMapInfoSizeBytes(var, varType, descKind, bounds, dataLayout: *dataLayout, |
| 820 | support); |
| 821 | } |
| 822 | |
| 823 | void populateSourceExtents(ValueRange bounds, ArrayRef<int64_t> shape, |
| 824 | OpBuilder &builder) { |
| 825 | if (shape.size() != bounds.size()) |
| 826 | return; |
| 827 | for (auto [boundValue, extent] : llvm::zip_equal(t&: bounds, u&: shape)) { |
| 828 | auto bound = boundValue.getDefiningOp<DataBoundsOp>(); |
| 829 | if (!bound || bound.getSourceExtent() || extent < 0) |
| 830 | continue; |
| 831 | OpBuilder::InsertionGuard guard(builder); |
| 832 | builder.setInsertionPoint(bound); |
| 833 | Value sourceExtent = |
| 834 | arith::ConstantIndexOp::create(builder, location: bound.getLoc(), value: extent); |
| 835 | bound.getSourceExtentMutable().assign(value: sourceExtent); |
| 836 | } |
| 837 | } |
| 838 | |
| 839 | } // namespace acc |
| 840 | } // namespace mlir |
| 841 | |