| 1 | //===- OpenACCUtils.cpp ---------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/OpenACC/OpenACCUtils.h" |
| 10 | |
| 11 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 12 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 13 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 14 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 15 | #include "mlir/IR/BuiltinOps.h" |
| 16 | #include "mlir/IR/BuiltinTypes.h" |
| 17 | #include "mlir/IR/Dominance.h" |
| 18 | #include "mlir/IR/SymbolTable.h" |
| 19 | #include "mlir/Interfaces/FunctionInterfaces.h" |
| 20 | #include "mlir/Interfaces/ViewLikeInterface.h" |
| 21 | #include "llvm/ADT/STLExtras.h" |
| 22 | #include "llvm/ADT/SetVector.h" |
| 23 | #include "llvm/ADT/TypeSwitch.h" |
| 24 | #include "llvm/IR/Intrinsics.h" |
| 25 | #include "llvm/Support/Casting.h" |
| 26 | |
| 27 | mlir::Operation *mlir::acc::getEnclosingComputeOp(mlir::Region ®ion) { |
| 28 | return region |
| 29 | .getParentOfType<ACC_COMPUTE_CONSTRUCT_OPS, mlir::acc::ComputeRegionOp>(); |
| 30 | } |
| 31 | |
| 32 | mlir::Value mlir::acc::getACCOperandForBlockArg(mlir::Value v) { |
| 33 | auto barg = mlir::dyn_cast<mlir::BlockArgument>(Val&: v); |
| 34 | if (!barg) |
| 35 | return nullptr; |
| 36 | |
| 37 | mlir::Block *block = barg.getOwner(); |
| 38 | auto computeReg = |
| 39 | mlir::dyn_cast<mlir::acc::ComputeRegionOp>(Val: block->getParentOp()); |
| 40 | if (!computeReg) |
| 41 | return nullptr; |
| 42 | assert(block == computeReg.getBody() && |
| 43 | "block must be the body of acc.compute_region" ); |
| 44 | return computeReg.getOperand(blockArg: barg); |
| 45 | } |
| 46 | |
| 47 | mlir::BlockArgument |
| 48 | mlir::acc::getACCBlockArgForOperand(mlir::OpOperand &operand) { |
| 49 | auto computeReg = |
| 50 | mlir::dyn_cast<mlir::acc::ComputeRegionOp>(Val: operand.getOwner()); |
| 51 | if (!computeReg) |
| 52 | return nullptr; |
| 53 | // `launch` then `ins` lead the operand list and map to the body arguments in |
| 54 | // order. |
| 55 | unsigned idx = operand.getOperandNumber(); |
| 56 | if (idx >= |
| 57 | computeReg.getLaunchArgs().size() + computeReg.getInputArgs().size()) |
| 58 | return nullptr; |
| 59 | return computeReg.getBody()->getArgument(i: idx); |
| 60 | } |
| 61 | |
| 62 | mlir::Operation *mlir::acc::getACCDataClauseOpForBlockArg(mlir::Value v) { |
| 63 | mlir::Value orig = getACCOperandForBlockArg(v); |
| 64 | if (!orig) |
| 65 | return nullptr; |
| 66 | mlir::Operation *def = orig.getDefiningOp(); |
| 67 | return mlir::isa_and_nonnull<ACC_DATA_ENTRY_OPS>(Val: def) ? def : nullptr; |
| 68 | } |
| 69 | |
| 70 | template <typename OpTy> |
| 71 | static bool isOnlyUsedByOpClauses(mlir::Value val, mlir::Region ®ion) { |
| 72 | auto checkIfUsedOnlyByOpInside = [&](mlir::Operation *user) { |
| 73 | // For any users which are not in the current acc region, we can ignore. |
| 74 | // Return true so that it can be used in a `all_of` check. |
| 75 | if (!region.isAncestor(other: user->getParentRegion())) |
| 76 | return true; |
| 77 | return mlir::isa<OpTy>(user); |
| 78 | }; |
| 79 | |
| 80 | return llvm::all_of(val.getUsers(), checkIfUsedOnlyByOpInside); |
| 81 | } |
| 82 | |
| 83 | bool mlir::acc::isOnlyUsedByPrivateClauses(mlir::Value val, |
| 84 | mlir::Region ®ion) { |
| 85 | return isOnlyUsedByOpClauses<mlir::acc::PrivateOp>(val, region); |
| 86 | } |
| 87 | |
| 88 | bool mlir::acc::isOnlyUsedByReductionClauses(mlir::Value val, |
| 89 | mlir::Region ®ion) { |
| 90 | return isOnlyUsedByOpClauses<mlir::acc::ReductionOp>(val, region); |
| 91 | } |
| 92 | |
| 93 | std::optional<mlir::acc::ClauseDefaultValue> |
| 94 | mlir::acc::getDefaultAttr(Operation *op) { |
| 95 | std::optional<mlir::acc::ClauseDefaultValue> defaultAttr; |
| 96 | Operation *currOp = op; |
| 97 | |
| 98 | // Iterate outwards until a default clause is found (since OpenACC |
| 99 | // specification notes that a visible default clause is the nearest default |
| 100 | // clause appearing on the compute construct or a lexically containing data |
| 101 | // construct. |
| 102 | while (!defaultAttr.has_value() && currOp) { |
| 103 | defaultAttr = |
| 104 | llvm::TypeSwitch<mlir::Operation *, |
| 105 | std::optional<mlir::acc::ClauseDefaultValue>>(currOp) |
| 106 | .Case<ACC_COMPUTE_CONSTRUCT_OPS, mlir::acc::DataOp>( |
| 107 | caseFn: [&](auto op) { return op.getDefaultAttr(); }) |
| 108 | .Default(defaultFn: [&](Operation *) { return std::nullopt; }); |
| 109 | currOp = currOp->getParentOp(); |
| 110 | } |
| 111 | |
| 112 | return defaultAttr; |
| 113 | } |
| 114 | |
| 115 | mlir::acc::VariableTypeCategory mlir::acc::getTypeCategory(mlir::Value var) { |
| 116 | mlir::acc::VariableTypeCategory typeCategory = |
| 117 | mlir::acc::VariableTypeCategory::uncategorized; |
| 118 | if (auto mappableTy = dyn_cast<mlir::acc::MappableType>(Val: var.getType())) |
| 119 | typeCategory = mappableTy.getTypeCategory(var); |
| 120 | else if (auto pointerLikeTy = |
| 121 | dyn_cast<mlir::acc::PointerLikeType>(Val: var.getType())) |
| 122 | typeCategory = pointerLikeTy.getPointeeTypeCategory( |
| 123 | varPtr: cast<TypedValue<mlir::acc::PointerLikeType>>(Val&: var), |
| 124 | varType: pointerLikeTy.getElementType()); |
| 125 | return typeCategory; |
| 126 | } |
| 127 | |
| 128 | llvm::StringLiteral mlir::acc::getVarNamePlaceholder() { |
| 129 | return llvm::StringLiteral("<acc.varname.placeholder>" ); |
| 130 | } |
| 131 | |
| 132 | std::string mlir::acc::getVariableName(mlir::Value v) { |
| 133 | Value current = v; |
| 134 | |
| 135 | // Walk through view operations until a name is found or can't go further |
| 136 | while (Operation *definingOp = current.getDefiningOp()) { |
| 137 | // For integer constants, return their value as a string. |
| 138 | if (std::optional<int64_t> constVal = getConstantIntValue(ofr: current)) |
| 139 | return std::to_string(val: *constVal); |
| 140 | |
| 141 | // Check for `acc.var_name` attribute |
| 142 | if (auto varNameAttr = definingOp->getDiscardableAttrOfType<VarNameAttr>( |
| 143 | name: getVarNameAttrName())) |
| 144 | return varNameAttr.getName().str(); |
| 145 | |
| 146 | // If it is a data entry operation, get name via getVarName |
| 147 | if (isa<ACC_DATA_ENTRY_OPS, MapInfoOp>(Val: definingOp)) |
| 148 | if (auto name = acc::getVarName(accOp: definingOp)) |
| 149 | return name->str(); |
| 150 | |
| 151 | // A global goes by the symbol it is addressed through. |
| 152 | if (auto addressOf = dyn_cast<AddressOfGlobalOpInterface>(Val: definingOp)) |
| 153 | return addressOf.getSymbol().getLeafReference().str(); |
| 154 | |
| 155 | // If it's a view operation, continue to the source |
| 156 | if (auto viewOp = dyn_cast<ViewLikeOpInterface>(Val: definingOp)) { |
| 157 | current = viewOp.getViewSource(); |
| 158 | continue; |
| 159 | } |
| 160 | |
| 161 | break; |
| 162 | } |
| 163 | |
| 164 | return "" ; |
| 165 | } |
| 166 | |
| 167 | std::string mlir::acc::getRecipeName(mlir::acc::RecipeKind kind, |
| 168 | mlir::Type type) { |
| 169 | assert(kind == mlir::acc::RecipeKind::private_recipe || |
| 170 | kind == mlir::acc::RecipeKind::firstprivate_recipe || |
| 171 | kind == mlir::acc::RecipeKind::reduction_recipe); |
| 172 | if (!llvm::isa<mlir::acc::PointerLikeType, mlir::acc::MappableType>(Val: type)) |
| 173 | return "" ; |
| 174 | |
| 175 | std::string recipeName; |
| 176 | llvm::raw_string_ostream ss(recipeName); |
| 177 | ss << (kind == mlir::acc::RecipeKind::private_recipe ? "privatization_" |
| 178 | : kind == mlir::acc::RecipeKind::firstprivate_recipe |
| 179 | ? "firstprivatization_" |
| 180 | : "reduction_" ); |
| 181 | |
| 182 | // Print the type using its dialect-defined textual format. |
| 183 | type.print(os&: ss); |
| 184 | ss.flush(); |
| 185 | |
| 186 | // Replace invalid characters (anything that's not a letter, number, or |
| 187 | // period) since this needs to be a valid MLIR identifier. |
| 188 | for (char &c : recipeName) { |
| 189 | if (!std::isalnum(static_cast<unsigned char>(c)) && c != '.' && c != '_') { |
| 190 | if (c == '?') |
| 191 | c = 'U'; |
| 192 | else if (c == '*') |
| 193 | c = 'Z'; |
| 194 | else if (c == '(' || c == ')' || c == '[' || c == ']' || c == '{' || |
| 195 | c == '}' || c == '<' || c == '>') |
| 196 | c = '_'; |
| 197 | else |
| 198 | c = 'X'; |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | return recipeName; |
| 203 | } |
| 204 | |
| 205 | mlir::Value mlir::acc::getBaseEntity(mlir::Value val) { |
| 206 | if (auto partialEntityAccessOp = |
| 207 | val.getDefiningOp<PartialEntityAccessOpInterface>()) { |
| 208 | if (!partialEntityAccessOp.isCompleteView()) |
| 209 | return partialEntityAccessOp.getBaseEntity(); |
| 210 | } |
| 211 | |
| 212 | return val; |
| 213 | } |
| 214 | |
| 215 | /// Look up `symbol` in the `gpu.module`s of the enclosing module. A |
| 216 | /// `gpu.module` is its own symbol table, so `lookupNearestSymbolFrom` from a |
| 217 | /// user outside of it does not find definitions placed inside. |
| 218 | static mlir::Operation *lookupSymbolInGPUModules(mlir::Operation *user, |
| 219 | mlir::SymbolRefAttr symbol) { |
| 220 | auto moduleOp = user->getParentOfType<mlir::ModuleOp>(); |
| 221 | if (!moduleOp) |
| 222 | return nullptr; |
| 223 | for (auto gpuModule : moduleOp.getOps<mlir::gpu::GPUModuleOp>()) { |
| 224 | if (mlir::Operation *op = mlir::SymbolTable::lookupSymbolIn( |
| 225 | op: gpuModule, symbol: symbol.getRootReference())) |
| 226 | return op; |
| 227 | } |
| 228 | return nullptr; |
| 229 | } |
| 230 | |
| 231 | bool mlir::acc::isValidSymbolUse(mlir::Operation *user, |
| 232 | mlir::SymbolRefAttr symbol, |
| 233 | mlir::Operation **definingOpPtr) { |
| 234 | // A pass may prepare the device-side definition of a symbol inside a |
| 235 | // `gpu.module` while the use is still a reference from host IR. Such a |
| 236 | // symbol is meant to be used on device, so the use is valid. |
| 237 | if (mlir::Operation *gpuOp = lookupSymbolInGPUModules(user, symbol)) { |
| 238 | if (definingOpPtr) |
| 239 | *definingOpPtr = gpuOp; |
| 240 | return true; |
| 241 | } |
| 242 | |
| 243 | mlir::Operation *definingOp = |
| 244 | mlir::SymbolTable::lookupNearestSymbolFrom(from: user, symbol); |
| 245 | |
| 246 | // If there are no defining ops, we have no way to ensure validity because |
| 247 | // we cannot check for any attributes. |
| 248 | if (!definingOp) |
| 249 | return false; |
| 250 | |
| 251 | if (definingOpPtr) |
| 252 | *definingOpPtr = definingOp; |
| 253 | |
| 254 | // Check if the defining op is a recipe. |
| 255 | // Recipes are valid as they get materialized before being offloaded to |
| 256 | // device. They are only instructions for how to materialize. |
| 257 | if (mlir::isa<mlir::accomp::RecipeInterface>(Val: definingOp)) |
| 258 | return true; |
| 259 | |
| 260 | // Check if the defining op is a global variable that is device data. |
| 261 | // Device data is already resident on the device and does not need mapping. |
| 262 | if (auto globalVar = |
| 263 | mlir::dyn_cast<mlir::acc::GlobalVariableOpInterface>(Val: definingOp)) |
| 264 | if (globalVar.isDeviceAccessible()) |
| 265 | return true; |
| 266 | |
| 267 | // Check if the defining op is a function |
| 268 | if (auto func = |
| 269 | mlir::dyn_cast_if_present<mlir::FunctionOpInterface>(Val: definingOp)) { |
| 270 | // If this symbol is actually an acc routine or a specialized acc routine - |
| 271 | // then it is expected for it to be offloaded - therefore it is valid. |
| 272 | if (func->hasDiscardableAttr(name: mlir::acc::getRoutineInfoAttrName()) || |
| 273 | func->hasDiscardableAttr(name: mlir::acc::getSpecializedRoutineAttrName())) |
| 274 | return true; |
| 275 | |
| 276 | // If this symbol is a call to an LLVM intrinsic, then it is likely valid. |
| 277 | // Check the following: |
| 278 | // 1. The function is private |
| 279 | // 2. The function has no body |
| 280 | // 3. Name starts with "llvm." |
| 281 | // 4. The function's name is a valid LLVM intrinsic name |
| 282 | if (func.getVisibility() == mlir::SymbolTable::Visibility::Private && |
| 283 | func.getFunctionBody().empty() && func.getName().starts_with(Prefix: "llvm." ) && |
| 284 | llvm::Intrinsic::lookupIntrinsicID(Name: func.getName()) != |
| 285 | llvm::Intrinsic::not_intrinsic) |
| 286 | return true; |
| 287 | } |
| 288 | |
| 289 | // A declare attribute is needed for symbol references. |
| 290 | bool hasDeclare = |
| 291 | definingOp->hasDiscardableAttr(name: mlir::acc::getDeclareAttrName()); |
| 292 | return hasDeclare; |
| 293 | } |
| 294 | |
| 295 | /// Finds the global variable that `addrOf` refers to, through `symbolTables` |
| 296 | /// when provided (cached lookup) and by scanning the nearest symbol table |
| 297 | /// otherwise. |
| 298 | static mlir::acc::GlobalVariableOpInterface |
| 299 | lookupGlobalVariable(mlir::Operation *addrOf, mlir::SymbolRefAttr symbol, |
| 300 | mlir::SymbolTableCollection *symbolTables) { |
| 301 | if (symbolTables) |
| 302 | return symbolTables |
| 303 | ->lookupNearestSymbolFrom<mlir::acc::GlobalVariableOpInterface>(from: addrOf, |
| 304 | symbol); |
| 305 | return mlir::SymbolTable::lookupNearestSymbolFrom< |
| 306 | mlir::acc::GlobalVariableOpInterface>(from: addrOf, symbol); |
| 307 | } |
| 308 | |
| 309 | bool mlir::acc::isDeviceAccessibleValue( |
| 310 | mlir::Value val, mlir::SymbolTableCollection *symbolTables) { |
| 311 | // Check if the value is device data via type interfaces. |
| 312 | // Device data is already resident on the device and does not need mapping. |
| 313 | if (auto mappableTy = dyn_cast<mlir::acc::MappableType>(Val: val.getType())) |
| 314 | if (mappableTy.isDeviceAccessible(var: val, symbolTables)) |
| 315 | return true; |
| 316 | |
| 317 | if (auto pointerLikeTy = dyn_cast<mlir::acc::PointerLikeType>(Val: val.getType())) |
| 318 | if (pointerLikeTy.isDeviceAccessible(var: val, symbolTables)) |
| 319 | return true; |
| 320 | |
| 321 | mlir::Operation *defOp = val.getDefiningOp(); |
| 322 | if (!defOp) |
| 323 | return false; |
| 324 | |
| 325 | // `acc.declare` with deviceptr marks data that is already associated with |
| 326 | // the device. |
| 327 | if (auto declareAttr = |
| 328 | defOp->getDiscardableAttrOfType<mlir::acc::DeclareAttr>( |
| 329 | name: mlir::acc::getDeclareAttrName())) |
| 330 | if (declareAttr.getDataClause().getValue() == |
| 331 | mlir::acc::DataClause::acc_deviceptr) |
| 332 | return true; |
| 333 | |
| 334 | // Handle operations that access a partial entity - check if the base entity |
| 335 | // is device data. |
| 336 | if (auto partialAccess = |
| 337 | dyn_cast<mlir::acc::PartialEntityAccessOpInterface>(Val: defOp)) { |
| 338 | if (mlir::Value base = partialAccess.getBaseEntity()) |
| 339 | return isDeviceAccessibleValue(val: base, symbolTables); |
| 340 | } |
| 341 | |
| 342 | // Handle address_of - check if the referenced global is device data. |
| 343 | if (auto addrOfIface = |
| 344 | dyn_cast<mlir::acc::AddressOfGlobalOpInterface>(Val: defOp)) { |
| 345 | auto symbol = addrOfIface.getSymbol(); |
| 346 | if (auto global = lookupGlobalVariable(addrOf: defOp, symbol, symbolTables)) |
| 347 | return global.isDeviceAccessible(); |
| 348 | } |
| 349 | |
| 350 | return false; |
| 351 | } |
| 352 | |
| 353 | bool mlir::acc::isInDeviceMemoryValue( |
| 354 | mlir::Value val, mlir::SymbolTableCollection *symbolTables) { |
| 355 | // In-device-memory data is a subset of device-accessible data: it must be |
| 356 | // accessible and its storage must physically reside in device memory. |
| 357 | if (auto mappableTy = dyn_cast<mlir::acc::MappableType>(Val: val.getType())) |
| 358 | if (mappableTy.isInDeviceMemory(var: val, symbolTables)) |
| 359 | return true; |
| 360 | |
| 361 | if (auto pointerLikeTy = dyn_cast<mlir::acc::PointerLikeType>(Val: val.getType())) |
| 362 | if (pointerLikeTy.isInDeviceMemory(var: val, symbolTables)) |
| 363 | return true; |
| 364 | |
| 365 | mlir::Operation *defOp = val.getDefiningOp(); |
| 366 | if (!defOp) |
| 367 | return false; |
| 368 | |
| 369 | // `acc.declare` with deviceptr marks data whose storage is already in device |
| 370 | // memory. |
| 371 | if (auto declareAttr = |
| 372 | defOp->getDiscardableAttrOfType<mlir::acc::DeclareAttr>( |
| 373 | name: mlir::acc::getDeclareAttrName())) |
| 374 | if (declareAttr.getDataClause().getValue() == |
| 375 | mlir::acc::DataClause::acc_deviceptr) |
| 376 | return true; |
| 377 | |
| 378 | // Handle operations that access a partial entity - check if the base entity |
| 379 | // is in device memory. |
| 380 | if (auto partialAccess = |
| 381 | dyn_cast<mlir::acc::PartialEntityAccessOpInterface>(Val: defOp)) { |
| 382 | if (mlir::Value base = partialAccess.getBaseEntity()) |
| 383 | return isInDeviceMemoryValue(val: base, symbolTables); |
| 384 | } |
| 385 | |
| 386 | // Handle address_of - check if the referenced global is in device memory. |
| 387 | if (auto addrOfIface = |
| 388 | dyn_cast<mlir::acc::AddressOfGlobalOpInterface>(Val: defOp)) { |
| 389 | auto symbol = addrOfIface.getSymbol(); |
| 390 | if (auto global = lookupGlobalVariable(addrOf: defOp, symbol, symbolTables)) |
| 391 | return global.isInDeviceMemory(); |
| 392 | } |
| 393 | |
| 394 | return false; |
| 395 | } |
| 396 | |
| 397 | bool mlir::acc::isValidValueUse(mlir::Value val, mlir::Region ®ion, |
| 398 | mlir::SymbolTableCollection *symbolTables) { |
| 399 | // Types that can be passed by value are legal. |
| 400 | Type type = val.getType(); |
| 401 | if (type.isIntOrIndexOrFloat() || isa<mlir::ComplexType>(Val: type) || |
| 402 | llvm::isa<mlir::VectorType>(Val: type)) |
| 403 | return true; |
| 404 | |
| 405 | // If this is produced by an ACC data entry operation, it is valid. |
| 406 | if (isa_and_nonnull<ACC_DATA_ENTRY_OPS>(Val: val.getDefiningOp())) |
| 407 | return true; |
| 408 | |
| 409 | // If the value is only used by private clauses, it is not a live-in. |
| 410 | if (isOnlyUsedByPrivateClauses(val, region)) |
| 411 | return true; |
| 412 | |
| 413 | // If this is device data, it is valid. |
| 414 | if (isDeviceAccessibleValue(val, symbolTables)) |
| 415 | return true; |
| 416 | |
| 417 | // Arguments of an enclosing acc routine are already on the device. |
| 418 | if (mlir::Operation *parent = region.getParentOp()) { |
| 419 | if (auto func = parent->getParentOfType<mlir::FunctionOpInterface>()) { |
| 420 | if ((mlir::acc::isAccRoutine(op: func) || |
| 421 | mlir::acc::isSpecializedAccRoutine(op: func)) && |
| 422 | llvm::is_contained(Range: func.getArguments(), Element: val)) |
| 423 | return true; |
| 424 | } |
| 425 | } |
| 426 | |
| 427 | return false; |
| 428 | } |
| 429 | |
| 430 | llvm::SmallVector<mlir::Value> |
| 431 | mlir::acc::getDominatingDataClauses(mlir::Operation *computeConstructOp, |
| 432 | mlir::DominanceInfo &domInfo, |
| 433 | mlir::PostDominanceInfo &postDomInfo) { |
| 434 | llvm::SmallSetVector<mlir::Value, 8> dominatingDataClauses; |
| 435 | |
| 436 | llvm::TypeSwitch<mlir::Operation *>(computeConstructOp) |
| 437 | .Case<mlir::acc::ParallelOp, mlir::acc::KernelsOp, mlir::acc::SerialOp>( |
| 438 | caseFn: [&](auto op) { |
| 439 | for (auto dataClause : op.getDataClauseOperands()) { |
| 440 | dominatingDataClauses.insert(dataClause); |
| 441 | } |
| 442 | }) |
| 443 | .Default(defaultFn: [](mlir::Operation *) {}); |
| 444 | |
| 445 | // Collect the data clauses from enclosing data constructs. |
| 446 | mlir::Operation *currParentOp = computeConstructOp->getParentOp(); |
| 447 | while (currParentOp) { |
| 448 | if (mlir::isa<mlir::acc::DataOp>(Val: currParentOp)) { |
| 449 | for (auto dataClause : mlir::dyn_cast<mlir::acc::DataOp>(Val: currParentOp) |
| 450 | .getDataClauseOperands()) { |
| 451 | dominatingDataClauses.insert(X: dataClause); |
| 452 | } |
| 453 | } |
| 454 | currParentOp = currParentOp->getParentOp(); |
| 455 | } |
| 456 | |
| 457 | // Find the enclosing function/subroutine |
| 458 | auto funcOp = |
| 459 | computeConstructOp->getParentOfType<mlir::FunctionOpInterface>(); |
| 460 | if (!funcOp) |
| 461 | return dominatingDataClauses.takeVector(); |
| 462 | |
| 463 | // Walk the function to find `acc.declare_enter`/`acc.declare_exit` pairs that |
| 464 | // dominate and post-dominate the compute construct and add their data |
| 465 | // clauses to the list. |
| 466 | funcOp->walk(callback: [&](mlir::acc::DeclareEnterOp declareEnterOp) { |
| 467 | if (domInfo.dominates(a: declareEnterOp.getOperation(), b: computeConstructOp)) { |
| 468 | // Collect all `acc.declare_exit` ops for this token. |
| 469 | llvm::SmallVector<mlir::acc::DeclareExitOp> exits; |
| 470 | for (auto *user : declareEnterOp.getToken().getUsers()) |
| 471 | if (auto declareExit = mlir::dyn_cast<mlir::acc::DeclareExitOp>(Val: user)) |
| 472 | exits.push_back(Elt: declareExit); |
| 473 | |
| 474 | // Only add clauses if every `acc.declare_exit` op post-dominates the |
| 475 | // compute construct. |
| 476 | if (!exits.empty() && |
| 477 | llvm::all_of(Range&: exits, P: [&](mlir::acc::DeclareExitOp exitOp) { |
| 478 | return postDomInfo.postDominates(a: exitOp, b: computeConstructOp); |
| 479 | })) { |
| 480 | for (auto dataClause : declareEnterOp.getDataClauseOperands()) |
| 481 | dominatingDataClauses.insert(X: dataClause); |
| 482 | } |
| 483 | } |
| 484 | }); |
| 485 | |
| 486 | return dominatingDataClauses.takeVector(); |
| 487 | } |
| 488 | |
| 489 | static mlir::Value getIfCondition(mlir::Operation *op) { |
| 490 | if (auto ifOp = mlir::dyn_cast<mlir::scf::IfOp>(Val: op)) |
| 491 | return ifOp.getCondition(); |
| 492 | |
| 493 | if (!mlir::isa<mlir::RegionBranchOpInterface>(Val: op) || |
| 494 | mlir::isa<mlir::LoopLikeOpInterface>(Val: op)) |
| 495 | return {}; |
| 496 | if (op->getNumOperands() != 1 || op->getNumRegions() != 2 || |
| 497 | !op->getOperand(idx: 0).getType().isSignlessInteger(width: 1)) |
| 498 | return {}; |
| 499 | return op->getOperand(idx: 0); |
| 500 | } |
| 501 | |
| 502 | bool mlir::acc::isInOffTargetBranch(mlir::Operation *op, |
| 503 | llvm::ArrayRef<int64_t> deviceTypes) { |
| 504 | // No device type is known to be true, so no branch can be classified. |
| 505 | if (deviceTypes.empty()) |
| 506 | return false; |
| 507 | |
| 508 | for (mlir::Operation *parent = op->getParentOp(); |
| 509 | parent && |
| 510 | !mlir::isa<ACC_COMPUTE_CONSTRUCT_OPS, mlir::acc::ComputeRegionOp, |
| 511 | mlir::FunctionOpInterface>(Val: parent); |
| 512 | parent = parent->getParentOp()) { |
| 513 | mlir::Value condition = getIfCondition(op: parent); |
| 514 | if (!condition) |
| 515 | continue; |
| 516 | |
| 517 | // The condition must be the `acc.on_device` result itself. |
| 518 | auto onDeviceOp = condition.getDefiningOp<mlir::acc::OnDeviceOp>(); |
| 519 | if (!onDeviceOp) |
| 520 | continue; |
| 521 | |
| 522 | std::optional<int64_t> deviceTypeValue = |
| 523 | mlir::getConstantIntValue(ofr: onDeviceOp.getDeviceType()); |
| 524 | if (!deviceTypeValue) |
| 525 | continue; |
| 526 | |
| 527 | bool onTarget = llvm::is_contained(Range&: deviceTypes, Element: *deviceTypeValue); |
| 528 | bool inThen = parent->getRegion(index: 0).isAncestor(other: op->getParentRegion()); |
| 529 | bool inElse = parent->getRegion(index: 1).isAncestor(other: op->getParentRegion()); |
| 530 | // Off the target: the then of a device type outside `deviceTypes`, or |
| 531 | // the else of a device type in `deviceTypes`. |
| 532 | if ((!onTarget && inThen) || (onTarget && inElse)) |
| 533 | return true; |
| 534 | } |
| 535 | return false; |
| 536 | } |
| 537 | |
| 538 | mlir::remark::detail::InFlightRemark |
| 539 | mlir::acc::(mlir::Operation *op, |
| 540 | const std::function<std::string()> &messageFn, |
| 541 | llvm::StringRef category) { |
| 542 | using namespace mlir::remark; |
| 543 | mlir::Location loc = op->getLoc(); |
| 544 | auto *engine = loc->getContext()->getRemarkEngine(); |
| 545 | if (!engine) |
| 546 | return remark::detail::InFlightRemark{}; |
| 547 | |
| 548 | llvm::StringRef funcName; |
| 549 | if (auto func = dyn_cast<mlir::FunctionOpInterface>(Val: op)) |
| 550 | funcName = func.getName(); |
| 551 | else if (auto funcOp = op->getParentOfType<mlir::FunctionOpInterface>()) |
| 552 | funcName = funcOp.getName(); |
| 553 | |
| 554 | auto opts = RemarkOpts::name(n: "openacc" ).category(v: category); |
| 555 | if (!funcName.empty()) |
| 556 | opts = opts.function(v: funcName); |
| 557 | |
| 558 | auto = engine->emitOptimizationRemark(loc, opts); |
| 559 | if (remark) |
| 560 | remark << messageFn(); |
| 561 | return remark; |
| 562 | } |
| 563 | |