| 1 | //===- ACCImplicitData.cpp ------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass implements the OpenACC specification for "Variables with |
| 10 | // Implicitly Determined Data Attributes" (OpenACC 3.4 spec, section 2.6.2). |
| 11 | // |
| 12 | // Overview: |
| 13 | // --------- |
| 14 | // The pass automatically generates data clause operations for variables used |
| 15 | // within OpenACC compute constructs (parallel, kernels, serial) that do not |
| 16 | // already have explicit data clauses. The semantics follow these rules: |
| 17 | // |
| 18 | // 1. If there is a default(none) clause visible, no implicit data actions |
| 19 | // apply. |
| 20 | // |
| 21 | // 2. An aggregate variable (arrays, derived types, etc.) will be treated as: |
| 22 | // - In a present clause when default(present) is visible. |
| 23 | // - In a copy clause otherwise. |
| 24 | // |
| 25 | // 3. A scalar variable will be treated as if it appears in: |
| 26 | // - A copy clause if the compute construct is a kernels construct. |
| 27 | // - A firstprivate clause otherwise (parallel, serial). |
| 28 | // |
| 29 | // Requirements: |
| 30 | // ------------- |
| 31 | // To use this pass in a pipeline, the following requirements must be met: |
| 32 | // |
| 33 | // 1. Type Interface Implementation: Variables from the dialect being used |
| 34 | // must implement one or both of the following MLIR interfaces: |
| 35 | // `acc::MappableType` and/or `acc::PointerLikeType` |
| 36 | // |
| 37 | // These interfaces provide the necessary methods for the pass to: |
| 38 | // - Determine variable type categories (scalar vs. aggregate) |
| 39 | // - Generate appropriate bounds information |
| 40 | // - Generate privatization recipes |
| 41 | // |
| 42 | // 2. Operation Interface Implementation: Operations that access partial |
| 43 | // entities or create views should implement the following MLIR |
| 44 | // interfaces: `acc::PartialEntityAccess` and/or |
| 45 | // `mlir::ViewLikeOpInterface` |
| 46 | // |
| 47 | // These interfaces are used for proper data clause ordering, ensuring |
| 48 | // that base entities are mapped before derived entities (e.g., a |
| 49 | // struct is mapped before its fields, an array is mapped before |
| 50 | // subarray views). |
| 51 | // |
| 52 | // 3. Analysis Registration (Optional): If custom behavior is needed for |
| 53 | // variable name extraction or alias analysis, the dialect should |
| 54 | // pre-register the `acc::OpenACCSupport` and `mlir::AliasAnalysis` analyses. |
| 55 | // |
| 56 | // If not registered, default behavior will be used. |
| 57 | // |
| 58 | // Implementation Details: |
| 59 | // ----------------------- |
| 60 | // The pass performs the following operations: |
| 61 | // |
| 62 | // 1. Finds candidate variables which are live-in to the compute region and |
| 63 | // are not already in a data clause or private clause. |
| 64 | // |
| 65 | // 2. Generates both data "entry" and "exit" clause operations that match |
| 66 | // the intended action depending on variable type: |
| 67 | // - copy -> acc.copyin (entry) + acc.copyout (exit) |
| 68 | // - present -> acc.present (entry) + acc.delete (exit) |
| 69 | // - firstprivate -> acc.firstprivate (entry only, no exit) |
| 70 | // |
| 71 | // 3. Ensures that default clause is taken into consideration by looking |
| 72 | // through current construct and parent constructs to find the "visible |
| 73 | // default clause". |
| 74 | // |
| 75 | // 4. Fixes up SSA value links so that uses in the acc region reference the |
| 76 | // result of the newly created data clause operations. |
| 77 | // |
| 78 | // 5. When generating implicit data clause operations, it also adds variable |
| 79 | // name information and marks them with the implicit flag. |
| 80 | // |
| 81 | // 6. Recipes are generated by calling the appropriate entrypoints in the |
| 82 | // MappableType and PointerLikeType interfaces. |
| 83 | // |
| 84 | // 7. AliasAnalysis is used to determine if a variable is already covered by |
| 85 | // an existing data clause (e.g., an interior pointer covered by its parent). |
| 86 | // |
| 87 | // Examples: |
| 88 | // --------- |
| 89 | // |
| 90 | // Example 1: Scalar in parallel construct (implicit firstprivate) |
| 91 | // |
| 92 | // Before: |
| 93 | // func.func @test() { |
| 94 | // %scalar = memref.alloca() {acc.var_name = "x"} : memref<f32> |
| 95 | // acc.parallel { |
| 96 | // %val = memref.load %scalar[] : memref<f32> |
| 97 | // acc.yield |
| 98 | // } |
| 99 | // } |
| 100 | // |
| 101 | // After: |
| 102 | // func.func @test() { |
| 103 | // %scalar = memref.alloca() {acc.var_name = "x"} : memref<f32> |
| 104 | // %firstpriv = acc.firstprivate varPtr(%scalar : memref<f32>) |
| 105 | // -> memref<f32> {implicit = true, name = "x"} |
| 106 | // acc.parallel firstprivate(@recipe -> %firstpriv : memref<f32>) { |
| 107 | // %val = memref.load %firstpriv[] : memref<f32> |
| 108 | // acc.yield |
| 109 | // } |
| 110 | // } |
| 111 | // |
| 112 | // Example 2: Scalar in kernels construct (implicit copy) |
| 113 | // |
| 114 | // Before: |
| 115 | // func.func @test() { |
| 116 | // %scalar = memref.alloca() {acc.var_name = "n"} : memref<i32> |
| 117 | // acc.kernels { |
| 118 | // %val = memref.load %scalar[] : memref<i32> |
| 119 | // acc.terminator |
| 120 | // } |
| 121 | // } |
| 122 | // |
| 123 | // After: |
| 124 | // func.func @test() { |
| 125 | // %scalar = memref.alloca() {acc.var_name = "n"} : memref<i32> |
| 126 | // %copyin = acc.copyin varPtr(%scalar : memref<i32>) -> memref<i32> |
| 127 | // {dataClause = #acc.data_clause<acc_copy>, |
| 128 | // implicit = true, name = "n"} |
| 129 | // acc.kernels dataOperands(%copyin : memref<i32>) { |
| 130 | // %val = memref.load %copyin[] : memref<i32> |
| 131 | // acc.terminator |
| 132 | // } |
| 133 | // acc.copyout accPtr(%copyin : memref<i32>) |
| 134 | // to varPtr(%scalar : memref<i32>) |
| 135 | // {dataClause = #acc.data_clause<acc_copy>, |
| 136 | // implicit = true, name = "n"} |
| 137 | // } |
| 138 | // |
| 139 | // Example 3: Array (aggregate) in parallel (implicit copy) |
| 140 | // |
| 141 | // Before: |
| 142 | // func.func @test() { |
| 143 | // %array = memref.alloca() {acc.var_name = "arr"} : memref<100xf32> |
| 144 | // acc.parallel { |
| 145 | // %c0 = arith.constant 0 : index |
| 146 | // %val = memref.load %array[%c0] : memref<100xf32> |
| 147 | // acc.yield |
| 148 | // } |
| 149 | // } |
| 150 | // |
| 151 | // After: |
| 152 | // func.func @test() { |
| 153 | // %array = memref.alloca() {acc.var_name = "arr"} : memref<100xf32> |
| 154 | // %copyin = acc.copyin varPtr(%array : memref<100xf32>) |
| 155 | // -> memref<100xf32> |
| 156 | // {dataClause = #acc.data_clause<acc_copy>, |
| 157 | // implicit = true, name = "arr"} |
| 158 | // acc.parallel dataOperands(%copyin : memref<100xf32>) { |
| 159 | // %c0 = arith.constant 0 : index |
| 160 | // %val = memref.load %copyin[%c0] : memref<100xf32> |
| 161 | // acc.yield |
| 162 | // } |
| 163 | // acc.copyout accPtr(%copyin : memref<100xf32>) |
| 164 | // to varPtr(%array : memref<100xf32>) |
| 165 | // {dataClause = #acc.data_clause<acc_copy>, |
| 166 | // implicit = true, name = "arr"} |
| 167 | // } |
| 168 | // |
| 169 | // Example 4: Array with default(present) |
| 170 | // |
| 171 | // Before: |
| 172 | // func.func @test() { |
| 173 | // %array = memref.alloca() {acc.var_name = "arr"} : memref<100xf32> |
| 174 | // acc.parallel { |
| 175 | // %c0 = arith.constant 0 : index |
| 176 | // %val = memref.load %array[%c0] : memref<100xf32> |
| 177 | // acc.yield |
| 178 | // } attributes {defaultAttr = #acc.defaultvalue<present>} |
| 179 | // } |
| 180 | // |
| 181 | // After: |
| 182 | // func.func @test() { |
| 183 | // %array = memref.alloca() {acc.var_name = "arr"} : memref<100xf32> |
| 184 | // %present = acc.present varPtr(%array : memref<100xf32>) |
| 185 | // -> memref<100xf32> |
| 186 | // {implicit = true, name = "arr"} |
| 187 | // acc.parallel dataOperands(%present : memref<100xf32>) |
| 188 | // attributes {defaultAttr = #acc.defaultvalue<present>} { |
| 189 | // %c0 = arith.constant 0 : index |
| 190 | // %val = memref.load %present[%c0] : memref<100xf32> |
| 191 | // acc.yield |
| 192 | // } |
| 193 | // acc.delete accPtr(%present : memref<100xf32>) |
| 194 | // {dataClause = #acc.data_clause<acc_present>, |
| 195 | // implicit = true, name = "arr"} |
| 196 | // } |
| 197 | // |
| 198 | //===----------------------------------------------------------------------===// |
| 199 | |
| 200 | #include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h" |
| 201 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h" |
| 202 | #include "llvm/ADT/SmallVectorExtras.h" |
| 203 | |
| 204 | #include "mlir/Analysis/AliasAnalysis.h" |
| 205 | #include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" |
| 206 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 207 | #include "mlir/Dialect/OpenACC/OpenACCUtils.h" |
| 208 | #include "mlir/IR/Builders.h" |
| 209 | #include "mlir/IR/BuiltinOps.h" |
| 210 | #include "mlir/IR/Dominance.h" |
| 211 | #include "mlir/IR/Operation.h" |
| 212 | #include "mlir/IR/Value.h" |
| 213 | #include "mlir/Interfaces/FunctionInterfaces.h" |
| 214 | #include "mlir/Interfaces/ViewLikeInterface.h" |
| 215 | #include "mlir/Transforms/RegionUtils.h" |
| 216 | #include "llvm/ADT/STLExtras.h" |
| 217 | #include "llvm/ADT/SmallVector.h" |
| 218 | #include "llvm/ADT/TypeSwitch.h" |
| 219 | #include "llvm/Support/ErrorHandling.h" |
| 220 | #include "llvm/Support/raw_ostream.h" |
| 221 | #include <string> |
| 222 | #include <type_traits> |
| 223 | |
| 224 | namespace mlir { |
| 225 | namespace acc { |
| 226 | #define GEN_PASS_DEF_ACCIMPLICITDATA |
| 227 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" |
| 228 | } // namespace acc |
| 229 | } // namespace mlir |
| 230 | |
| 231 | #define DEBUG_TYPE "acc-implicit-data" |
| 232 | |
| 233 | using namespace mlir; |
| 234 | |
| 235 | namespace { |
| 236 | class ACCImplicitData : public acc::impl::ACCImplicitDataBase<ACCImplicitData> { |
| 237 | public: |
| 238 | using acc::impl::ACCImplicitDataBase<ACCImplicitData>::ACCImplicitDataBase; |
| 239 | |
| 240 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 241 | acc::impl::ACCImplicitDataBase<ACCImplicitData>::getDependentDialects( |
| 242 | registry); |
| 243 | acc::registerAllExtensions(registry); |
| 244 | } |
| 245 | |
| 246 | void runOnOperation() override; |
| 247 | |
| 248 | private: |
| 249 | /// Looks through the `dominatingDataClauses` to find the original data clause |
| 250 | /// op for an alias. Returns nullptr if no original data clause op is found. |
| 251 | template <typename OpT> |
| 252 | Operation *getOriginalDataClauseOpForAlias( |
| 253 | Value var, OpBuilder &builder, OpT computeConstructOp, |
| 254 | const SmallVector<Value> &dominatingDataClauses); |
| 255 | |
| 256 | /// Generates the appropriate `acc.copyin`, `acc.present`,`acc.firstprivate`, |
| 257 | /// etc. data clause op for a candidate variable. |
| 258 | template <typename OpT> |
| 259 | Operation *generateDataClauseOpForCandidate( |
| 260 | Value var, ModuleOp &module, OpBuilder &builder, OpT computeConstructOp, |
| 261 | const SmallVector<Value> &dominatingDataClauses, |
| 262 | const std::optional<acc::ClauseDefaultValue> &defaultClause); |
| 263 | |
| 264 | /// Generates the implicit data ops for a compute construct. |
| 265 | template <typename OpT> |
| 266 | void |
| 267 | generateImplicitDataOps(ModuleOp &module, OpT computeConstructOp, |
| 268 | std::optional<acc::ClauseDefaultValue> &defaultClause, |
| 269 | acc::OpenACCSupport &accSupport); |
| 270 | |
| 271 | /// Generates a private recipe for a variable. |
| 272 | acc::PrivateRecipeOp generatePrivateRecipe(ModuleOp &module, Value var, |
| 273 | Location loc, OpBuilder &builder, |
| 274 | acc::OpenACCSupport &accSupport); |
| 275 | |
| 276 | /// Generates a firstprivate recipe for a variable. |
| 277 | acc::FirstprivateRecipeOp |
| 278 | generateFirstprivateRecipe(ModuleOp &module, Value var, Location loc, |
| 279 | OpBuilder &builder, |
| 280 | acc::OpenACCSupport &accSupport); |
| 281 | |
| 282 | /// Generates recipes for a list of variables. |
| 283 | void generateRecipes(ModuleOp &module, OpBuilder &builder, |
| 284 | Operation *computeConstructOp, |
| 285 | const SmallVector<Value> &newOperands); |
| 286 | |
| 287 | /// Caches the symbol tables used to resolve the globals referenced by the |
| 288 | /// candidate variables. Resolving each reference without it scans every |
| 289 | /// top-level op of the module. The globals are not changed by this pass, so |
| 290 | /// the cached tables stay valid. It points to a collection local to |
| 291 | /// `runOnOperation` (the collection itself is not copyable, and passes |
| 292 | /// must be) and is only set while that runs. |
| 293 | SymbolTableCollection *symbolTables = nullptr; |
| 294 | }; |
| 295 | |
| 296 | /// Determines if a variable is a candidate for implicit data mapping. |
| 297 | /// Returns true if the variable is a candidate, false otherwise. |
| 298 | static bool isCandidateForImplicitData(Value val, Region &accRegion, |
| 299 | acc::OpenACCSupport &accSupport, |
| 300 | SymbolTableCollection *symbolTables) { |
| 301 | // Ensure the variable is an allowed type for data clause. |
| 302 | if (!acc::isPointerLikeType(type: val.getType()) && |
| 303 | !acc::isMappableType(type: val.getType())) |
| 304 | return false; |
| 305 | |
| 306 | // If this is already coming from a data clause, we do not need to generate |
| 307 | // another. |
| 308 | if (isa_and_nonnull<ACC_DATA_ENTRY_OPS>(Val: val.getDefiningOp())) |
| 309 | return false; |
| 310 | |
| 311 | // Device data is a candidate - it will get a deviceptr clause. |
| 312 | if (acc::isDeviceAccessibleValue(val, symbolTables)) |
| 313 | return true; |
| 314 | |
| 315 | // If it is otherwise valid, skip it. |
| 316 | if (accSupport.isValidValueUse(v: val, region&: accRegion, symbolTables)) |
| 317 | return false; |
| 318 | |
| 319 | return true; |
| 320 | } |
| 321 | |
| 322 | template <typename OpT> |
| 323 | Operation *ACCImplicitData::getOriginalDataClauseOpForAlias( |
| 324 | Value var, OpBuilder &builder, OpT computeConstructOp, |
| 325 | const SmallVector<Value> &dominatingDataClauses) { |
| 326 | auto &aliasAnalysis = this->getAnalysis<AliasAnalysis>(); |
| 327 | for (auto dataClause : dominatingDataClauses) { |
| 328 | if (auto *dataClauseOp = dataClause.getDefiningOp()) { |
| 329 | // Only accept clauses that guarantee that the alias is present. |
| 330 | if (isa<acc::CopyinOp, acc::CreateOp, acc::PresentOp, acc::NoCreateOp, |
| 331 | acc::DevicePtrOp>(Val: dataClauseOp)) |
| 332 | if (aliasAnalysis.alias(lhs: acc::getVar(accDataClauseOp: dataClauseOp), rhs: var).isMust()) { |
| 333 | LLVM_DEBUG(llvm::dbgs() |
| 334 | << "Using existing data clause:\n\t" << *dataClauseOp |
| 335 | << "\n\tas reference when processing var:\n\t" << var |
| 336 | << "\n" ;); |
| 337 | return dataClauseOp; |
| 338 | } |
| 339 | } |
| 340 | } |
| 341 | return nullptr; |
| 342 | } |
| 343 | |
| 344 | // Generates bounds for variables that have unknown dimensions |
| 345 | static void fillInBoundsForUnknownDimensions(Operation *dataClauseOp, |
| 346 | OpBuilder &builder) { |
| 347 | |
| 348 | if (!acc::getBounds(accDataClauseOp: dataClauseOp).empty()) |
| 349 | // If bounds are already present, do not overwrite them. |
| 350 | return; |
| 351 | |
| 352 | // For types that have unknown dimensions, attempt to generate bounds by |
| 353 | // relying on MappableType being able to extract it from the IR. |
| 354 | auto var = acc::getVar(accDataClauseOp: dataClauseOp); |
| 355 | auto type = var.getType(); |
| 356 | if (auto mappableTy = dyn_cast<acc::MappableType>(Val&: type)) { |
| 357 | if (mappableTy.hasUnknownDimensions()) { |
| 358 | TypeSwitch<Operation *>(dataClauseOp) |
| 359 | .Case<ACC_DATA_ENTRY_OPS, ACC_DATA_EXIT_OPS>(caseFn: [&](auto dataClauseOp) { |
| 360 | if (std::is_same_v<decltype(dataClauseOp), acc::DevicePtrOp>) |
| 361 | return; |
| 362 | OpBuilder::InsertionGuard guard(builder); |
| 363 | builder.setInsertionPoint(dataClauseOp); |
| 364 | auto bounds = mappableTy.generateAccBounds(var, builder); |
| 365 | if (!bounds.empty()) |
| 366 | dataClauseOp.getBoundsMutable().assign(bounds); |
| 367 | }); |
| 368 | } |
| 369 | } |
| 370 | } |
| 371 | |
| 372 | acc::PrivateRecipeOp |
| 373 | ACCImplicitData::generatePrivateRecipe(ModuleOp &module, Value var, |
| 374 | Location loc, OpBuilder &builder, |
| 375 | acc::OpenACCSupport &accSupport) { |
| 376 | auto type = var.getType(); |
| 377 | std::string recipeName = |
| 378 | accSupport.getRecipeName(kind: acc::RecipeKind::private_recipe, type, var); |
| 379 | |
| 380 | // Check if recipe already exists |
| 381 | auto existingRecipe = module.lookupSymbol<acc::PrivateRecipeOp>(name: recipeName); |
| 382 | if (existingRecipe) |
| 383 | return existingRecipe; |
| 384 | |
| 385 | // Set insertion point to module body in a scoped way |
| 386 | OpBuilder::InsertionGuard guard(builder); |
| 387 | builder.setInsertionPointToStart(module.getBody()); |
| 388 | |
| 389 | auto recipe = |
| 390 | acc::PrivateRecipeOp::createAndPopulate(builder, loc, recipeName, hostVar: var); |
| 391 | if (!recipe.has_value()) |
| 392 | return accSupport.emitNYI(loc, message: "implicit private" ), nullptr; |
| 393 | return recipe.value(); |
| 394 | } |
| 395 | |
| 396 | acc::FirstprivateRecipeOp |
| 397 | ACCImplicitData::generateFirstprivateRecipe(ModuleOp &module, Value var, |
| 398 | Location loc, OpBuilder &builder, |
| 399 | acc::OpenACCSupport &accSupport) { |
| 400 | auto type = var.getType(); |
| 401 | std::string recipeName = |
| 402 | accSupport.getRecipeName(kind: acc::RecipeKind::firstprivate_recipe, type, var); |
| 403 | |
| 404 | // Check if recipe already exists |
| 405 | auto existingRecipe = |
| 406 | module.lookupSymbol<acc::FirstprivateRecipeOp>(name: recipeName); |
| 407 | if (existingRecipe) |
| 408 | return existingRecipe; |
| 409 | |
| 410 | // Set insertion point to module body in a scoped way |
| 411 | OpBuilder::InsertionGuard guard(builder); |
| 412 | builder.setInsertionPointToStart(module.getBody()); |
| 413 | |
| 414 | auto recipe = acc::FirstprivateRecipeOp::createAndPopulate(builder, loc, |
| 415 | recipeName, hostVar: var); |
| 416 | if (!recipe.has_value()) |
| 417 | return accSupport.emitNYI(loc, message: "implicit firstprivate" ), nullptr; |
| 418 | return recipe.value(); |
| 419 | } |
| 420 | |
| 421 | void ACCImplicitData::generateRecipes(ModuleOp &module, OpBuilder &builder, |
| 422 | Operation *computeConstructOp, |
| 423 | const SmallVector<Value> &newOperands) { |
| 424 | auto &accSupport = this->getAnalysis<acc::OpenACCSupport>(); |
| 425 | for (auto var : newOperands) { |
| 426 | auto loc{var.getLoc()}; |
| 427 | if (auto privateOp = var.getDefiningOp<acc::PrivateOp>()) { |
| 428 | auto recipe = generatePrivateRecipe( |
| 429 | module, var: acc::getVar(accDataClauseOp: var.getDefiningOp()), loc, builder, accSupport); |
| 430 | if (recipe) |
| 431 | privateOp.setRecipeAttr( |
| 432 | SymbolRefAttr::get(ctx: module->getContext(), value: recipe.getSymName())); |
| 433 | } else if (auto firstprivateOp = var.getDefiningOp<acc::FirstprivateOp>()) { |
| 434 | auto recipe = generateFirstprivateRecipe( |
| 435 | module, var: acc::getVar(accDataClauseOp: var.getDefiningOp()), loc, builder, accSupport); |
| 436 | if (recipe) |
| 437 | firstprivateOp.setRecipeAttr(SymbolRefAttr::get( |
| 438 | ctx: module->getContext(), value: recipe.getSymName().str())); |
| 439 | } else { |
| 440 | accSupport.emitNYI(loc: var.getLoc(), message: "implicit reduction" ); |
| 441 | } |
| 442 | } |
| 443 | } |
| 444 | |
| 445 | // Generates the data entry data op clause so that it adheres to OpenACC |
| 446 | // rules as follows (line numbers and specification from OpenACC 3.4): |
| 447 | // 1388 An aggregate variable will be treated as if it appears either: |
| 448 | // 1389 - In a present clause if there is a default(present) clause visible at |
| 449 | // the compute construct. |
| 450 | // 1391 - In a copy clause otherwise. |
| 451 | // 1392 A scalar variable will be treated as if it appears either: |
| 452 | // 1393 - In a copy clause if the compute construct is a kernels construct. |
| 453 | // 1394 - In a firstprivate clause otherwise. |
| 454 | template <typename OpT> |
| 455 | Operation *ACCImplicitData::generateDataClauseOpForCandidate( |
| 456 | Value var, ModuleOp &module, OpBuilder &builder, OpT computeConstructOp, |
| 457 | const SmallVector<Value> &dominatingDataClauses, |
| 458 | const std::optional<acc::ClauseDefaultValue> &defaultClause) { |
| 459 | auto &accSupport = this->getAnalysis<acc::OpenACCSupport>(); |
| 460 | acc::VariableTypeCategory typeCategory = |
| 461 | acc::VariableTypeCategory::uncategorized; |
| 462 | if (auto mappableTy = dyn_cast<acc::MappableType>(Val: var.getType())) { |
| 463 | typeCategory = mappableTy.getTypeCategory(var); |
| 464 | } else if (auto pointerLikeTy = |
| 465 | dyn_cast<acc::PointerLikeType>(Val: var.getType())) { |
| 466 | typeCategory = pointerLikeTy.getPointeeTypeCategory( |
| 467 | varPtr: cast<TypedValue<acc::PointerLikeType>>(Val&: var), |
| 468 | varType: pointerLikeTy.getElementType()); |
| 469 | } |
| 470 | |
| 471 | bool isScalar = |
| 472 | acc::bitEnumContainsAny(bits: typeCategory, bit: acc::VariableTypeCategory::scalar); |
| 473 | bool isAnyAggregate = acc::bitEnumContainsAny( |
| 474 | bits: typeCategory, bit: acc::VariableTypeCategory::aggregate); |
| 475 | Location loc = computeConstructOp->getLoc(); |
| 476 | |
| 477 | // `deviceptr` asserts the value's storage is already in device memory; no |
| 478 | // runtime mapping or attach is performed. Storage that is device-accessible |
| 479 | // but physically shared with the host may migrate on demand, so it is not |
| 480 | // guaranteed to be in device memory and must still be mapped rather than |
| 481 | // treated as deviceptr. |
| 482 | if (acc::isInDeviceMemoryValue(val: var, symbolTables)) { |
| 483 | // If the variable is in device memory, use deviceptr clause. |
| 484 | LLVM_DEBUG(llvm::dbgs() << "Using deviceptr clause because variable is " |
| 485 | "in device memory\n" ); |
| 486 | return acc::DevicePtrOp::create(builder, location: loc, var, |
| 487 | /*structured=*/true, /*implicit=*/true, |
| 488 | name: accSupport.getVariableName(v: var)); |
| 489 | } |
| 490 | |
| 491 | Operation *op = nullptr; |
| 492 | op = getOriginalDataClauseOpForAlias(var, builder, computeConstructOp, |
| 493 | dominatingDataClauses); |
| 494 | if (op) { |
| 495 | if (isa<acc::NoCreateOp>(Val: op)) |
| 496 | return acc::NoCreateOp::create(builder, location: loc, var, |
| 497 | /*structured=*/true, /*implicit=*/true, |
| 498 | name: accSupport.getVariableName(v: var), |
| 499 | bounds: acc::getBounds(accDataClauseOp: op)); |
| 500 | |
| 501 | if (isa<acc::DevicePtrOp>(Val: op)) |
| 502 | return acc::DevicePtrOp::create(builder, location: loc, var, |
| 503 | /*structured=*/true, /*implicit=*/true, |
| 504 | name: accSupport.getVariableName(v: var), |
| 505 | bounds: acc::getBounds(accDataClauseOp: op)); |
| 506 | |
| 507 | // The original data clause op is a PresentOp, CopyinOp, or CreateOp, |
| 508 | // hence guaranteed to be present. |
| 509 | return acc::PresentOp::create(builder, location: loc, var, |
| 510 | /*structured=*/true, /*implicit=*/true, |
| 511 | name: accSupport.getVariableName(v: var), |
| 512 | bounds: acc::getBounds(accDataClauseOp: op)); |
| 513 | } |
| 514 | |
| 515 | if (isScalar) { |
| 516 | if (enableImplicitReductionCopy && |
| 517 | acc::isOnlyUsedByReductionClauses(val: var, |
| 518 | region&: computeConstructOp->getRegion(0))) { |
| 519 | auto copyinOp = |
| 520 | acc::CopyinOp::create(builder, location: loc, var, |
| 521 | /*structured=*/true, /*implicit=*/true, |
| 522 | name: accSupport.getVariableName(v: var)); |
| 523 | copyinOp.setDataClause(acc::DataClause::acc_reduction); |
| 524 | return copyinOp.getOperation(); |
| 525 | } |
| 526 | if constexpr (std::is_same_v<OpT, acc::KernelsOp>) { |
| 527 | // Scalars are implicit copyin in kernels construct. |
| 528 | auto copyinOp = |
| 529 | acc::CopyinOp::create(builder, location: loc, var, |
| 530 | /*structured=*/true, /*implicit=*/true, |
| 531 | name: accSupport.getVariableName(v: var)); |
| 532 | copyinOp.setDataClause(acc::DataClause::acc_copy); |
| 533 | return copyinOp.getOperation(); |
| 534 | } else { |
| 535 | // Scalars are implicit firstprivate in parallel and serial construct. |
| 536 | return acc::FirstprivateOp::create(builder, location: loc, var, |
| 537 | /*structured=*/true, /*implicit=*/true, |
| 538 | name: accSupport.getVariableName(v: var)); |
| 539 | } |
| 540 | } else if (isAnyAggregate) { |
| 541 | Operation *newDataOp = nullptr; |
| 542 | |
| 543 | // When default(present) is true, the implicit behavior is present. |
| 544 | if (defaultClause.has_value() && |
| 545 | defaultClause.value() == acc::ClauseDefaultValue::Present) { |
| 546 | newDataOp = acc::PresentOp::create(builder, location: loc, var, |
| 547 | /*structured=*/true, /*implicit=*/true, |
| 548 | name: accSupport.getVariableName(v: var)); |
| 549 | newDataOp->setDiscardableAttr(name: acc::getFromDefaultClauseAttrName(), |
| 550 | value: builder.getUnitAttr()); |
| 551 | } else { |
| 552 | auto copyinOp = |
| 553 | acc::CopyinOp::create(builder, location: loc, var, |
| 554 | /*structured=*/true, /*implicit=*/true, |
| 555 | name: accSupport.getVariableName(v: var)); |
| 556 | copyinOp.setDataClause(acc::DataClause::acc_copy); |
| 557 | newDataOp = copyinOp.getOperation(); |
| 558 | } |
| 559 | |
| 560 | return newDataOp; |
| 561 | } else { |
| 562 | // This is not a fatal error - for example when the element type is |
| 563 | // pointer type (aka we have a pointer of pointer), it is potentially a |
| 564 | // deep copy scenario which is not being handled here. |
| 565 | // Other types need to be canonicalized. Thus just log unhandled cases. |
| 566 | LLVM_DEBUG(llvm::dbgs() |
| 567 | << "Unhandled case for implicit data mapping " << var << "\n" ); |
| 568 | } |
| 569 | return nullptr; |
| 570 | } |
| 571 | |
| 572 | // Ensures that result values from the acc data clause ops are used inside the |
| 573 | // acc region. ie: |
| 574 | // acc.kernels { |
| 575 | // use %val |
| 576 | // } |
| 577 | // => |
| 578 | // %dev = acc.dataop %val |
| 579 | // acc.kernels { |
| 580 | // use %dev |
| 581 | // } |
| 582 | static void legalizeValuesInRegion(Region &accRegion, |
| 583 | SmallVector<Value> &newPrivateOperands, |
| 584 | SmallVector<Value> &newDataClauseOperands) { |
| 585 | for (Value dataClause : |
| 586 | llvm::concat<Value>(Ranges&: newDataClauseOperands, Ranges&: newPrivateOperands)) { |
| 587 | Value var = acc::getVar(accDataClauseOp: dataClause.getDefiningOp()); |
| 588 | replaceAllUsesInRegionWith(orig: var, replacement: dataClause, region&: accRegion); |
| 589 | } |
| 590 | } |
| 591 | |
| 592 | // Adds the private operands to the compute construct operation. |
| 593 | template <typename OpT> |
| 594 | static void addNewPrivateOperands(OpT &accOp, |
| 595 | const SmallVector<Value> &privateOperands) { |
| 596 | if (privateOperands.empty()) |
| 597 | return; |
| 598 | |
| 599 | for (auto priv : privateOperands) { |
| 600 | if (isa<acc::PrivateOp>(Val: priv.getDefiningOp())) { |
| 601 | accOp.getPrivateOperandsMutable().append(priv); |
| 602 | } else if (isa<acc::FirstprivateOp>(Val: priv.getDefiningOp())) { |
| 603 | accOp.getFirstprivateOperandsMutable().append(priv); |
| 604 | } else { |
| 605 | llvm_unreachable("unhandled reduction operand" ); |
| 606 | } |
| 607 | } |
| 608 | } |
| 609 | |
| 610 | static Operation *findDataExitOp(Operation *dataEntryOp) { |
| 611 | auto res = acc::getAccVar(accDataClauseOp: dataEntryOp); |
| 612 | for (auto *user : res.getUsers()) |
| 613 | if (isa<ACC_DATA_EXIT_OPS>(Val: user)) |
| 614 | return user; |
| 615 | return nullptr; |
| 616 | } |
| 617 | |
| 618 | // Generates matching data exit operation as described in the acc dialect |
| 619 | // for how data clauses are decomposed: |
| 620 | // https://mlir.llvm.org/docs/Dialects/OpenACCDialect/#operation-categories |
| 621 | // Key ones used here: |
| 622 | // * acc {construct} copy -> acc.copyin (before region) + acc.copyout (after |
| 623 | // region) |
| 624 | // * acc {construct} present -> acc.present (before region) + acc.delete |
| 625 | // (after region) |
| 626 | static void |
| 627 | generateDataExitOperations(OpBuilder &builder, Operation *accOp, |
| 628 | const SmallVector<Value> &newDataClauseOperands, |
| 629 | const SmallVector<Value> &sortedDataClauseOperands) { |
| 630 | builder.setInsertionPointAfter(accOp); |
| 631 | Value lastDataClause = nullptr; |
| 632 | for (auto dataEntry : llvm::reverse(C: sortedDataClauseOperands)) { |
| 633 | if (llvm::find(Range: newDataClauseOperands, Val: dataEntry) == |
| 634 | newDataClauseOperands.end()) { |
| 635 | // If this is not a new data clause operand, we should not generate an |
| 636 | // exit operation for it. |
| 637 | lastDataClause = dataEntry; |
| 638 | continue; |
| 639 | } |
| 640 | if (lastDataClause) |
| 641 | if (auto *dataExitOp = findDataExitOp(dataEntryOp: lastDataClause.getDefiningOp())) |
| 642 | builder.setInsertionPointAfter(dataExitOp); |
| 643 | Operation *dataEntryOp = dataEntry.getDefiningOp(); |
| 644 | if (isa<acc::CopyinOp>(Val: dataEntryOp)) { |
| 645 | auto copyoutOp = acc::CopyoutOp::create( |
| 646 | builder, location: dataEntryOp->getLoc(), accVar: dataEntry, var: acc::getVar(accDataClauseOp: dataEntryOp), |
| 647 | /*structured=*/true, /*implicit=*/true, |
| 648 | name: acc::getVarName(accOp: dataEntryOp).value(), bounds: acc::getBounds(accDataClauseOp: dataEntryOp)); |
| 649 | copyoutOp.setDataClause(acc::DataClause::acc_copy); |
| 650 | } else if (isa<acc::PresentOp, acc::NoCreateOp>(Val: dataEntryOp)) { |
| 651 | auto deleteOp = acc::DeleteOp::create( |
| 652 | builder, location: dataEntryOp->getLoc(), accVar: dataEntry, |
| 653 | /*structured=*/true, /*implicit=*/true, |
| 654 | name: acc::getVarName(accOp: dataEntryOp).value(), bounds: acc::getBounds(accDataClauseOp: dataEntryOp)); |
| 655 | deleteOp.setDataClause(acc::getDataClause(accDataEntryOp: dataEntryOp).value()); |
| 656 | } else if (isa<acc::DevicePtrOp>(Val: dataEntryOp)) { |
| 657 | // Do nothing. |
| 658 | } else { |
| 659 | llvm_unreachable("unhandled data exit" ); |
| 660 | } |
| 661 | lastDataClause = dataEntry; |
| 662 | } |
| 663 | } |
| 664 | |
| 665 | /// Returns all base references of a value in order. |
| 666 | /// So for example, if we have a reference to a struct field like |
| 667 | /// s.f1.f2.f3, this will return <s, s.f1, s.f1.f2, s.f1.f2.f3>. |
| 668 | /// Any intermediate casts/view-like operations are included in the |
| 669 | /// chain as well. |
| 670 | static SmallVector<Value> getBaseRefsChain(Value val) { |
| 671 | SmallVector<Value> baseRefs; |
| 672 | baseRefs.push_back(Elt: val); |
| 673 | while (true) { |
| 674 | Value prevVal = val; |
| 675 | |
| 676 | val = acc::getBaseEntity(val); |
| 677 | if (val != baseRefs.front()) |
| 678 | baseRefs.insert(I: baseRefs.begin(), Elt: val); |
| 679 | |
| 680 | // If this is a view-like operation, it is effectively another |
| 681 | // view of the same entity so we should add it to the chain also. |
| 682 | if (auto viewLikeOp = val.getDefiningOp<ViewLikeOpInterface>()) { |
| 683 | val = viewLikeOp.getViewSource(); |
| 684 | baseRefs.insert(I: baseRefs.begin(), Elt: val); |
| 685 | } |
| 686 | |
| 687 | // Continue loop if we made any progress |
| 688 | if (val == prevVal) |
| 689 | break; |
| 690 | } |
| 691 | |
| 692 | return baseRefs; |
| 693 | } |
| 694 | |
| 695 | static void insertInSortedOrder(SmallVector<Value> &sortedDataClauseOperands, |
| 696 | Operation *newClause) { |
| 697 | auto *insertPos = |
| 698 | std::find_if(first: sortedDataClauseOperands.begin(), |
| 699 | last: sortedDataClauseOperands.end(), pred: [&](Value dataClauseVal) { |
| 700 | // Get the base refs for the current clause we are looking |
| 701 | // at. |
| 702 | auto var = acc::getVar(accDataClauseOp: dataClauseVal.getDefiningOp()); |
| 703 | auto baseRefs = getBaseRefsChain(val: var); |
| 704 | |
| 705 | // If the newClause is of a base ref of an existing clause, |
| 706 | // we should insert it right before the current clause. |
| 707 | // Thus return true to stop iteration when this is the |
| 708 | // case. |
| 709 | return std::find(first: baseRefs.begin(), last: baseRefs.end(), |
| 710 | val: acc::getVar(accDataClauseOp: newClause)) != baseRefs.end(); |
| 711 | }); |
| 712 | |
| 713 | if (insertPos != sortedDataClauseOperands.end()) { |
| 714 | newClause->moveBefore(existingOp: insertPos->getDefiningOp()); |
| 715 | sortedDataClauseOperands.insert(I: insertPos, Elt: acc::getAccVar(accDataClauseOp: newClause)); |
| 716 | } else { |
| 717 | sortedDataClauseOperands.push_back(Elt: acc::getAccVar(accDataClauseOp: newClause)); |
| 718 | } |
| 719 | } |
| 720 | |
| 721 | template <typename OpT> |
| 722 | void ACCImplicitData::generateImplicitDataOps( |
| 723 | ModuleOp &module, OpT computeConstructOp, |
| 724 | std::optional<acc::ClauseDefaultValue> &defaultClause, |
| 725 | acc::OpenACCSupport &accSupport) { |
| 726 | // Implicit data attributes are only applied if "[t]here is no default(none) |
| 727 | // clause visible at the compute construct", unless ignoreDefaultNone is set. |
| 728 | if (!ignoreDefaultNone && defaultClause.has_value() && |
| 729 | defaultClause.value() == acc::ClauseDefaultValue::None) |
| 730 | return; |
| 731 | assert(!defaultClause.has_value() || |
| 732 | defaultClause.value() == acc::ClauseDefaultValue::Present || |
| 733 | (ignoreDefaultNone && |
| 734 | defaultClause.value() == acc::ClauseDefaultValue::None)); |
| 735 | |
| 736 | // 1) Collect live-in values. |
| 737 | Region &accRegion = computeConstructOp->getRegion(0); |
| 738 | SetVector<Value> liveInValues; |
| 739 | getUsedValuesDefinedAbove(regions: accRegion, values&: liveInValues); |
| 740 | |
| 741 | // 2) Run the filtering to find relevant pointers that need copied. |
| 742 | auto isCandidate{[&](Value val) -> bool { |
| 743 | return isCandidateForImplicitData(val, accRegion, accSupport, symbolTables); |
| 744 | }}; |
| 745 | auto candidateVars(llvm::filter_to_vector(liveInValues, isCandidate)); |
| 746 | if (candidateVars.empty()) |
| 747 | return; |
| 748 | |
| 749 | // 3) Generate data clauses for the variables. |
| 750 | SmallVector<Value> newPrivateOperands; |
| 751 | SmallVector<Value> newDataClauseOperands; |
| 752 | OpBuilder builder(computeConstructOp); |
| 753 | if (!candidateVars.empty()) { |
| 754 | LLVM_DEBUG(llvm::dbgs() << "== Generating clauses for ==\n" |
| 755 | << computeConstructOp << "\n" ); |
| 756 | } |
| 757 | auto &domInfo = this->getAnalysis<DominanceInfo>(); |
| 758 | auto &postDomInfo = this->getAnalysis<PostDominanceInfo>(); |
| 759 | auto dominatingDataClauses = |
| 760 | acc::getDominatingDataClauses(computeConstructOp, domInfo, postDomInfo); |
| 761 | for (auto var : candidateVars) { |
| 762 | auto newDataClauseOp = generateDataClauseOpForCandidate( |
| 763 | var, module, builder, computeConstructOp, dominatingDataClauses, |
| 764 | defaultClause); |
| 765 | fillInBoundsForUnknownDimensions(newDataClauseOp, builder); |
| 766 | LLVM_DEBUG(llvm::dbgs() << "Generated data clause for " << var << ":\n" |
| 767 | << "\t" << *newDataClauseOp << "\n" ); |
| 768 | if (isa_and_nonnull<acc::PrivateOp, acc::FirstprivateOp, acc::ReductionOp>( |
| 769 | newDataClauseOp)) { |
| 770 | newPrivateOperands.push_back(Elt: acc::getAccVar(accDataClauseOp: newDataClauseOp)); |
| 771 | } else if (isa_and_nonnull<ACC_DATA_CLAUSE_OPS>(newDataClauseOp)) { |
| 772 | newDataClauseOperands.push_back(Elt: acc::getAccVar(accDataClauseOp: newDataClauseOp)); |
| 773 | dominatingDataClauses.push_back(acc::getAccVar(accDataClauseOp: newDataClauseOp)); |
| 774 | } |
| 775 | } |
| 776 | |
| 777 | // 4) Legalize values in region (aka the uses in the region are the result |
| 778 | // of the data clause ops) |
| 779 | legalizeValuesInRegion(accRegion, newPrivateOperands, newDataClauseOperands); |
| 780 | |
| 781 | // 5) Generate private recipes which are required for properly attaching |
| 782 | // private operands. |
| 783 | if constexpr (!std::is_same_v<OpT, acc::KernelsOp>) |
| 784 | generateRecipes(module, builder, computeConstructOp, newOperands: newPrivateOperands); |
| 785 | |
| 786 | // 6) Figure out insertion order for the new data clause operands. |
| 787 | SmallVector<Value> sortedDataClauseOperands( |
| 788 | computeConstructOp.getDataClauseOperands()); |
| 789 | for (auto newClause : newDataClauseOperands) |
| 790 | insertInSortedOrder(sortedDataClauseOperands, newClause: newClause.getDefiningOp()); |
| 791 | |
| 792 | // 7) Generate the data exit operations. |
| 793 | generateDataExitOperations(builder, computeConstructOp, newDataClauseOperands, |
| 794 | sortedDataClauseOperands); |
| 795 | // 8) Add all of the new operands to the compute construct op. |
| 796 | if constexpr (!std::is_same_v<OpT, acc::KernelsOp>) |
| 797 | addNewPrivateOperands(computeConstructOp, newPrivateOperands); |
| 798 | computeConstructOp.getDataClauseOperandsMutable().assign( |
| 799 | sortedDataClauseOperands); |
| 800 | } |
| 801 | |
| 802 | void ACCImplicitData::runOnOperation() { |
| 803 | ModuleOp module = this->getOperation(); |
| 804 | |
| 805 | acc::OpenACCSupport &accSupport = getAnalysis<acc::OpenACCSupport>(); |
| 806 | SymbolTableCollection localSymbolTables; |
| 807 | symbolTables = &localSymbolTables; |
| 808 | |
| 809 | module.walk(callback: [&](Operation *op) { |
| 810 | if (isa<ACC_COMPUTE_CONSTRUCT_OPS>(Val: op)) { |
| 811 | assert(op->getNumRegions() == 1 && "must have 1 region" ); |
| 812 | |
| 813 | auto defaultClause = acc::getDefaultAttr(op); |
| 814 | llvm::TypeSwitch<Operation *, void>(op) |
| 815 | .Case<ACC_COMPUTE_CONSTRUCT_OPS>(caseFn: [&](auto op) { |
| 816 | generateImplicitDataOps(module, op, defaultClause, accSupport); |
| 817 | }) |
| 818 | .Default(defaultFn: [&](Operation *) {}); |
| 819 | } |
| 820 | }); |
| 821 | symbolTables = nullptr; |
| 822 | } |
| 823 | } // namespace |
| 824 | |