1//===- OpenACCUtils.cpp ---------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/OpenACC/OpenACCUtils.h"
10
11#include "mlir/Dialect/GPU/IR/GPUDialect.h"
12#include "mlir/Dialect/OpenACC/OpenACC.h"
13#include "mlir/Dialect/SCF/IR/SCF.h"
14#include "mlir/Dialect/Utils/StaticValueUtils.h"
15#include "mlir/IR/BuiltinOps.h"
16#include "mlir/IR/BuiltinTypes.h"
17#include "mlir/IR/Dominance.h"
18#include "mlir/IR/SymbolTable.h"
19#include "mlir/Interfaces/FunctionInterfaces.h"
20#include "mlir/Interfaces/ViewLikeInterface.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/SetVector.h"
23#include "llvm/ADT/TypeSwitch.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/Support/Casting.h"
26
27mlir::Operation *mlir::acc::getEnclosingComputeOp(mlir::Region &region) {
28 return region
29 .getParentOfType<ACC_COMPUTE_CONSTRUCT_OPS, mlir::acc::ComputeRegionOp>();
30}
31
32mlir::Value mlir::acc::getACCOperandForBlockArg(mlir::Value v) {
33 auto barg = mlir::dyn_cast<mlir::BlockArgument>(Val&: v);
34 if (!barg)
35 return nullptr;
36
37 mlir::Block *block = barg.getOwner();
38 auto computeReg =
39 mlir::dyn_cast<mlir::acc::ComputeRegionOp>(Val: block->getParentOp());
40 if (!computeReg)
41 return nullptr;
42 assert(block == computeReg.getBody() &&
43 "block must be the body of acc.compute_region");
44 return computeReg.getOperand(blockArg: barg);
45}
46
47mlir::BlockArgument
48mlir::acc::getACCBlockArgForOperand(mlir::OpOperand &operand) {
49 auto computeReg =
50 mlir::dyn_cast<mlir::acc::ComputeRegionOp>(Val: operand.getOwner());
51 if (!computeReg)
52 return nullptr;
53 // `launch` then `ins` lead the operand list and map to the body arguments in
54 // order.
55 unsigned idx = operand.getOperandNumber();
56 if (idx >=
57 computeReg.getLaunchArgs().size() + computeReg.getInputArgs().size())
58 return nullptr;
59 return computeReg.getBody()->getArgument(i: idx);
60}
61
62mlir::Operation *mlir::acc::getACCDataClauseOpForBlockArg(mlir::Value v) {
63 mlir::Value orig = getACCOperandForBlockArg(v);
64 if (!orig)
65 return nullptr;
66 mlir::Operation *def = orig.getDefiningOp();
67 return mlir::isa_and_nonnull<ACC_DATA_ENTRY_OPS>(Val: def) ? def : nullptr;
68}
69
70template <typename OpTy>
71static bool isOnlyUsedByOpClauses(mlir::Value val, mlir::Region &region) {
72 auto checkIfUsedOnlyByOpInside = [&](mlir::Operation *user) {
73 // For any users which are not in the current acc region, we can ignore.
74 // Return true so that it can be used in a `all_of` check.
75 if (!region.isAncestor(other: user->getParentRegion()))
76 return true;
77 return mlir::isa<OpTy>(user);
78 };
79
80 return llvm::all_of(val.getUsers(), checkIfUsedOnlyByOpInside);
81}
82
83bool mlir::acc::isOnlyUsedByPrivateClauses(mlir::Value val,
84 mlir::Region &region) {
85 return isOnlyUsedByOpClauses<mlir::acc::PrivateOp>(val, region);
86}
87
88bool mlir::acc::isOnlyUsedByReductionClauses(mlir::Value val,
89 mlir::Region &region) {
90 return isOnlyUsedByOpClauses<mlir::acc::ReductionOp>(val, region);
91}
92
93std::optional<mlir::acc::ClauseDefaultValue>
94mlir::acc::getDefaultAttr(Operation *op) {
95 std::optional<mlir::acc::ClauseDefaultValue> defaultAttr;
96 Operation *currOp = op;
97
98 // Iterate outwards until a default clause is found (since OpenACC
99 // specification notes that a visible default clause is the nearest default
100 // clause appearing on the compute construct or a lexically containing data
101 // construct.
102 while (!defaultAttr.has_value() && currOp) {
103 defaultAttr =
104 llvm::TypeSwitch<mlir::Operation *,
105 std::optional<mlir::acc::ClauseDefaultValue>>(currOp)
106 .Case<ACC_COMPUTE_CONSTRUCT_OPS, mlir::acc::DataOp>(
107 caseFn: [&](auto op) { return op.getDefaultAttr(); })
108 .Default(defaultFn: [&](Operation *) { return std::nullopt; });
109 currOp = currOp->getParentOp();
110 }
111
112 return defaultAttr;
113}
114
115mlir::acc::VariableTypeCategory mlir::acc::getTypeCategory(mlir::Value var) {
116 mlir::acc::VariableTypeCategory typeCategory =
117 mlir::acc::VariableTypeCategory::uncategorized;
118 if (auto mappableTy = dyn_cast<mlir::acc::MappableType>(Val: var.getType()))
119 typeCategory = mappableTy.getTypeCategory(var);
120 else if (auto pointerLikeTy =
121 dyn_cast<mlir::acc::PointerLikeType>(Val: var.getType()))
122 typeCategory = pointerLikeTy.getPointeeTypeCategory(
123 varPtr: cast<TypedValue<mlir::acc::PointerLikeType>>(Val&: var),
124 varType: pointerLikeTy.getElementType());
125 return typeCategory;
126}
127
128llvm::StringLiteral mlir::acc::getVarNamePlaceholder() {
129 return llvm::StringLiteral("<acc.varname.placeholder>");
130}
131
132std::string mlir::acc::getVariableName(mlir::Value v) {
133 Value current = v;
134
135 // Walk through view operations until a name is found or can't go further
136 while (Operation *definingOp = current.getDefiningOp()) {
137 // For integer constants, return their value as a string.
138 if (std::optional<int64_t> constVal = getConstantIntValue(ofr: current))
139 return std::to_string(val: *constVal);
140
141 // Check for `acc.var_name` attribute
142 if (auto varNameAttr = definingOp->getDiscardableAttrOfType<VarNameAttr>(
143 name: getVarNameAttrName()))
144 return varNameAttr.getName().str();
145
146 // If it is a data entry operation, get name via getVarName
147 if (isa<ACC_DATA_ENTRY_OPS, MapInfoOp>(Val: definingOp))
148 if (auto name = acc::getVarName(accOp: definingOp))
149 return name->str();
150
151 // A global goes by the symbol it is addressed through.
152 if (auto addressOf = dyn_cast<AddressOfGlobalOpInterface>(Val: definingOp))
153 return addressOf.getSymbol().getLeafReference().str();
154
155 // If it's a view operation, continue to the source
156 if (auto viewOp = dyn_cast<ViewLikeOpInterface>(Val: definingOp)) {
157 current = viewOp.getViewSource();
158 continue;
159 }
160
161 break;
162 }
163
164 return "";
165}
166
167std::string mlir::acc::getRecipeName(mlir::acc::RecipeKind kind,
168 mlir::Type type) {
169 assert(kind == mlir::acc::RecipeKind::private_recipe ||
170 kind == mlir::acc::RecipeKind::firstprivate_recipe ||
171 kind == mlir::acc::RecipeKind::reduction_recipe);
172 if (!llvm::isa<mlir::acc::PointerLikeType, mlir::acc::MappableType>(Val: type))
173 return "";
174
175 std::string recipeName;
176 llvm::raw_string_ostream ss(recipeName);
177 ss << (kind == mlir::acc::RecipeKind::private_recipe ? "privatization_"
178 : kind == mlir::acc::RecipeKind::firstprivate_recipe
179 ? "firstprivatization_"
180 : "reduction_");
181
182 // Print the type using its dialect-defined textual format.
183 type.print(os&: ss);
184 ss.flush();
185
186 // Replace invalid characters (anything that's not a letter, number, or
187 // period) since this needs to be a valid MLIR identifier.
188 for (char &c : recipeName) {
189 if (!std::isalnum(static_cast<unsigned char>(c)) && c != '.' && c != '_') {
190 if (c == '?')
191 c = 'U';
192 else if (c == '*')
193 c = 'Z';
194 else if (c == '(' || c == ')' || c == '[' || c == ']' || c == '{' ||
195 c == '}' || c == '<' || c == '>')
196 c = '_';
197 else
198 c = 'X';
199 }
200 }
201
202 return recipeName;
203}
204
205mlir::Value mlir::acc::getBaseEntity(mlir::Value val) {
206 if (auto partialEntityAccessOp =
207 val.getDefiningOp<PartialEntityAccessOpInterface>()) {
208 if (!partialEntityAccessOp.isCompleteView())
209 return partialEntityAccessOp.getBaseEntity();
210 }
211
212 return val;
213}
214
215/// Look up `symbol` in the `gpu.module`s of the enclosing module. A
216/// `gpu.module` is its own symbol table, so `lookupNearestSymbolFrom` from a
217/// user outside of it does not find definitions placed inside.
218static mlir::Operation *lookupSymbolInGPUModules(mlir::Operation *user,
219 mlir::SymbolRefAttr symbol) {
220 auto moduleOp = user->getParentOfType<mlir::ModuleOp>();
221 if (!moduleOp)
222 return nullptr;
223 for (auto gpuModule : moduleOp.getOps<mlir::gpu::GPUModuleOp>()) {
224 if (mlir::Operation *op = mlir::SymbolTable::lookupSymbolIn(
225 op: gpuModule, symbol: symbol.getRootReference()))
226 return op;
227 }
228 return nullptr;
229}
230
231bool mlir::acc::isValidSymbolUse(mlir::Operation *user,
232 mlir::SymbolRefAttr symbol,
233 mlir::Operation **definingOpPtr) {
234 // A pass may prepare the device-side definition of a symbol inside a
235 // `gpu.module` while the use is still a reference from host IR. Such a
236 // symbol is meant to be used on device, so the use is valid.
237 if (mlir::Operation *gpuOp = lookupSymbolInGPUModules(user, symbol)) {
238 if (definingOpPtr)
239 *definingOpPtr = gpuOp;
240 return true;
241 }
242
243 mlir::Operation *definingOp =
244 mlir::SymbolTable::lookupNearestSymbolFrom(from: user, symbol);
245
246 // If there are no defining ops, we have no way to ensure validity because
247 // we cannot check for any attributes.
248 if (!definingOp)
249 return false;
250
251 if (definingOpPtr)
252 *definingOpPtr = definingOp;
253
254 // Check if the defining op is a recipe.
255 // Recipes are valid as they get materialized before being offloaded to
256 // device. They are only instructions for how to materialize.
257 if (mlir::isa<mlir::accomp::RecipeInterface>(Val: definingOp))
258 return true;
259
260 // Check if the defining op is a global variable that is device data.
261 // Device data is already resident on the device and does not need mapping.
262 if (auto globalVar =
263 mlir::dyn_cast<mlir::acc::GlobalVariableOpInterface>(Val: definingOp))
264 if (globalVar.isDeviceAccessible())
265 return true;
266
267 // Check if the defining op is a function
268 if (auto func =
269 mlir::dyn_cast_if_present<mlir::FunctionOpInterface>(Val: definingOp)) {
270 // If this symbol is actually an acc routine or a specialized acc routine -
271 // then it is expected for it to be offloaded - therefore it is valid.
272 if (func->hasDiscardableAttr(name: mlir::acc::getRoutineInfoAttrName()) ||
273 func->hasDiscardableAttr(name: mlir::acc::getSpecializedRoutineAttrName()))
274 return true;
275
276 // If this symbol is a call to an LLVM intrinsic, then it is likely valid.
277 // Check the following:
278 // 1. The function is private
279 // 2. The function has no body
280 // 3. Name starts with "llvm."
281 // 4. The function's name is a valid LLVM intrinsic name
282 if (func.getVisibility() == mlir::SymbolTable::Visibility::Private &&
283 func.getFunctionBody().empty() && func.getName().starts_with(Prefix: "llvm.") &&
284 llvm::Intrinsic::lookupIntrinsicID(Name: func.getName()) !=
285 llvm::Intrinsic::not_intrinsic)
286 return true;
287 }
288
289 // A declare attribute is needed for symbol references.
290 bool hasDeclare =
291 definingOp->hasDiscardableAttr(name: mlir::acc::getDeclareAttrName());
292 return hasDeclare;
293}
294
295/// Finds the global variable that `addrOf` refers to, through `symbolTables`
296/// when provided (cached lookup) and by scanning the nearest symbol table
297/// otherwise.
298static mlir::acc::GlobalVariableOpInterface
299lookupGlobalVariable(mlir::Operation *addrOf, mlir::SymbolRefAttr symbol,
300 mlir::SymbolTableCollection *symbolTables) {
301 if (symbolTables)
302 return symbolTables
303 ->lookupNearestSymbolFrom<mlir::acc::GlobalVariableOpInterface>(from: addrOf,
304 symbol);
305 return mlir::SymbolTable::lookupNearestSymbolFrom<
306 mlir::acc::GlobalVariableOpInterface>(from: addrOf, symbol);
307}
308
309bool mlir::acc::isDeviceAccessibleValue(
310 mlir::Value val, mlir::SymbolTableCollection *symbolTables) {
311 // Check if the value is device data via type interfaces.
312 // Device data is already resident on the device and does not need mapping.
313 if (auto mappableTy = dyn_cast<mlir::acc::MappableType>(Val: val.getType()))
314 if (mappableTy.isDeviceAccessible(var: val, symbolTables))
315 return true;
316
317 if (auto pointerLikeTy = dyn_cast<mlir::acc::PointerLikeType>(Val: val.getType()))
318 if (pointerLikeTy.isDeviceAccessible(var: val, symbolTables))
319 return true;
320
321 mlir::Operation *defOp = val.getDefiningOp();
322 if (!defOp)
323 return false;
324
325 // `acc.declare` with deviceptr marks data that is already associated with
326 // the device.
327 if (auto declareAttr =
328 defOp->getDiscardableAttrOfType<mlir::acc::DeclareAttr>(
329 name: mlir::acc::getDeclareAttrName()))
330 if (declareAttr.getDataClause().getValue() ==
331 mlir::acc::DataClause::acc_deviceptr)
332 return true;
333
334 // Handle operations that access a partial entity - check if the base entity
335 // is device data.
336 if (auto partialAccess =
337 dyn_cast<mlir::acc::PartialEntityAccessOpInterface>(Val: defOp)) {
338 if (mlir::Value base = partialAccess.getBaseEntity())
339 return isDeviceAccessibleValue(val: base, symbolTables);
340 }
341
342 // Handle address_of - check if the referenced global is device data.
343 if (auto addrOfIface =
344 dyn_cast<mlir::acc::AddressOfGlobalOpInterface>(Val: defOp)) {
345 auto symbol = addrOfIface.getSymbol();
346 if (auto global = lookupGlobalVariable(addrOf: defOp, symbol, symbolTables))
347 return global.isDeviceAccessible();
348 }
349
350 return false;
351}
352
353bool mlir::acc::isInDeviceMemoryValue(
354 mlir::Value val, mlir::SymbolTableCollection *symbolTables) {
355 // In-device-memory data is a subset of device-accessible data: it must be
356 // accessible and its storage must physically reside in device memory.
357 if (auto mappableTy = dyn_cast<mlir::acc::MappableType>(Val: val.getType()))
358 if (mappableTy.isInDeviceMemory(var: val, symbolTables))
359 return true;
360
361 if (auto pointerLikeTy = dyn_cast<mlir::acc::PointerLikeType>(Val: val.getType()))
362 if (pointerLikeTy.isInDeviceMemory(var: val, symbolTables))
363 return true;
364
365 mlir::Operation *defOp = val.getDefiningOp();
366 if (!defOp)
367 return false;
368
369 // `acc.declare` with deviceptr marks data whose storage is already in device
370 // memory.
371 if (auto declareAttr =
372 defOp->getDiscardableAttrOfType<mlir::acc::DeclareAttr>(
373 name: mlir::acc::getDeclareAttrName()))
374 if (declareAttr.getDataClause().getValue() ==
375 mlir::acc::DataClause::acc_deviceptr)
376 return true;
377
378 // Handle operations that access a partial entity - check if the base entity
379 // is in device memory.
380 if (auto partialAccess =
381 dyn_cast<mlir::acc::PartialEntityAccessOpInterface>(Val: defOp)) {
382 if (mlir::Value base = partialAccess.getBaseEntity())
383 return isInDeviceMemoryValue(val: base, symbolTables);
384 }
385
386 // Handle address_of - check if the referenced global is in device memory.
387 if (auto addrOfIface =
388 dyn_cast<mlir::acc::AddressOfGlobalOpInterface>(Val: defOp)) {
389 auto symbol = addrOfIface.getSymbol();
390 if (auto global = lookupGlobalVariable(addrOf: defOp, symbol, symbolTables))
391 return global.isInDeviceMemory();
392 }
393
394 return false;
395}
396
397bool mlir::acc::isValidValueUse(mlir::Value val, mlir::Region &region,
398 mlir::SymbolTableCollection *symbolTables) {
399 // Types that can be passed by value are legal.
400 Type type = val.getType();
401 if (type.isIntOrIndexOrFloat() || isa<mlir::ComplexType>(Val: type) ||
402 llvm::isa<mlir::VectorType>(Val: type))
403 return true;
404
405 // If this is produced by an ACC data entry operation, it is valid.
406 if (isa_and_nonnull<ACC_DATA_ENTRY_OPS>(Val: val.getDefiningOp()))
407 return true;
408
409 // If the value is only used by private clauses, it is not a live-in.
410 if (isOnlyUsedByPrivateClauses(val, region))
411 return true;
412
413 // If this is device data, it is valid.
414 if (isDeviceAccessibleValue(val, symbolTables))
415 return true;
416
417 // Arguments of an enclosing acc routine are already on the device.
418 if (mlir::Operation *parent = region.getParentOp()) {
419 if (auto func = parent->getParentOfType<mlir::FunctionOpInterface>()) {
420 if ((mlir::acc::isAccRoutine(op: func) ||
421 mlir::acc::isSpecializedAccRoutine(op: func)) &&
422 llvm::is_contained(Range: func.getArguments(), Element: val))
423 return true;
424 }
425 }
426
427 return false;
428}
429
430llvm::SmallVector<mlir::Value>
431mlir::acc::getDominatingDataClauses(mlir::Operation *computeConstructOp,
432 mlir::DominanceInfo &domInfo,
433 mlir::PostDominanceInfo &postDomInfo) {
434 llvm::SmallSetVector<mlir::Value, 8> dominatingDataClauses;
435
436 llvm::TypeSwitch<mlir::Operation *>(computeConstructOp)
437 .Case<mlir::acc::ParallelOp, mlir::acc::KernelsOp, mlir::acc::SerialOp>(
438 caseFn: [&](auto op) {
439 for (auto dataClause : op.getDataClauseOperands()) {
440 dominatingDataClauses.insert(dataClause);
441 }
442 })
443 .Default(defaultFn: [](mlir::Operation *) {});
444
445 // Collect the data clauses from enclosing data constructs.
446 mlir::Operation *currParentOp = computeConstructOp->getParentOp();
447 while (currParentOp) {
448 if (mlir::isa<mlir::acc::DataOp>(Val: currParentOp)) {
449 for (auto dataClause : mlir::dyn_cast<mlir::acc::DataOp>(Val: currParentOp)
450 .getDataClauseOperands()) {
451 dominatingDataClauses.insert(X: dataClause);
452 }
453 }
454 currParentOp = currParentOp->getParentOp();
455 }
456
457 // Find the enclosing function/subroutine
458 auto funcOp =
459 computeConstructOp->getParentOfType<mlir::FunctionOpInterface>();
460 if (!funcOp)
461 return dominatingDataClauses.takeVector();
462
463 // Walk the function to find `acc.declare_enter`/`acc.declare_exit` pairs that
464 // dominate and post-dominate the compute construct and add their data
465 // clauses to the list.
466 funcOp->walk(callback: [&](mlir::acc::DeclareEnterOp declareEnterOp) {
467 if (domInfo.dominates(a: declareEnterOp.getOperation(), b: computeConstructOp)) {
468 // Collect all `acc.declare_exit` ops for this token.
469 llvm::SmallVector<mlir::acc::DeclareExitOp> exits;
470 for (auto *user : declareEnterOp.getToken().getUsers())
471 if (auto declareExit = mlir::dyn_cast<mlir::acc::DeclareExitOp>(Val: user))
472 exits.push_back(Elt: declareExit);
473
474 // Only add clauses if every `acc.declare_exit` op post-dominates the
475 // compute construct.
476 if (!exits.empty() &&
477 llvm::all_of(Range&: exits, P: [&](mlir::acc::DeclareExitOp exitOp) {
478 return postDomInfo.postDominates(a: exitOp, b: computeConstructOp);
479 })) {
480 for (auto dataClause : declareEnterOp.getDataClauseOperands())
481 dominatingDataClauses.insert(X: dataClause);
482 }
483 }
484 });
485
486 return dominatingDataClauses.takeVector();
487}
488
489static mlir::Value getIfCondition(mlir::Operation *op) {
490 if (auto ifOp = mlir::dyn_cast<mlir::scf::IfOp>(Val: op))
491 return ifOp.getCondition();
492
493 if (!mlir::isa<mlir::RegionBranchOpInterface>(Val: op) ||
494 mlir::isa<mlir::LoopLikeOpInterface>(Val: op))
495 return {};
496 if (op->getNumOperands() != 1 || op->getNumRegions() != 2 ||
497 !op->getOperand(idx: 0).getType().isSignlessInteger(width: 1))
498 return {};
499 return op->getOperand(idx: 0);
500}
501
502bool mlir::acc::isInOffTargetBranch(mlir::Operation *op,
503 llvm::ArrayRef<int64_t> deviceTypes) {
504 // No device type is known to be true, so no branch can be classified.
505 if (deviceTypes.empty())
506 return false;
507
508 for (mlir::Operation *parent = op->getParentOp();
509 parent &&
510 !mlir::isa<ACC_COMPUTE_CONSTRUCT_OPS, mlir::acc::ComputeRegionOp,
511 mlir::FunctionOpInterface>(Val: parent);
512 parent = parent->getParentOp()) {
513 mlir::Value condition = getIfCondition(op: parent);
514 if (!condition)
515 continue;
516
517 // The condition must be the `acc.on_device` result itself.
518 auto onDeviceOp = condition.getDefiningOp<mlir::acc::OnDeviceOp>();
519 if (!onDeviceOp)
520 continue;
521
522 std::optional<int64_t> deviceTypeValue =
523 mlir::getConstantIntValue(ofr: onDeviceOp.getDeviceType());
524 if (!deviceTypeValue)
525 continue;
526
527 bool onTarget = llvm::is_contained(Range&: deviceTypes, Element: *deviceTypeValue);
528 bool inThen = parent->getRegion(index: 0).isAncestor(other: op->getParentRegion());
529 bool inElse = parent->getRegion(index: 1).isAncestor(other: op->getParentRegion());
530 // Off the target: the then of a device type outside `deviceTypes`, or
531 // the else of a device type in `deviceTypes`.
532 if ((!onTarget && inThen) || (onTarget && inElse))
533 return true;
534 }
535 return false;
536}
537
538mlir::remark::detail::InFlightRemark
539mlir::acc::emitRemark(mlir::Operation *op,
540 const std::function<std::string()> &messageFn,
541 llvm::StringRef category) {
542 using namespace mlir::remark;
543 mlir::Location loc = op->getLoc();
544 auto *engine = loc->getContext()->getRemarkEngine();
545 if (!engine)
546 return remark::detail::InFlightRemark{};
547
548 llvm::StringRef funcName;
549 if (auto func = dyn_cast<mlir::FunctionOpInterface>(Val: op))
550 funcName = func.getName();
551 else if (auto funcOp = op->getParentOfType<mlir::FunctionOpInterface>())
552 funcName = funcOp.getName();
553
554 auto opts = RemarkOpts::name(n: "openacc").category(v: category);
555 if (!funcName.empty())
556 opts = opts.function(v: funcName);
557
558 auto remark = engine->emitOptimizationRemark(loc, opts);
559 if (remark)
560 remark << messageFn();
561 return remark;
562}
563