1//===- ACCRecipeMaterialization.cpp - Materialize ACC recipes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Overview:
10// ---------
11// OpenACC compute constructs (acc.parallel, acc.serial, acc.kernels) and
12// acc.loop can carry data clauses (acc.private, acc.firstprivate,
13// acc.reduction) that refer to recipes (acc.private.recipe,
14// acc.firstprivate.recipe, acc.reduction.recipe). Recipes define how to
15// initialize, copy, combine, or destroy a particular variable. This pass clones
16// those regions into the construct and ensures the materialized SSA values are
17// used instead.
18//
19// Transforms:
20// -----------
21// 1. Firstprivate: Inserts acc.firstprivate_map so the initial value is
22// available on the device, then clones the recipe init and copy regions
23// into the construct and replaces uses with the materialized alloca.
24// Optional destroy region is cloned before the region terminator.
25//
26// 2. Private: Clones the recipe init region into the construct (at the
27// region entry or at the loop op for acc.loop private). Replaces uses
28// of the recipe result with the materialized alloca. Optional destroy
29// region is cloned before the region terminator.
30//
31// 3. Reduction: Creates acc.reduction_init (init region inlined) and
32// acc.reduction_combine_region (combiner region inlined). Uses within
33// the region are updated to the reduction init result.
34// In addition, creates appropriate acc.copyin/copyout around
35// the compute region; the reduction's initial value is taken
36// from the copied in variable, and the final reduction value
37// is copied out to the variable. If there are existing
38// data operations for the reduction variable, no new data
39// operations are added.
40//
41// Requirements:
42// -------------
43// 1. OpenACCSupport: The pass uses the `acc::OpenACCSupport` analysis
44// including emitNYI for unsupported cases.
45//
46//===----------------------------------------------------------------------===//
47
48#include "mlir/Dialect/Arith/IR/Arith.h"
49#include "mlir/Dialect/Arith/Utils/Utils.h"
50#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
51#include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h"
52#include "mlir/Dialect/OpenACC/OpenACC.h"
53#include "mlir/Dialect/OpenACC/OpenACCParMapping.h"
54#include "mlir/Dialect/OpenACC/OpenACCUtils.h"
55#include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h"
56#include "mlir/Dialect/OpenACC/OpenACCUtilsLoop.h"
57#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
58#include "mlir/IR/Block.h"
59#include "mlir/IR/Builders.h"
60#include "mlir/IR/IRMapping.h"
61#include "mlir/IR/SymbolTable.h"
62#include "mlir/IR/Value.h"
63#include "mlir/IR/ValueRange.h"
64#include "mlir/Interfaces/LoopLikeInterface.h"
65#include "mlir/Support/LLVM.h"
66#include "mlir/Transforms/RegionUtils.h"
67#include "llvm/ADT/DenseSet.h"
68#include "llvm/ADT/STLExtras.h"
69#include "llvm/ADT/TypeSwitch.h"
70#include "llvm/Support/Debug.h"
71#include "llvm/Support/ErrorHandling.h"
72
73namespace mlir {
74namespace acc {
75#define GEN_PASS_DEF_ACCRECIPEMATERIALIZATION
76#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
77} // namespace acc
78} // namespace mlir
79
80#define DEBUG_TYPE "acc-recipe-materialization"
81
82namespace {
83using namespace mlir;
84
85static void setLocation(Region &region, Location loc) {
86 // Since recipes are generated per type and not per variable, the location
87 // of the recipe operations which get inlined will not necessarily be the
88 // same as the location of the op that is being materialized. Force an update
89 // of the location of the recipe operations to the location of the op that is
90 // being materialized.
91 region.walk(callback: [&](Operation *op) { op->setLoc(loc); });
92}
93
94// Complex multiplication mixes the real and imaginary parts, so a reduction
95// of that operator cannot be lowered as two independent component reductions.
96// Keep unwrapping pointer-like types so a reference to a reference still
97// reports the element type.
98static bool isComplexReductionType(Type type) {
99 while (auto ptrTy = dyn_cast<acc::PointerLikeType>(Val&: type)) {
100 Type elementType = ptrTy.getElementType();
101 if (!elementType || elementType == type)
102 return false;
103 type = elementType;
104 }
105 return isa<ComplexType>(Val: type);
106}
107
108static void saveVarName(StringRef name, Value dst) {
109 if (name.empty())
110 return;
111 if (Operation *dstOp = dst.getDefiningOp()) {
112 if (dstOp->getDiscardableAttrOfType<acc::VarNameAttr>(
113 name: acc::getVarNameAttrName()))
114 return;
115 if (isa<ACC_DATA_ENTRY_OPS>(Val: dstOp))
116 return;
117 dstOp->setDiscardableAttr(name: acc::getVarNameAttrName(),
118 value: acc::VarNameAttr::get(context: dstOp->getContext(), name));
119 return;
120 }
121 auto blockArg = dyn_cast<BlockArgument>(Val&: dst);
122 if (!blockArg)
123 return;
124 Block *block = blockArg.getOwner();
125 Region *region = block ? block->getParent() : nullptr;
126 if (!region || !block->isEntryBlock())
127 return;
128 Operation *parent = region->getParentOp();
129 if (!parent)
130 return;
131 auto funcOp = dyn_cast<FunctionOpInterface>(Val: parent);
132 if (!funcOp)
133 return;
134 unsigned argIdx = blockArg.getArgNumber();
135 if (argIdx >= funcOp.getNumArguments())
136 return;
137 if (funcOp.getArgAttr(index: argIdx, name: acc::getVarNameAttrName()))
138 return;
139 funcOp.setArgAttr(index: argIdx, name: acc::getVarNameAttrName(),
140 value: acc::VarNameAttr::get(context: parent->getContext(), name));
141}
142
143static void saveVarName(Value src, Value dst) {
144 saveVarName(name: acc::getVariableName(v: src), dst);
145}
146
147static void resolveVarNamePlaceholders(Block *block, Block::iterator ip,
148 StringRef name) {
149 StringRef placeholder = acc::getVarNamePlaceholder();
150 for (auto it = block->begin(); it != std::next(x: ip); ++it) {
151 it->walk(callback: [&](Operation *op) {
152 auto attr = op->getDiscardableAttrOfType<acc::VarNameAttr>(
153 name: acc::getVarNameAttrName());
154 if (!attr || attr.getName() != placeholder)
155 return;
156 if (name.empty())
157 op->removeDiscardableAttr(name: acc::getVarNameAttrName());
158 else
159 op->setDiscardableAttr(name: acc::getVarNameAttrName(),
160 value: acc::VarNameAttr::get(context: op->getContext(), name));
161 });
162 }
163}
164
165// Clone the destroy region of the recipe before the terminator of the provided
166// block. Values must be provided for the destroy region block arguments
167// according to the recipe specifications.
168template <typename RecipeOpTy>
169static void cloneDestroy(Location loc, RecipeOpTy recipe, mlir::Block *block,
170 Block::iterator ip,
171 const llvm::SmallVector<mlir::Value> &arguments) {
172 IRMapping mapping{};
173 Region &destroyRegion = recipe.getDestroyRegion();
174 assert(destroyRegion.getBlocks().front().getNumArguments() ==
175 arguments.size() &&
176 "unexpected acc recipe destroy block arguments");
177
178 setLocation(region&: destroyRegion, loc);
179
180 mapping.map(from: destroyRegion.getBlocks().front().getArguments(), to: arguments);
181 acc::cloneACCRegionInto(src: &destroyRegion, dest: block, inlinePoint: ip, mapping,
182 /*resultsToReplace=*/{});
183}
184
185class ACCRecipeMaterialization
186 : public acc::impl::ACCRecipeMaterializationBase<ACCRecipeMaterialization> {
187public:
188 using acc::impl::ACCRecipeMaterializationBase<
189 ACCRecipeMaterialization>::ACCRecipeMaterializationBase;
190
191 void getDependentDialects(DialectRegistry &registry) const override {
192 acc::impl::ACCRecipeMaterializationBase<
193 ACCRecipeMaterialization>::getDependentDialects(registry);
194 acc::registerAllExtensions(registry);
195 }
196 void runOnOperation() override;
197
198private:
199 // When the recipe reads the original variable, its initial value needs to be
200 // available on the GPU. One way to get that value there is to map the
201 // variable through global memory.
202 // Thus, when we materialize a firstprivate, we materialize it into
203 // a mapping action first. This function ends up with doing the following:
204 // %dev = acc.firstprivate var(%var)
205 // =>
206 // %copy = acc.firstprivate_map var(%var)
207 // %dev = acc.firstprivate var(%copy)
208 // When the recipe materialization happens, the `acc.firstprivate` ends up
209 // being removed. But because of the way we chain it to the
210 // `acc.firstprivate_map`, then its result becomes live-in to the
211 // compute region and used as the variable the initial value is loaded from.
212 template <typename OpTy>
213 void handleInitialValueMapping(OpTy op) const;
214 template <typename OpTy>
215 void removeRecipe(
216 OpTy op, ModuleOp moduleOp,
217 const std::optional<llvm::DenseSet<StringAttr>> &usedSymbols) const;
218 template <typename OpTy, typename RecipeOpTy, typename AccOpTy>
219 LogicalResult materialize(OpTy op, RecipeOpTy recipe, AccOpTy accOp,
220 acc::OpenACCSupport &accSupport,
221 acc::ACCToGPUMappingPolicy &policy,
222 Value materializationVar = {}) const;
223 template <typename OpTy>
224 LogicalResult materializeForACCOp(OpTy accOp, acc::OpenACCSupport &accSupport,
225 acc::ACCToGPUMappingPolicy &policy) const;
226};
227
228template <typename OpTy>
229void ACCRecipeMaterialization::handleInitialValueMapping(OpTy op) const {
230 OpBuilder builder(op);
231 auto mapInitialOp = acc::FirstprivateMapInitialOp::create(
232 builder, op.getLoc(), op.getVar(), op.getStructured(), op.getImplicit(),
233 op.getBounds());
234 mapInitialOp.setName(op.getName());
235 op.getVarMutable().assign(mapInitialOp.getAccVar());
236}
237
238// Whether a recipe region reads the variable it privatizes - a descriptor
239// recipe loads it for the bounds, while a scalar one ignores it. Both init and
240// destroy receive it as their first argument.
241static bool readsVar(Region &region) {
242 if (region.empty() || region.getNumArguments() == 0)
243 return false;
244 return !region.getArgument(i: 0).use_empty();
245}
246
247template <typename OpTy>
248void ACCRecipeMaterialization::removeRecipe(
249 OpTy op, ModuleOp moduleOp,
250 const std::optional<llvm::DenseSet<StringAttr>> &usedSymbols) const {
251 auto recipeName = op.getNameAttr();
252 // Fall back to scanning the module when the symbol uses could not be
253 // gathered up front.
254 bool useEmpty = usedSymbols
255 ? !usedSymbols->contains(V: recipeName)
256 : SymbolTable::symbolKnownUseEmpty(recipeName, moduleOp);
257 if (useEmpty) {
258 LLVM_DEBUG(llvm::dbgs() << "erasing recipe: " << recipeName << "\n");
259 op.erase();
260 } else {
261 LLVM_DEBUG({
262 std::optional<SymbolTable::UseRange> symbolUses =
263 op.getSymbolUses(moduleOp);
264 if (symbolUses.has_value()) {
265 for (SymbolTable::SymbolUse symbolUse : *symbolUses) {
266 llvm::dbgs() << "symbol use: ";
267 symbolUse.getUser()->dump();
268 }
269 }
270 });
271 llvm_unreachable("expected no use of recipe symbol");
272 }
273}
274
275template <typename OpTy, typename RecipeOpTy, typename AccOpTy>
276LogicalResult ACCRecipeMaterialization::materialize(
277 OpTy op, RecipeOpTy recipe, AccOpTy accOp, acc::OpenACCSupport &accSupport,
278 acc::ACCToGPUMappingPolicy &policy, Value materializationVar) const {
279 Region &region = accOp.getRegion();
280 Value origPtr = materializationVar ? materializationVar : op.getVar();
281 Value accPtr = op.getAccVar();
282 assert(accPtr && "invalid op: null acc var");
283
284 OpBuilder b(op);
285 SmallVector<Value> triples;
286
287 // Clone init block into the region at the insertion point specified.
288 Region &initRegion = recipe.getInitRegion();
289 unsigned initNumArguments =
290 initRegion.getBlocks().front().getArguments().size();
291 if (initNumArguments > 1) {
292 // Code from C/C++ will most likely only provide extent arguments to the
293 // recipe arguments.
294 if ((initNumArguments - 1) % 3 != 0) {
295 (void)accSupport.emitNYI(loc: recipe.getLoc(),
296 message: "privatization of array section with extents");
297 return failure();
298 }
299 // The remaining arguments must be the bounds triples
300 // (lower-bound, upper-bound, step), ...
301 unsigned argIdx = 1;
302 // Cast the given value to the type of the combiner region's argument
303 // at position argIdx, and increment argIdx.
304 auto castValueToArgType = [&](Location loc, Value v) {
305 return convertScalarToDtype(
306 b, loc, operand: v,
307 toType: initRegion.getBlocks().front().getArgument(i: argIdx++).getType(),
308 /*isUnsignedCast=*/false);
309 };
310 for (Value bound : acc::getBounds(accDataClauseOp: op)) {
311 auto dataBound = bound.getDefiningOp<acc::DataBoundsOp>();
312 assert(dataBound &&
313 "acc.reduction's bound must be defined by acc.bounds");
314 // NOTE: we should probably generate get_lowerbound, get_upperbound
315 // and get_stride here, so that we can stop looking for the acc.bounds
316 // operation above, and just use the `bound` value.
317 Value lb =
318 castValueToArgType(dataBound.getLoc(), dataBound.getLowerbound());
319 Value ub =
320 castValueToArgType(dataBound.getLoc(), dataBound.getUpperbound());
321 Value step =
322 castValueToArgType(dataBound.getLoc(), dataBound.getStride());
323 triples.append(IL: {lb, ub, step});
324 }
325 assert(triples.size() + 1 == initNumArguments &&
326 "mismatch between number bounds and number of recipe init block "
327 "arguments");
328 }
329
330 IRMapping mapping;
331 SmallVector<Value> initArgs{origPtr};
332 initArgs.append(RHS: triples);
333 mapping.map(from: initRegion.getBlocks().front().getArguments(), to&: initArgs);
334
335 Location loc = op.getLoc();
336 setLocation(region&: initRegion, loc);
337
338 if constexpr (std::is_same_v<OpTy, acc::PrivateOp>) {
339 // Clone the init region for a private.
340 Block *block = &region.front();
341 auto [results, ip] = acc::cloneACCRegionInto(
342 src: &initRegion, dest: block, inlinePoint: block->begin(), mapping, resultsToReplace: {accPtr});
343 assert(!results.empty() && "expected a result from init region");
344 saveVarName(op.getAccVar(), results[0]);
345 resolveVarNamePlaceholders(block, ip, acc::getVariableName(v: op.getAccVar()));
346 // Clone the destroy region for a private, if it exists.
347 if (!recipe.getDestroyRegion().empty()) {
348 results.insert(I: results.begin(), Elt: origPtr);
349 results.append(RHS: triples);
350 cloneDestroy(loc, recipe, block, std::prev(x: block->end()), results);
351 }
352 } else if constexpr (std::is_same_v<OpTy, acc::FirstprivateOp>) {
353 // Clone the init region for a firstprivate.
354 Block *block = &region.front();
355 auto [results, ip] = acc::cloneACCRegionInto(
356 src: &initRegion, dest: block, inlinePoint: block->begin(), mapping, resultsToReplace: {accPtr});
357 assert(!results.empty() && "expected a result from init region");
358 saveVarName(op.getAccVar(), results[0]);
359 resolveVarNamePlaceholders(block, ip, acc::getVariableName(v: op.getAccVar()));
360 // The copy only consumes the original and user-visible private value.
361 SmallVector<Value> copyArgs{origPtr, results.front()};
362 copyArgs.append(RHS: triples);
363 // Destruction also consumes any cleanup values yielded by init.
364 SmallVector<Value> destroyArgs{origPtr};
365 destroyArgs.append(RHS: results);
366 destroyArgs.append(RHS: triples);
367
368 // Clone the copy region for a firstprivate
369 mapping.clear();
370 mapping.map(recipe.getCopyRegion().front().getArguments(), copyArgs);
371 // Clone the copy region for a firstprivate.
372 Region &copyRegion = recipe.getCopyRegion();
373 setLocation(region&: copyRegion, loc);
374 acc::cloneACCRegionInto(src: &copyRegion, dest: block, inlinePoint: std::next(x: ip), mapping, resultsToReplace: {});
375 if (!recipe.getDestroyRegion().empty()) {
376 cloneDestroy(loc, recipe, block, std::prev(x: block->end()), destroyArgs);
377 }
378 } else if constexpr (std::is_same_v<OpTy, acc::ReductionOp>) {
379 if (recipe.getReductionOperator() == acc::ReductionOperator::AccMul &&
380 isComplexReductionType(recipe.getType())) {
381 (void)accSupport.emitNYI(loc, message: "reduction: complex multiply reduction");
382 return failure();
383 }
384
385 auto cloneRegionIntoAccRegion = [&](Region *src, Region *dest,
386 bool hasResult) {
387 src->cloneInto(dest, mapper&: mapping);
388 Block *block = &dest->front();
389 Operation *terminator = block->getTerminator();
390 b.setInsertionPoint(terminator);
391 if (hasResult)
392 acc::YieldOp::create(b, op.getLoc(), terminator->getOperands());
393 else
394 acc::YieldOp::create(b, op.getLoc(), ValueRange{});
395 terminator->erase();
396 };
397
398 // Clone the init region into acc.reduction_init.
399 if constexpr (std::is_same_v<AccOpTy, acc::ParallelOp>)
400 b.setInsertionPointToStart(&region.front());
401 else if constexpr (std::is_same_v<AccOpTy, acc::LoopOp>)
402 b.setInsertionPoint(op);
403 else
404 llvm_unreachable("unexpected acc op with reduction recipe");
405
406 SmallVector<Value> reductionBounds(acc::getBounds(accDataClauseOp: op));
407 auto reductionOp =
408 acc::ReductionInitOp::create(b, op.getLoc(), origPtr, reductionBounds,
409 recipe.getReductionOperatorAttr());
410 saveVarName(op.getAccVar(), reductionOp.getResult());
411 cloneRegionIntoAccRegion(&initRegion, &reductionOp.getRegion(),
412 /*hasResult=*/true);
413 Block *initBlock = &reductionOp.getRegion().front();
414 resolveVarNamePlaceholders(initBlock, std::prev(x: initBlock->end()),
415 acc::getVariableName(v: op.getAccVar()));
416
417 // Update the uses within the loop to use the reduction op result.
418 replaceAllUsesInRegionWith(accPtr, reductionOp.getResult(), region);
419
420 // Clone the combiner region into acc.reduction_combine_region.
421 Region &combinerRegion = recipe.getCombinerRegion();
422 setLocation(region&: combinerRegion, loc);
423
424 Block *entryBlock = &combinerRegion.front();
425
426 if constexpr (std::is_same_v<AccOpTy, acc::ParallelOp>)
427 b.setInsertionPoint(region.back().getTerminator());
428 else if constexpr (std::is_same_v<AccOpTy, acc::LoopOp>)
429 b.setInsertionPointAfter(accOp);
430 else
431 llvm_unreachable("unexpected acc op with reduction recipe");
432
433 // Map the first two block arguments to the original and private
434 // reduction variables. If the recipe's combiner region has the bounds
435 // arguments, we have to map them to the corresponding operands of
436 // acc.reduction operation.
437 mapping.clear();
438 SmallVector<Value, 2> argsRemapping{origPtr, reductionOp.getResult()};
439 argsRemapping.append(RHS: triples);
440 mapping.map(from: entryBlock->getArguments(), to&: argsRemapping);
441
442 auto combineRegionOp = acc::ReductionCombineRegionOp::create(
443 b, op.getLoc(), origPtr, reductionOp.getResult());
444 cloneRegionIntoAccRegion(&combinerRegion, &combineRegionOp.getRegion(),
445 /*hasResult=*/false);
446
447 auto *ctx = b.getContext();
448
449 // For reductions that come from parallel constructs, explicitly set the
450 // GPU parallel dimensions attribute to blockXDim since they will always be
451 // gang private. GPU parallel dimensions cannot be determined for acc.loop
452 // at this point.
453 if constexpr (std::is_same_v<AccOpTy, acc::ParallelOp>) {
454 acc::GPUParallelDimsAttr parDimsAttr;
455 if (accOp.isEffectivelySerial()) {
456 // If acc.serial has been lowered to a parallel op that is effectively
457 // sequential
458 parDimsAttr = acc::getSeqParDimsAttr(ctx, policy);
459 } else {
460 parDimsAttr = acc::getGangDim1ParDimsAttr(ctx, policy);
461 }
462 acc::setParDimsAttr(op: reductionOp, attr: parDimsAttr);
463 acc::setParDimsAttr(op: combineRegionOp, attr: parDimsAttr);
464 }
465
466 // Set sequential parallel dimensions attribute for loops in the recipe.
467 auto setSeqParDimsForRecipeLoops = [&](Region *r) {
468 r->walk([&](LoopLikeOpInterface loopLike) {
469 acc::setParDimsAttr(op: loopLike, attr: acc::getSeqParDimsAttr(ctx, policy));
470 });
471 };
472 setSeqParDimsForRecipeLoops(&reductionOp.getRegion());
473 setSeqParDimsForRecipeLoops(&combineRegionOp.getRegion());
474
475 if (!recipe.getDestroyRegion().empty()) {
476 SmallVector<Value> results{origPtr, reductionOp.getResult()};
477 results.append(RHS: triples);
478 Block::iterator ip = std::next(x: Block::iterator(combineRegionOp));
479 cloneDestroy(loc, recipe, combineRegionOp->getBlock(), ip, results);
480 }
481 } else {
482 llvm_unreachable("unexpected op type");
483 }
484
485 op.erase();
486 return success();
487}
488
489template <typename OpTy>
490LogicalResult ACCRecipeMaterialization::materializeForACCOp(
491 OpTy accOp, acc::OpenACCSupport &accSupport,
492 acc::ACCToGPUMappingPolicy &policy) const {
493 assert(isa<ACC_COMPUTE_CONSTRUCT_AND_LOOP_OPS>(accOp));
494
495 // Reduction recipes use the original variable both to initialize the
496 // private reduction value and to combine the result. Preserve copy semantics
497 // when materializing reductions on compute constructs, before the
498 // acc.reduction operation carrying that intent is erased. Loop reductions
499 // are excluded: their original variable can be an outer private reduction
500 // value rather than a host variable. Keep acc.reduction referring to its
501 // original host variable and pass the mapped value to materialization
502 // separately.
503 struct ReductionMapping {
504 Value originalVar;
505 Value mappedVar;
506 };
507 SmallVector<ReductionMapping> mappedReductionVars;
508 if constexpr (!std::is_same_v<OpTy, acc::LoopOp>) {
509 for (Value dataOperand : accOp.getDataClauseOperands()) {
510 Operation *dataOp = dataOperand.getDefiningOp();
511 if (dataOp && isa<ACC_DATA_ENTRY_OPS>(Val: dataOp))
512 mappedReductionVars.push_back({acc::getVar(accDataClauseOp: dataOp), dataOperand});
513 }
514 }
515
516 auto getMappedReductionVar = [&](acc::ReductionOp reductionOp) -> Value {
517 if constexpr (std::is_same_v<OpTy, acc::LoopOp>) {
518 return reductionOp.getVar();
519 } else {
520 Value originalVar = reductionOp.getVar();
521 if (isa_and_nonnull<ACC_DATA_ENTRY_OPS>(Val: originalVar.getDefiningOp()))
522 return originalVar;
523
524 // Note that we do not require matching bounds here.
525 // Bounds may be represented by different SSA values while evaluating to
526 // the same values at runtime. Data clauses for the same variable are
527 // expected to specify matching bounds.
528 auto existing = llvm::find_if(mappedReductionVars,
529 [&](const ReductionMapping &mapping) {
530 return mapping.originalVar == originalVar;
531 });
532 if (existing != mappedReductionVars.end())
533 return existing->mappedVar;
534
535 OpBuilder builder(reductionOp);
536 acc::CopyinOp copyinOp;
537 if (std::optional<StringRef> name = reductionOp.getName())
538 copyinOp =
539 acc::CopyinOp::create(builder, location: reductionOp.getLoc(), var: originalVar,
540 /*structured=*/true, /*implicit=*/true, name: *name,
541 bounds: reductionOp.getBounds());
542 else
543 copyinOp = acc::CopyinOp::create(
544 builder, location: reductionOp.getLoc(), var: originalVar,
545 /*structured=*/true, /*implicit=*/true, bounds: reductionOp.getBounds());
546 copyinOp.setDataClause(acc::DataClause::acc_reduction);
547 accOp.getDataClauseOperandsMutable().append(copyinOp.getAccVar());
548
549 builder.setInsertionPointAfter(accOp);
550 acc::CopyoutOp copyoutOp;
551 if (std::optional<StringRef> name = reductionOp.getName())
552 copyoutOp = acc::CopyoutOp::create(
553 builder, location: reductionOp.getLoc(), accVar: copyinOp.getAccVar(), var: originalVar,
554 /*structured=*/true, /*implicit=*/true, name: *name,
555 bounds: reductionOp.getBounds());
556 else
557 copyoutOp = acc::CopyoutOp::create(
558 builder, location: reductionOp.getLoc(), accVar: copyinOp.getAccVar(), var: originalVar,
559 /*structured=*/true, /*implicit=*/true, bounds: reductionOp.getBounds());
560 copyoutOp.setDataClause(acc::DataClause::acc_reduction);
561
562 mappedReductionVars.push_back({originalVar, copyinOp.getAccVar()});
563 return copyinOp.getAccVar();
564 }
565 };
566
567 if (!accOp.getFirstprivateOperands().empty()) {
568 // Clear the firstprivate operands list so there will be no uses after
569 // the recipe is materialized.
570 SmallVector<Value> operands(accOp.getFirstprivateOperands());
571 accOp.getFirstprivateOperandsMutable().clear();
572 for (Value operand : operands) {
573 auto firstprivateOp = cast<acc::FirstprivateOp>(Val: operand.getDefiningOp());
574 auto symbolRef = cast<SymbolRefAttr>(Val: firstprivateOp.getRecipeAttr());
575 auto decl = SymbolTable::lookupNearestSymbolFrom(accOp, symbolRef);
576 auto recipeOp = cast<acc::FirstprivateRecipeOp>(decl);
577 LLVM_DEBUG(llvm::dbgs() << "materializing: " << firstprivateOp << "\n"
578 << symbolRef << "\n");
579 handleInitialValueMapping(op: firstprivateOp);
580 if (failed(
581 materialize(firstprivateOp, recipeOp, accOp, accSupport, policy)))
582 return failure();
583 }
584 }
585
586 if (!accOp.getPrivateOperands().empty()) {
587 // Clear the private operands list so there will be no uses after
588 // the recipe is materialized.
589 SmallVector<Value> operands(accOp.getPrivateOperands());
590 accOp.getPrivateOperandsMutable().clear();
591 for (Value operand : operands) {
592 auto privateOp = cast<acc::PrivateOp>(Val: operand.getDefiningOp());
593 auto symbolRef = cast<SymbolRefAttr>(Val: privateOp.getRecipeAttr());
594 auto decl = SymbolTable::lookupNearestSymbolFrom(accOp, symbolRef);
595 auto recipeOp = cast<acc::PrivateRecipeOp>(decl);
596 LLVM_DEBUG(llvm::dbgs() << "materializing: " << privateOp << "\n"
597 << symbolRef << "\n");
598 if (readsVar(recipeOp.getInitRegion()) ||
599 readsVar(recipeOp.getDestroyRegion()))
600 handleInitialValueMapping(op: privateOp);
601 if (failed(materialize(privateOp, recipeOp, accOp, accSupport, policy)))
602 return failure();
603 }
604 }
605
606 if (!accOp.getReductionOperands().empty()) {
607 // Clear the reduction operands list so there will be no uses after
608 // the recipe is materialized.
609 SmallVector<Value> operands(accOp.getReductionOperands());
610 accOp.getReductionOperandsMutable().clear();
611 for (Value operand : operands) {
612 auto reductionOp = cast<acc::ReductionOp>(Val: operand.getDefiningOp());
613 auto symbolRef = cast<SymbolRefAttr>(Val: reductionOp.getRecipeAttr());
614 auto decl = SymbolTable::lookupNearestSymbolFrom(accOp, symbolRef);
615 auto recipeOp = cast<acc::ReductionRecipeOp>(decl);
616 LLVM_DEBUG(llvm::dbgs() << "materializing: " << reductionOp << "\n"
617 << symbolRef << "\n");
618 Value mappedVar = getMappedReductionVar(reductionOp);
619 if (failed(materialize(reductionOp, recipeOp, accOp, accSupport, policy,
620 mappedVar)))
621 return failure();
622 }
623 }
624 return success();
625}
626
627void ACCRecipeMaterialization::runOnOperation() {
628 ModuleOp moduleOp = getOperation();
629 acc::OpenACCSupport &accSupport = getAnalysis<acc::OpenACCSupport>();
630
631 acc::DefaultACCToGPUMappingPolicy policy;
632
633 // Materialize all recipes for all compute constructs and loop constructs.
634 bool anyFailed = false;
635 moduleOp.walk(callback: [&](Operation *op) {
636 if (anyFailed)
637 return;
638 TypeSwitch<Operation *>(op).Case<ACC_COMPUTE_CONSTRUCT_AND_LOOP_OPS>(
639 caseFn: [&](auto constructOp) {
640 if (failed(materializeForACCOp(constructOp, accSupport, policy)))
641 anyFailed = true;
642 });
643 });
644 if (anyFailed) {
645 signalPassFailure();
646 return;
647 }
648
649 // Remove all recipes. Gather the symbol uses that are left with a single
650 // walk: asking whether each recipe is still referenced walks the whole module
651 // again for every recipe, and recipes are generated per type, so there can be
652 // many of them.
653 std::optional<llvm::DenseSet<StringAttr>> usedSymbols;
654 // The module region, not the module op, is the symbol table scope: asking
655 // for the uses on the op itself would not walk into the body.
656 if (std::optional<SymbolTable::UseRange> uses =
657 SymbolTable::getSymbolUses(from: &moduleOp.getBodyRegion())) {
658 usedSymbols.emplace();
659 for (const SymbolTable::SymbolUse &use : *uses)
660 usedSymbols->insert(V: use.getSymbolRef().getLeafReference());
661 }
662
663 moduleOp.walk(callback: [&](Operation *op) {
664 if (auto recipe = dyn_cast<acc::ReductionRecipeOp>(Val: op))
665 removeRecipe(op: recipe, moduleOp, usedSymbols);
666 else if (auto recipe = dyn_cast<acc::PrivateRecipeOp>(Val: op))
667 removeRecipe(op: recipe, moduleOp, usedSymbols);
668 else if (auto recipe = dyn_cast<acc::FirstprivateRecipeOp>(Val: op))
669 removeRecipe(op: recipe, moduleOp, usedSymbols);
670 });
671}
672} // namespace
673