1//===- ACCDataDirectivePatterns.cpp - ACC data to LLVM ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Lowering of the OpenACC data constructs - data, enter data, exit data,
10// update and declare - to the data entry points of the OpenACC runtime. Each
11// construct becomes one call per point where it establishes or tears down its
12// mappings, which carries the argument arrays its operands are packed into,
13// the queue its async clause selects and, before them, the wait its wait
14// clause asks for. An if clause guards the calls it applies to, and the body
15// of a structured construct is left in place of the construct.
16//
17//===----------------------------------------------------------------------===//
18
19#include "mlir/Conversion/OpenACCToLLVM/ACCDataRuntime.h"
20#include "mlir/Conversion/OpenACCToLLVM/ACCToLLVM.h"
21#include "mlir/Conversion/OpenACCToLLVM/ACCToLLVMUtils.h"
22
23#include "mlir/Conversion/LLVMCommon/Pattern.h"
24#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
25#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
26#include "mlir/Dialect/OpenACC/OpenACC.h"
27#include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h"
28#include "mlir/Interfaces/FunctionInterfaces.h"
29
30using namespace mlir;
31using namespace mlir::acc;
32
33/// Emits the call to the mapping entry point \p fn for \p mappingOperands, the
34/// data-clause operands of a data construct, paired with their type converted
35/// values in \p convertedOperands. \p regionFlags describe the data region
36/// the call opens or closes.
37static LogicalResult emitDataRuntimeCall(
38 Location loc, RuntimeFunction fn, ValueRange mappingOperands,
39 ValueRange convertedOperands, ACCDataCallKind callKind,
40 ConversionPatternRewriter &rewriter, OpenACCSupport &accSupport,
41 Region &globalSymbolRegion, SymbolTable &symbolTable,
42 const ACCRuntimeCallConfig &config, ArrayRef<Value> trailingArguments,
43 ACCDataRegionFlags regionFlags = ACCDataRegionFlags::None) {
44 ACCDataCallOptions options;
45 options.callKind = callKind;
46 options.flags = static_cast<int64_t>(regionFlags);
47 return emitACCDataRuntimeCall(
48 loc, fn, mappingOperands, convertedOperands, rewriter, accSupport,
49 globalSymbolRegion, symbolTable, config, trailingArguments, options);
50}
51
52/// Returns the action that the declare-action recipe holding \p op carries
53/// out, which the recipe states by naming itself in the matching slot of its
54/// `acc.declare_action` attribute. Returns a null attribute when \p op is not
55/// in such a recipe.
56static DeclareActionAttr getEnclosingDeclareAction(Operation *op) {
57 auto func = dyn_cast<FunctionOpInterface>(Val: op);
58 if (!func)
59 func = op->getParentOfType<FunctionOpInterface>();
60 if (!func)
61 return {};
62 return func->getAttrOfType<DeclareActionAttr>(name: getDeclareActionAttrName());
63}
64
65static Value getDeclareAsyncQueue(Location loc,
66 ConversionPatternRewriter &rewriter,
67 const ACCRuntimeCallConfig &config) {
68 return LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI64Type(),
69 value: config.getAsyncSyncRuntimeValue());
70}
71
72namespace {
73template <typename OpTy>
74struct ACCDataDirectivePattern : public ConvertOpToLLVMPattern<OpTy> {
75 ACCDataDirectivePattern(const LLVMTypeConverter &converter,
76 OpenACCSupport &accSupport,
77 Region &globalSymbolRegion, SymbolTable &symbolTable,
78 const ACCRuntimeCallConfig &config,
79 DeviceType clauseDeviceType,
80 PatternBenefit benefit = 1)
81 : ConvertOpToLLVMPattern<OpTy>(converter, benefit),
82 accSupport(accSupport), globalSymbolRegion(globalSymbolRegion),
83 symbolTable(symbolTable), config(config),
84 clauseDeviceType(clauseDeviceType) {}
85
86 /// Returns the queue that the mapping calls of \p op run on.
87 Value getAsyncQueue(OpTy op, ConversionPatternRewriter &rewriter) const {
88 return acc::getAsyncQueue(op, clauseDeviceType, rewriter, config);
89 }
90
91 /// Emits the wait that a wait clause on \p op asks for before its mapping
92 /// calls.
93 LogicalResult emitWaitClause(OpTy op, Value asyncQueue,
94 ConversionPatternRewriter &rewriter) const {
95 return acc::emitWaitClause(op, clauseDeviceType, asyncQueue, rewriter,
96 accSupport, globalSymbolRegion, symbolTable,
97 config);
98 }
99
100 /// Emits one mapping call for the data-clause operands of \p op.
101 LogicalResult emitMappingCall(OpTy op, ValueRange convertedOperands,
102 Location loc, RuntimeFunction fn,
103 ACCDataCallKind callKind, Value asyncQueue,
104 ConversionPatternRewriter &rewriter) const {
105 return emitDataRuntimeCall(loc, fn, op.getDataClauseOperands(),
106 convertedOperands, callKind, rewriter,
107 accSupport, globalSymbolRegion, symbolTable,
108 config, {asyncQueue});
109 }
110
111 OpenACCSupport &accSupport;
112 Region &globalSymbolRegion;
113 SymbolTable &symbolTable;
114 ACCRuntimeCallConfig config;
115 DeviceType clauseDeviceType;
116};
117
118struct DataOpLowering : public ACCDataDirectivePattern<DataOp> {
119 using ACCDataDirectivePattern<DataOp>::ACCDataDirectivePattern;
120
121 LogicalResult
122 matchAndRewrite(DataOp op, DataOp::Adaptor adaptor,
123 ConversionPatternRewriter &rewriter) const override {
124 // The mappings are established where the clause operations are, and torn
125 // down where the exit operations of the construct were.
126 ValueRange mappingOperands = op.getDataClauseOperands();
127 Location beginLoc = mappingOperands.empty()
128 ? op.getLoc()
129 : mappingOperands.front().getLoc();
130 Location endLoc =
131 acc::getMappingExitLoc(dataClauseOperands: mappingOperands).value_or(u&: beginLoc);
132 Value asyncQueue = getAsyncQueue(op, rewriter);
133 if (failed(Result: emitWaitClause(op, asyncQueue, rewriter)))
134 return failure();
135
136 auto emitBegin = [&]() {
137 return emitMappingCall(op, convertedOperands: adaptor.getDataClauseOperands(), loc: beginLoc,
138 fn: RuntimeFunction::ACCRTL_tgt_acc_data_begin,
139 callKind: ACCDataCallKind::DataEnter, asyncQueue, rewriter);
140 };
141 if (failed(Result: acc::emitGuardedByIfCond(loc: beginLoc, ifCond: adaptor.getIfCond(), rewriter,
142 emitFn: emitBegin)))
143 return failure();
144
145 rewriter.setInsertionPointAfter(op);
146 auto emitEnd = [&]() {
147 return emitMappingCall(op, convertedOperands: adaptor.getDataClauseOperands(), loc: endLoc,
148 fn: RuntimeFunction::ACCRTL_tgt_acc_data_end,
149 callKind: ACCDataCallKind::DataExit, asyncQueue, rewriter);
150 };
151 if (failed(Result: acc::emitGuardedByIfCond(loc: endLoc, ifCond: adaptor.getIfCond(), rewriter,
152 emitFn: emitEnd)))
153 return failure();
154
155 acc::spliceConstructRegion(op, region&: op.getRegion(), rewriter);
156 rewriter.eraseOp(op);
157 return success();
158 }
159};
160
161/// Lowering shared by the directives that map their operands with a single
162/// call: enter data, exit data and update.
163template <typename OpTy, RuntimeFunction fn, ACCDataCallKind callKind>
164struct ACCDataCallLowering : public ACCDataDirectivePattern<OpTy> {
165 using ACCDataDirectivePattern<OpTy>::ACCDataDirectivePattern;
166
167 LogicalResult
168 matchAndRewrite(OpTy op, typename OpTy::Adaptor adaptor,
169 ConversionPatternRewriter &rewriter) const override {
170 Location loc = op.getLoc();
171 Value asyncQueue = this->getAsyncQueue(op, rewriter);
172 if (failed(this->emitWaitClause(op, asyncQueue, rewriter)))
173 return failure();
174
175 auto emit = [&]() {
176 return this->emitMappingCall(op, adaptor.getDataClauseOperands(), loc, fn,
177 callKind, asyncQueue, rewriter);
178 };
179 if (failed(
180 acc::emitGuardedByIfCond(loc, ifCond: adaptor.getIfCond(), rewriter, emitFn: emit)))
181 return failure();
182 rewriter.eraseOp(op);
183 return success();
184 }
185};
186
187using EnterDataOpLowering =
188 ACCDataCallLowering<EnterDataOp, RuntimeFunction::ACCRTL_tgt_acc_data_enter,
189 ACCDataCallKind::DataEnter>;
190using ExitDataOpLowering =
191 ACCDataCallLowering<ExitDataOp, RuntimeFunction::ACCRTL_tgt_acc_data_exit,
192 ACCDataCallKind::DataExit>;
193using UpdateOpLowering =
194 ACCDataCallLowering<UpdateOp, RuntimeFunction::ACCRTL_tgt_acc_data_update,
195 ACCDataCallKind::DataEnter>;
196
197/// Structured `acc.declare_enter` (token used by a matching exit) maps like
198/// `acc.data` enter, with the region flags telling the runtime that a declare
199/// directive opened the region. In a declare-action recipe the object the
200/// directive names is allocated by the host, so only the device mirror of it is
201/// allocated here. An unstructured enter registers the objects it names with
202/// `__tgt_acc_declare`.
203struct DeclareEnterOpLowering : public ConvertOpToLLVMPattern<DeclareEnterOp> {
204 DeclareEnterOpLowering(const LLVMTypeConverter &converter,
205 OpenACCSupport &accSupport, Region &globalSymbolRegion,
206 SymbolTable &symbolTable,
207 const ACCRuntimeCallConfig &config,
208 PatternBenefit benefit = 1)
209 : ConvertOpToLLVMPattern<DeclareEnterOp>(converter, benefit),
210 accSupport(accSupport), globalSymbolRegion(globalSymbolRegion),
211 symbolTable(symbolTable), config(config) {}
212
213 LogicalResult
214 matchAndRewrite(DeclareEnterOp op, DeclareEnterOp::Adaptor adaptor,
215 ConversionPatternRewriter &rewriter) const override {
216 Location loc = op.getLoc();
217 ValueRange mappingOperands = op.getDataClauseOperands();
218 ValueRange convertedOperands = adaptor.getDataClauseOperands();
219 Value asyncQueue = getDeclareAsyncQueue(loc, rewriter, config);
220
221 LogicalResult emitted = success();
222 if (getEnclosingDeclareAction(op)) {
223 emitted = emitDataRuntimeCall(
224 loc, fn: RuntimeFunction::ACCRTL_tgt_acc_mirror_alloc, mappingOperands,
225 convertedOperands, callKind: ACCDataCallKind::DataEnter, rewriter, accSupport,
226 globalSymbolRegion, symbolTable, config, trailingArguments: {});
227 } else if (!op.getToken().use_empty()) {
228 emitted = emitDataRuntimeCall(
229 loc, fn: RuntimeFunction::ACCRTL_tgt_acc_data_begin, mappingOperands,
230 convertedOperands, callKind: ACCDataCallKind::DataEnter, rewriter, accSupport,
231 globalSymbolRegion, symbolTable, config, trailingArguments: {asyncQueue},
232 regionFlags: ACCDataRegionFlags::Declare);
233 } else {
234 Value binaryDescriptor =
235 config.createDeclareBinaryDescriptor(loc, builder&: rewriter);
236 emitted = emitDataRuntimeCall(
237 loc, fn: RuntimeFunction::ACCRTL_tgt_acc_declare, mappingOperands,
238 convertedOperands, callKind: ACCDataCallKind::DataEnter, rewriter, accSupport,
239 globalSymbolRegion, symbolTable, config,
240 trailingArguments: {asyncQueue, binaryDescriptor});
241 }
242 if (failed(Result: emitted))
243 return failure();
244
245 if (op.getToken().use_empty()) {
246 rewriter.eraseOp(op);
247 } else {
248 // The token only ties a structured enter to its exit. Keep a converted
249 // placeholder until the exit is erased; it carries no runtime state.
250 Type tokenTy = getTypeConverter()->convertType(t: op.getToken().getType());
251 rewriter.replaceOp(op, newOp: LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: tokenTy));
252 }
253 return success();
254 }
255
256 OpenACCSupport &accSupport;
257 Region &globalSymbolRegion;
258 SymbolTable &symbolTable;
259 ACCRuntimeCallConfig config;
260};
261
262/// A recipe that runs before the host object is deallocated frees the device
263/// mirror of it, as the mapping is torn down once the host storage is gone.
264/// Every other `acc.declare_exit` ends the mapping with `__tgt_acc_data_end`;
265/// a structured exit (one holding the token of its enter) carries the declare
266/// region flag, as its enter did.
267struct DeclareExitOpLowering : public ConvertOpToLLVMPattern<DeclareExitOp> {
268 DeclareExitOpLowering(const LLVMTypeConverter &converter,
269 OpenACCSupport &accSupport, Region &globalSymbolRegion,
270 SymbolTable &symbolTable,
271 const ACCRuntimeCallConfig &config,
272 PatternBenefit benefit = 1)
273 : ConvertOpToLLVMPattern<DeclareExitOp>(converter, benefit),
274 accSupport(accSupport), globalSymbolRegion(globalSymbolRegion),
275 symbolTable(symbolTable), config(config) {}
276
277 LogicalResult
278 matchAndRewrite(DeclareExitOp op, DeclareExitOp::Adaptor adaptor,
279 ConversionPatternRewriter &rewriter) const override {
280 Location loc = op.getLoc();
281 ValueRange mappingOperands = op.getDataClauseOperands();
282 ValueRange convertedOperands = adaptor.getDataClauseOperands();
283 Value asyncQueue = getDeclareAsyncQueue(loc, rewriter, config);
284
285 LogicalResult emitted = success();
286 DeclareActionAttr action = getEnclosingDeclareAction(op);
287 if (action && action.getPreDealloc()) {
288 emitted = emitDataRuntimeCall(
289 loc, fn: RuntimeFunction::ACCRTL_tgt_acc_mirror_dealloc, mappingOperands,
290 convertedOperands, callKind: ACCDataCallKind::DataEnter, rewriter, accSupport,
291 globalSymbolRegion, symbolTable, config, trailingArguments: {});
292 } else {
293 ACCDataRegionFlags regionFlags = op.getToken()
294 ? ACCDataRegionFlags::Declare
295 : ACCDataRegionFlags::None;
296 emitted = emitDataRuntimeCall(
297 loc, fn: RuntimeFunction::ACCRTL_tgt_acc_data_end, mappingOperands,
298 convertedOperands, callKind: ACCDataCallKind::DataExit, rewriter, accSupport,
299 globalSymbolRegion, symbolTable, config, trailingArguments: {asyncQueue}, regionFlags);
300 }
301 if (failed(Result: emitted))
302 return failure();
303 rewriter.eraseOp(op);
304 return success();
305 }
306
307 OpenACCSupport &accSupport;
308 Region &globalSymbolRegion;
309 SymbolTable &symbolTable;
310 ACCRuntimeCallConfig config;
311};
312
313/// A data clause operation only describes a mapping; the runtime calls of the
314/// construct carry everything it said, so the operation itself goes away once
315/// they are in place. Whatever else referred to the mapped object is given the
316/// address of that object, which is what the clause result stood for.
317template <typename OpTy>
318struct DataEntryOpLowering : public ConvertOpToLLVMPattern<OpTy> {
319 using ConvertOpToLLVMPattern<OpTy>::ConvertOpToLLVMPattern;
320 using OpAdaptor = typename OpTy::Adaptor;
321
322 LogicalResult
323 matchAndRewrite(OpTy op, OpAdaptor adaptor,
324 ConversionPatternRewriter &rewriter) const override {
325 rewriter.replaceOp(op, adaptor.getVar());
326 return success();
327 }
328};
329
330/// An `acc.getdeviceptr` clause stands for the device address of an object
331/// whose mapping was established elsewhere. Where it names the object of a
332/// data exit operation, the runtime call of that construct carries the host
333/// address and looks the device one up itself, so the clause is replaced by
334/// the address of the object as any other data entry clause is. Read as a
335/// value, it is the runtime that has to be asked for the address.
336struct GetDevicePtrOpLowering
337 : public ConvertOpToLLVMPattern<acc::GetDevicePtrOp> {
338 GetDevicePtrOpLowering(const LLVMTypeConverter &converter,
339 OpenACCSupport &accSupport, Region &globalSymbolRegion,
340 SymbolTable &symbolTable,
341 const ACCRuntimeCallConfig &config,
342 PatternBenefit benefit = 1)
343 : ConvertOpToLLVMPattern<acc::GetDevicePtrOp>(converter, benefit),
344 accSupport(accSupport), globalSymbolRegion(globalSymbolRegion),
345 symbolTable(symbolTable), config(config) {}
346
347 /// Returns whether \p user only states the mapping the clause describes,
348 /// which is what a data exit operation and the construct holding the clause
349 /// do.
350 static bool isMappingUse(Operation *user) {
351 return isa<ACC_DATA_EXIT_OPS, ACC_COMPUTE_AND_DATA_CONSTRUCT_OPS,
352 acc::KernelEnvironmentOp>(Val: user);
353 }
354
355 LogicalResult
356 matchAndRewrite(acc::GetDevicePtrOp op, OpAdaptor adaptor,
357 ConversionPatternRewriter &rewriter) const override {
358 bool readAsValue = llvm::any_of(
359 Range: op->getUsers(), P: [](Operation *user) { return !isMappingUse(user); });
360 if (!readAsValue) {
361 rewriter.replaceOp(op, newValues: adaptor.getVar());
362 return success();
363 }
364 // The mapping hands the host address of the object to the runtime call of
365 // the construct, so the clause cannot stand for the device address at the
366 // same time.
367 if (llvm::any_of(Range: op->getUsers(), P: isMappingUse)) {
368 (void)accSupport.emitNYI(
369 loc: op.getLoc(), message: "device address read from a clause that also states a "
370 "mapping of the object");
371 return failure();
372 }
373
374 FailureOr<Value> devicePtr = acc::emitGetDevicePtrCall(
375 clauseOp: op, hostPtr: adaptor.getVar(), /*ifPresent=*/false, builder&: rewriter, globalSymbolRegion,
376 symbolTable, config);
377 if (failed(Result: devicePtr))
378 return failure();
379 rewriter.replaceOp(op, newValues: *devicePtr);
380 return success();
381 }
382
383 OpenACCSupport &accSupport;
384 Region &globalSymbolRegion;
385 SymbolTable &symbolTable;
386 ACCRuntimeCallConfig config;
387};
388
389/// A data exit operation and the bounds of a mapping have no result to stand
390/// in for, so they are simply removed with the construct that used them.
391template <typename OpTy>
392struct EraseDataClauseOp : public ConvertOpToLLVMPattern<OpTy> {
393 using ConvertOpToLLVMPattern<OpTy>::ConvertOpToLLVMPattern;
394 using OpAdaptor = typename OpTy::Adaptor;
395
396 LogicalResult
397 matchAndRewrite(OpTy op, OpAdaptor,
398 ConversionPatternRewriter &rewriter) const override {
399 rewriter.eraseOp(op);
400 return success();
401 }
402};
403
404} // namespace
405
406void mlir::configureACCDataDirectiveConversionLegality(
407 ConversionTarget &target) {
408 // The map entries themselves stay legal: a construct removes the ones it
409 // consumes, and the ones describing a construct that is lowered elsewhere
410 // have to survive this conversion. An `acc.getdeviceptr` read as a value is
411 // the exception, as the address it stands for is only known to the runtime,
412 // so leaving it behind would state a mapping that nothing carries out.
413 target.addIllegalOp<acc::DataOp, acc::EnterDataOp, acc::ExitDataOp,
414 acc::UpdateOp, acc::DeclareEnterOp, acc::DeclareExitOp,
415 acc::GetDevicePtrOp>();
416}
417
418void mlir::populateACCDataClauseOpPatterns(
419 LLVMTypeConverter &converter, RewritePatternSet &patterns,
420 acc::OpenACCSupport &accSupport, Region &globalSymbolRegion,
421 SymbolTable &symbolTable, const acc::ACCRuntimeCallConfig &config) {
422 // Bounds describe a mapping rather than data, and they are removed together
423 // with the clause operations holding them, so their type only has to survive
424 // this conversion.
425 converter.addConversion(
426 callback: [](acc::DataBoundsType type) -> Type { return type; });
427
428 patterns.add<
429 DataEntryOpLowering<acc::MapInfoOp>, DataEntryOpLowering<acc::CopyinOp>,
430 DataEntryOpLowering<acc::CreateOp>, DataEntryOpLowering<acc::PresentOp>,
431 DataEntryOpLowering<acc::NoCreateOp>, DataEntryOpLowering<acc::AttachOp>,
432 DataEntryOpLowering<acc::DevicePtrOp>,
433 DataEntryOpLowering<acc::UpdateDeviceOp>,
434 DataEntryOpLowering<acc::PrivateOp>,
435 DataEntryOpLowering<acc::FirstprivateOp>,
436 DataEntryOpLowering<acc::FirstprivateMapInitialOp>,
437 DataEntryOpLowering<acc::DeclareDeviceResidentOp>,
438 DataEntryOpLowering<acc::DeclareLinkOp>,
439 EraseDataClauseOp<acc::CopyoutOp>, EraseDataClauseOp<acc::DeleteOp>,
440 EraseDataClauseOp<acc::DetachOp>, EraseDataClauseOp<acc::UpdateHostOp>,
441 EraseDataClauseOp<acc::DataBoundsOp>>(arg&: converter);
442 patterns.add<GetDevicePtrOpLowering>(arg&: converter, args&: accSupport,
443 args&: globalSymbolRegion, args&: symbolTable, args: config);
444}
445
446void mlir::populateACCDataDirectivePatterns(
447 LLVMTypeConverter &converter, RewritePatternSet &patterns,
448 acc::OpenACCSupport &accSupport, Region &globalSymbolRegion,
449 SymbolTable &symbolTable, const acc::ACCRuntimeCallConfig &config,
450 acc::DeviceType clauseDeviceType) {
451 converter.addConversion(callback: [](acc::DeclareTokenType type) -> Type {
452 return LLVM::LLVMPointerType::get(context: type.getContext());
453 });
454
455 patterns.add<DataOpLowering, EnterDataOpLowering, ExitDataOpLowering,
456 UpdateOpLowering>(arg&: converter, args&: accSupport, args&: globalSymbolRegion,
457 args&: symbolTable, args: config, args&: clauseDeviceType);
458 patterns.add<DeclareEnterOpLowering, DeclareExitOpLowering>(
459 arg&: converter, args&: accSupport, args&: globalSymbolRegion, args&: symbolTable, args: config);
460}
461