1//===- ACCKernelLaunchPatterns.cpp - Kernel launch to LLVM ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Lower acc.kernel_environment and the gpu.launch_func operations in its body
10// together. The environment establishes one implicit data region. Its entry
11// call returns the device addresses that every contained kernel launch uses.
12//
13//===----------------------------------------------------------------------===//
14
15#include "mlir/Conversion/OpenACCToLLVM/ACCDataRuntime.h"
16#include "mlir/Conversion/OpenACCToLLVM/ACCToLLVM.h"
17#include "mlir/Conversion/OpenACCToLLVM/ACCToLLVMUtils.h"
18
19#include "mlir/Conversion/LLVMCommon/Pattern.h"
20#include "mlir/Dialect/Func/IR/FuncOps.h"
21#include "mlir/Dialect/GPU/IR/GPUDialect.h"
22#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
23#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
24#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
25#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
26#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
27#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
28#include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h"
29#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
30#include "mlir/Dialect/OpenACC/OpenACC.h"
31#include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h"
32#include "mlir/Dialect/OpenACC/OpenACCUtilsType.h"
33#include "mlir/IR/BuiltinOps.h"
34#include "llvm/ADT/DenseMap.h"
35#include "llvm/ADT/STLExtras.h"
36#include "llvm/Support/raw_ostream.h"
37
38#include <array>
39
40using namespace mlir;
41using namespace mlir::acc;
42
43static bool isDevicePtrMapping(Value operand) {
44 Operation *mapOp = operand.getDefiningOp();
45 if (!mapOp)
46 return false;
47 if (std::optional<MapFlags> flags = getMapFlags(mapEntryOp: mapOp))
48 return bitEnumContainsAny(bits: *flags, bit: MapFlags::devptr);
49 if (std::optional<DataClause> clause = getDataClause(accDataEntryOp: mapOp))
50 return *clause == DataClause::acc_deviceptr;
51 return false;
52}
53
54static bool isDataEntryInsideEnvironment(KernelEnvironmentOp environment,
55 Value operand) {
56 Operation *mapOp = operand.getDefiningOp();
57 return isDataEntryOp(op: mapOp) && environment->isAncestor(other: mapOp);
58}
59
60/// The flags of a data clause operation that maps a kernel argument. A kernel
61/// argument only looks up the mapping its clause established, so the clauses
62/// that establish or tear down something else contribute no flag for it.
63static MapFlags getKernelDataClauseMapFlags(Operation *entryOp) {
64 bool ptrAndObj = hasAttachPoint(mapEntryOp: entryOp);
65 MapFlags ptrAndObjFlag = ptrAndObj ? MapFlags::ptr_and_obj : MapFlags::none;
66 std::optional<DataClause> clause = getDataClause(accDataEntryOp: entryOp);
67 if (clause == DataClause::acc_declare_device_resident)
68 return MapFlags::present | ptrAndObjFlag;
69 if (clause == DataClause::acc_attach || clause == DataClause::acc_detach)
70 return ptrAndObjFlag;
71 return bitEnumClear(bits: computeDataClauseMapFlags(entryOp, ptrAndObj),
72 bit: MapFlags::device_resident);
73}
74
75/// The flags a kernel argument is passed with. An operand the environment
76/// already mapped is passed as the device address its entry call returned. An
77/// operand that no clause maps is passed by value.
78static MapFlags getKernelArgMapFlags(Value operand, bool alreadyMapped,
79 const ACCRuntimeCallConfig &config) {
80 Operation *mapOp = operand.getDefiningOp();
81 if (!isDataEntryOp(op: mapOp))
82 return mapOp ? config.postProcessMapFlags(mapOp, flags: MapFlags::literal)
83 : MapFlags::literal;
84
85 MapFlags flags;
86 if (alreadyMapped)
87 flags = MapFlags::devptr | MapFlags::literal;
88 else if (auto mapInfo = dyn_cast<MapInfoOp>(Val: mapOp))
89 flags = mapInfo.getMapFlags();
90 else
91 flags = getKernelDataClauseMapFlags(entryOp: mapOp);
92 if (getImplicitFlag(accDataEntryOp: mapOp))
93 flags = flags | MapFlags::implicit;
94 return config.postProcessMapFlags(mapOp, flags);
95}
96
97/// The descriptor of a kernel argument. The device address the environment
98/// returned is passed as is, since the runtime would map an address that
99/// carries a descriptor again.
100static FailureOr<Value>
101getKernelArgDescriptor(Location loc, Value operand, Value converted,
102 bool alreadyMapped, OpenACCSupport &accSupport,
103 ConversionPatternRewriter &rewriter) {
104 Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext());
105 Operation *mapOp = operand.getDefiningOp();
106 if (alreadyMapped)
107 return Value(LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrTy));
108 if (isDataEntryOp(op: mapOp))
109 return createACCArgumentDescriptor(mapOp, convertedOperand: converted, accSupport, rewriter);
110 if (auto memrefType = dyn_cast<MemRefType>(Val: operand.getType())) {
111 ModuleOp module = rewriter.getInsertionBlock()
112 ->getParentOp()
113 ->getParentOfType<ModuleOp>();
114 auto sizeAndAlignment = acc::getTypeSizeAndAlignment(
115 ty: memrefType.getElementType(), module, support: &accSupport);
116 if (!sizeAndAlignment) {
117 std::string message;
118 llvm::raw_string_ostream os(message);
119 os << "unhandled type size computation for type: "
120 << memrefType.getElementType();
121 (void)accSupport.emitNYI(loc, message);
122 return failure();
123 }
124 return createACCMemRefDescriptorWrapperArg(
125 loc, memrefType, convertedMemref: converted, /*bounds=*/{},
126 elementSize: constantI64(loc, value: sizeAndAlignment->first.getFixedValue(), builder&: rewriter),
127 rewriter);
128 }
129 return Value(LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrTy));
130}
131
132/// Emits the argument arrays of the kernel launched by \p launch, one entry per
133/// kernel operand. Each operand is one parameter of the kernel.
134static FailureOr<ACCDataRuntimeArgs> emitKernelRuntimeArgs(
135 gpu::LaunchFuncOp launch, gpu::LaunchFuncOpAdaptor adaptor,
136 ValueRange alreadyMappedOperands, ConversionPatternRewriter &rewriter,
137 OpenACCSupport &accSupport, Region &globalSymbolRegion,
138 SymbolTable &symbolTable, const ACCRuntimeCallConfig &config) {
139 Location loc = launch.getLoc();
140 SmallVector<ACCMappedObjectArgs> objects;
141 for (auto [operand, converted] : llvm::zip_equal(
142 t: launch.getKernelOperands(), u: adaptor.getKernelOperands())) {
143 bool alreadyMapped = llvm::is_contained(Range&: alreadyMappedOperands, Element: operand);
144 MapFlags mapFlags = getKernelArgMapFlags(operand, alreadyMapped, config);
145 Operation *mapOp = operand.getDefiningOp();
146
147 ACCMappedObjectArgs args;
148 // A kernel argument states the base of the mapping it looks up, which for
149 // an object the declare directive made resident is that of its entry.
150 args.base = createACCMappedObjectBase(
151 loc, mapOp: isDataEntryOp(op: mapOp) ? mapOp : nullptr, mapFlags, rewriter);
152 args.pointer = createACCMappedObjectPointer(loc, operand, converted,
153 mapFlags, rewriter);
154 FailureOr<Value> size = createACCMappedObjectSize(
155 loc, operand, converted, mapFlags, accSupport, rewriter);
156 if (failed(Result: size))
157 return failure();
158 args.size = *size;
159 args.type =
160 constantI64(loc, value: config.getMapFlagsRuntimeValue(flags: mapFlags), builder&: rewriter);
161 args.name = createACCMappedObjectName(
162 loc, operand, accSupport, globalSymbolRegion, symbolTable: &symbolTable, rewriter);
163 FailureOr<Value> descriptor = getKernelArgDescriptor(
164 loc, operand, converted, alreadyMapped, accSupport, rewriter);
165 if (failed(Result: descriptor))
166 return failure();
167 args.descriptor = *descriptor;
168 objects.push_back(Elt: args);
169 }
170
171 ACCDataRuntimeArgs runtimeArgs;
172 createACCMappedObjectArrays(loc, objects, runtimeArgs, rewriter);
173 return runtimeArgs;
174}
175
176static FailureOr<Value> createKernelArgumentStruct(
177 gpu::LaunchFuncOp launch, gpu::LaunchFuncOpAdaptor adaptor,
178 ValueRange alreadyMappedOperands, ConversionPatternRewriter &rewriter,
179 OpenACCSupport &accSupport, Region &globalSymbolRegion,
180 SymbolTable &symbolTable, const ACCRuntimeCallConfig &config) {
181 FailureOr<ACCDataRuntimeArgs> emitted = emitKernelRuntimeArgs(
182 launch, adaptor, alreadyMappedOperands, rewriter, accSupport,
183 globalSymbolRegion, symbolTable, config);
184 if (failed(Result: emitted))
185 return failure();
186 ACCDataRuntimeArgs runtimeArgs = *emitted;
187
188 Location loc = launch.getLoc();
189 MLIRContext *context = rewriter.getContext();
190 Type i32Ty = rewriter.getI32Type();
191 Type ptrTy = LLVM::LLVMPointerType::get(context);
192 Type argsType = getKernelArgumentType(ctx: context);
193 Value args = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: argsType);
194 auto insert = [&](Value value, KernelArgumentField field) {
195 args = LLVM::InsertValueOp::create(
196 builder&: rewriter, location: loc, res: argsType, container: args, value,
197 position: ArrayRef<int64_t>{getKernelArgumentFieldIndex(field)});
198 };
199 insert(runtimeArgs.argNum, KernelArgumentField::ArgNum);
200 insert(runtimeArgs.argBasePtrs, KernelArgumentField::ArgBasePtrs);
201 insert(runtimeArgs.argPtrs, KernelArgumentField::ArgPtrs);
202 insert(runtimeArgs.argSizes, KernelArgumentField::ArgSizes);
203 insert(runtimeArgs.argTypes, KernelArgumentField::ArgTypes);
204 insert(runtimeArgs.argNames, KernelArgumentField::ArgNames);
205 insert(runtimeArgs.argMappers, KernelArgumentField::ArgMappers);
206 insert(runtimeArgs.argDescs, KernelArgumentField::ArgDescs);
207
208 // The vector length is the first block dimension. The worker count is the
209 // product of the remaining ones.
210 Value blockY = castToI64(loc, value: adaptor.getBlockSizeY(), builder&: rewriter);
211 Value blockZ = castToI64(loc, value: adaptor.getBlockSizeZ(), builder&: rewriter);
212 insert(LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: blockY, rhs: blockZ),
213 KernelArgumentField::NumWorkers);
214 insert(castToI64(loc, value: adaptor.getBlockSizeX(), builder&: rewriter),
215 KernelArgumentField::VectorLength);
216 Value dynamicSharedMemorySize =
217 adaptor.getDynamicSharedMemorySize()
218 ? castToI64(loc, value: adaptor.getDynamicSharedMemorySize(), builder&: rewriter)
219 : constantI64(loc, value: 0, builder&: rewriter);
220 insert(dynamicSharedMemorySize, KernelArgumentField::SmemSize);
221
222 // The grid sizes are the entries of the array in the NumGangs field.
223 int64_t numGangsField =
224 getKernelArgumentFieldIndex(field: KernelArgumentField::NumGangs);
225 std::array<Value, 3> gridSizes = {
226 adaptor.getGridSizeX(), adaptor.getGridSizeY(), adaptor.getGridSizeZ()};
227 for (auto [dimension, gridSize] : llvm::enumerate(First&: gridSizes))
228 args = LLVM::InsertValueOp::create(
229 builder&: rewriter, location: loc, res: argsType, container: args, value: castToI64(loc, value: gridSize, builder&: rewriter),
230 position: ArrayRef<int64_t>{numGangsField, static_cast<int64_t>(dimension)});
231
232 Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: 1);
233 Value argsPtr = LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: argsType, arraySize: one);
234 LLVM::StoreOp::create(builder&: rewriter, location: loc, value: args, addr: argsPtr);
235 return argsPtr;
236}
237
238static FailureOr<Value> getKernelPointer(gpu::LaunchFuncOp launch,
239 ConversionPatternRewriter &rewriter,
240 Region &globalSymbolRegion,
241 SymbolTable &symbolTable) {
242 Location loc = launch.getLoc();
243 Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext());
244 StringRef kernelName = launch.getKernel().getLeafReference().getValue();
245 if (auto function = symbolTable.lookup<LLVM::LLVMFuncOp>(name: kernelName))
246 return LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: ptrTy, global_name: function.getName())
247 .getResult();
248 if (auto function = symbolTable.lookup<func::FuncOp>(name: kernelName))
249 return LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: ptrTy, global_name: function.getName())
250 .getResult();
251
252 auto gpuFunction = SymbolTable::lookupNearestSymbolFrom<gpu::GPUFuncOp>(
253 from: launch, symbol: launch.getKernel());
254 if (!gpuFunction)
255 return failure();
256
257 {
258 OpBuilder::InsertionGuard guard(rewriter);
259 rewriter.setInsertionPointToStart(&globalSymbolRegion.front());
260 auto functionType = LLVM::LLVMFunctionType::get(
261 result: LLVM::LLVMVoidType::get(ctx: rewriter.getContext()), arguments: {});
262 auto declaration =
263 LLVM::LLVMFuncOp::create(builder&: rewriter, location: loc, name: kernelName, type: functionType);
264 symbolTable.insert(symbol: declaration);
265 }
266 return LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: ptrTy, global_name: kernelName)
267 .getResult();
268}
269
270static void declareLibc(SymbolTable &symbolTable, Region &globalSymbolRegion,
271 Location loc, StringRef name, Type result,
272 ArrayRef<Type> arguments) {
273 if (symbolTable.lookup<LLVM::LLVMFuncOp>(name))
274 return;
275 OpBuilder builder = OpBuilder::atBlockEnd(block: &globalSymbolRegion.front());
276 auto functionType = LLVM::LLVMFunctionType::get(result, arguments);
277 symbolTable.insert(
278 symbol: LLVM::LLVMFuncOp::create(builder, location: loc, name, type: functionType));
279}
280
281/// The kernel entry point reports a status other than success when the kernel
282/// could not run on the device. The program is then meant to run the region on
283/// the host, which is not implemented, so the failure is reported and the
284/// program stops.
285static void emitHostFallbackNotImplemented(Location loc, Value status,
286 ConversionPatternRewriter &rewriter,
287 Region &globalSymbolRegion,
288 SymbolTable &symbolTable) {
289 MLIRContext *context = rewriter.getContext();
290 Type i32Ty = rewriter.getI32Type();
291 Type ptrTy = LLVM::LLVMPointerType::get(context);
292 Type voidTy = LLVM::LLVMVoidType::get(ctx: context);
293
294 Block *preBlock = rewriter.getInsertionBlock();
295 Block *continueBlock =
296 rewriter.splitBlock(block: preBlock, before: rewriter.getInsertionPoint());
297 Block *errorBlock =
298 rewriter.createBlock(parent: preBlock->getParent(), insertPt: continueBlock->getIterator());
299 rewriter.setInsertionPointToEnd(preBlock);
300 Value success = LLVM::ConstantOp::create(
301 builder&: rewriter, location: loc, type: i32Ty, value: static_cast<int32_t>(RuntimeResult::Success));
302 Value failed = LLVM::ICmpOp::create(builder&: rewriter, location: loc, predicate: LLVM::ICmpPredicate::ne,
303 lhs: status, rhs: success);
304 LLVM::CondBrOp::create(builder&: rewriter, location: loc, condition: failed, trueDest: errorBlock, trueOperands: ValueRange{},
305 falseDest: continueBlock, falseOperands: ValueRange{});
306
307 // TODO: Runtime API should provide a mechanism to report this error.
308 rewriter.setInsertionPointToStart(errorBlock);
309 declareLibc(symbolTable, globalSymbolRegion, loc, name: "puts", result: i32Ty, arguments: {ptrTy});
310 declareLibc(symbolTable, globalSymbolRegion, loc, name: "fflush", result: i32Ty, arguments: {ptrTy});
311 declareLibc(symbolTable, globalSymbolRegion, loc, name: "abort", result: voidTy, arguments: {});
312 Value message = getOrCreateGlobalString(
313 loc, builder&: rewriter, name: getInternalGlobalName(kind: "message", detail: "host_fallback_nyi"),
314 value: "host fallback execution is not implemented\n", globalSymbolRegion,
315 symbolTable: &symbolTable);
316 LLVM::CallOp::create(builder&: rewriter, location: loc, results: TypeRange{i32Ty},
317 callee: SymbolRefAttr::get(ctx: context, value: "puts"),
318 args: ValueRange{message});
319 Value nullFile = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrTy);
320 LLVM::CallOp::create(builder&: rewriter, location: loc, results: TypeRange{i32Ty},
321 callee: SymbolRefAttr::get(ctx: context, value: "fflush"),
322 args: ValueRange{nullFile});
323 LLVM::CallOp::create(builder&: rewriter, location: loc, results: TypeRange{},
324 callee: SymbolRefAttr::get(ctx: context, value: "abort"), args: ValueRange{});
325 LLVM::UnreachableOp::create(builder&: rewriter, location: loc);
326 rewriter.setInsertionPointToStart(continueBlock);
327}
328
329static LogicalResult
330emitKernelCall(gpu::LaunchFuncOp launch, gpu::LaunchFuncOpAdaptor adaptor,
331 Value asyncQueue, ValueRange alreadyMappedOperands,
332 ConversionPatternRewriter &rewriter, OpenACCSupport &accSupport,
333 Region &globalSymbolRegion, SymbolTable &symbolTable,
334 const ACCRuntimeCallConfig &config) {
335 Location loc = launch.getLoc();
336 FailureOr<Value> kernelPointer =
337 getKernelPointer(launch, rewriter, globalSymbolRegion, symbolTable);
338 if (failed(Result: kernelPointer)) {
339 (void)accSupport.emitNYI(loc, message: "undefined kernel");
340 return failure();
341 }
342 FailureOr<Value> arguments = createKernelArgumentStruct(
343 launch, adaptor, alreadyMappedOperands, rewriter, accSupport,
344 globalSymbolRegion, symbolTable, config);
345 if (failed(Result: arguments))
346 return failure();
347
348 StringRef kernelName = launch.getKernel().getLeafReference().getValue();
349 Value kernelNamePtr = getOrCreateGlobalString(
350 loc, builder&: rewriter, name: getInternalGlobalName(kind: "kernel_name", detail: kernelName),
351 value: kernelName, globalSymbolRegion, symbolTable: &symbolTable);
352 Value ident = createIdent(loc, functionName: getParentFunctionName(op: launch), builder&: rewriter,
353 globalSymbolRegion, config, symbolTable: &symbolTable);
354 // The kernel call uses the explicit data-region flag.
355 Value flags = constantI64(loc, value: static_cast<int64_t>(ACCDataRegionFlags::None),
356 builder&: rewriter);
357 // The launch is made for the default device type, as the data calls of the
358 // environment are.
359 Value deviceType = constantI64(
360 loc, value: config.getDeviceTypeRuntimeValue(type: DeviceType::None), builder&: rewriter);
361 Value binaryDescriptor = config.createKernelBinaryDescriptor(loc, builder&: rewriter);
362 FailureOr<LLVM::CallOp> call =
363 createRuntimeCall(loc, builder&: rewriter, globalSymbolRegion, symbolTable,
364 fn: RuntimeFunction::ACCRTL_tgt_acc_kernel, config,
365 arguments: {ident, *kernelPointer, flags, deviceType, *arguments,
366 asyncQueue, kernelNamePtr, binaryDescriptor});
367 if (failed(Result: call))
368 return failure();
369 emitHostFallbackNotImplemented(loc, status: call->getResult(), rewriter,
370 globalSymbolRegion, symbolTable);
371 return success();
372}
373
374namespace {
375struct KernelEnvironmentLowering
376 : public ConvertOpToLLVMPattern<KernelEnvironmentOp> {
377 KernelEnvironmentLowering(const LLVMTypeConverter &converter,
378 OpenACCSupport &accSupport,
379 Region &globalSymbolRegion,
380 SymbolTable &symbolTable,
381 const ACCRuntimeCallConfig &config,
382 DeviceType clauseDeviceType,
383 PatternBenefit benefit = 1)
384 : ConvertOpToLLVMPattern<KernelEnvironmentOp>(converter, benefit),
385 accSupport(accSupport), globalSymbolRegion(globalSymbolRegion),
386 symbolTable(symbolTable), config(config),
387 clauseDeviceType(clauseDeviceType) {}
388
389 LogicalResult
390 matchAndRewrite(KernelEnvironmentOp environment, OpAdaptor adaptor,
391 ConversionPatternRewriter &rewriter) const override {
392 ValueRange mappingOperands = environment.getDataClauseOperands();
393 ValueRange convertedMappingOperands = adaptor.getDataClauseOperands();
394 Location beginLoc = mappingOperands.empty()
395 ? environment.getLoc()
396 : mappingOperands.front().getLoc();
397 Location endLoc = getMappingExitLoc(dataClauseOperands: mappingOperands).value_or(u&: beginLoc);
398 Value asyncQueue =
399 getAsyncQueue(op: environment, deviceType: clauseDeviceType, rewriter, config);
400 if (failed(Result: emitWaitClause(op: environment, deviceType: clauseDeviceType, asyncQueue,
401 rewriter, accSupport, globalSymbolRegion,
402 symbolTable, config)))
403 return failure();
404
405 SmallVector<Value> alreadyMappedOperands;
406 for (Value operand : mappingOperands)
407 if (!isDevicePtrMapping(operand))
408 alreadyMappedOperands.push_back(Elt: operand);
409
410 ACCDataCallOptions beginOptions;
411 beginOptions.flags = static_cast<int64_t>(ACCDataRegionFlags::Implicit);
412 beginOptions.returnAddressOperands = alreadyMappedOperands;
413 FailureOr<SmallVector<Value>> devicePointers = emitACCDataRuntimeCall(
414 loc: beginLoc, fn: RuntimeFunction::ACCRTL_tgt_acc_data_begin, mappingOperands,
415 convertedOperands: convertedMappingOperands, rewriter, accSupport, globalSymbolRegion,
416 symbolTable, config, trailingArguments: {asyncQueue}, options: beginOptions);
417 if (failed(Result: devicePointers))
418 return failure();
419
420 llvm::DenseMap<Value, Value> deviceAddress;
421 for (auto [operand, pointer] :
422 llvm::zip_equal(t&: alreadyMappedOperands, u&: *devicePointers))
423 deviceAddress[operand] = pointer;
424
425 SmallVector<gpu::LaunchFuncOp> launches;
426 environment.walk(
427 callback: [&](gpu::LaunchFuncOp launch) { launches.push_back(Elt: launch); });
428 for (gpu::LaunchFuncOp launch : launches) {
429 SmallVector<Value> converted;
430 converted.reserve(N: launch->getNumOperands());
431 for (Value operand : launch->getOperands()) {
432 Value value;
433 // A data entry in the region is mapped by this kernel, so it is passed
434 // as the variable of its clause.
435 if (isDataEntryInsideEnvironment(environment, operand))
436 value = remapOrSelf(value: getVar(accDataClauseOp: operand.getDefiningOp()), rewriter);
437 else
438 value = deviceAddress.lookup(Val: operand);
439 if (!value)
440 value = rewriter.getRemappedValue(key: operand);
441 if (!value)
442 return rewriter.notifyMatchFailure(arg&: launch,
443 msg: "unconverted kernel operand");
444 converted.push_back(Elt: value);
445 }
446
447 gpu::LaunchFuncOpAdaptor launchAdaptor(converted, launch);
448 Value queue = launchAdaptor.getAsyncObject();
449 queue = queue ? castToI64(loc: launch.getLoc(), value: queue, builder&: rewriter) : asyncQueue;
450 rewriter.setInsertionPoint(launch);
451 if (failed(Result: emitKernelCall(launch, adaptor: launchAdaptor, asyncQueue: queue,
452 alreadyMappedOperands, rewriter, accSupport,
453 globalSymbolRegion, symbolTable, config)))
454 return failure();
455 rewriter.eraseOp(op: launch);
456 }
457
458 rewriter.setInsertionPointAfter(environment);
459 ACCDataCallOptions endOptions;
460 endOptions.callKind = ACCDataCallKind::DataExit;
461 endOptions.flags = static_cast<int64_t>(ACCDataRegionFlags::Implicit);
462 if (failed(Result: emitACCDataRuntimeCall(
463 loc: endLoc, fn: RuntimeFunction::ACCRTL_tgt_acc_data_end, mappingOperands,
464 convertedOperands: convertedMappingOperands, rewriter, accSupport, globalSymbolRegion,
465 symbolTable, config, trailingArguments: {asyncQueue}, options: endOptions)))
466 return failure();
467
468 spliceConstructRegion(op: environment, region&: environment.getRegion(), rewriter);
469 rewriter.eraseOp(op: environment);
470 return success();
471 }
472
473 OpenACCSupport &accSupport;
474 Region &globalSymbolRegion;
475 SymbolTable &symbolTable;
476 ACCRuntimeCallConfig config;
477 DeviceType clauseDeviceType;
478};
479} // namespace
480
481void mlir::configureACCKernelLaunchConversionLegality(
482 ConversionTarget &target) {
483 target.addIllegalOp<KernelEnvironmentOp>();
484}
485
486void mlir::populateACCKernelLaunchPatterns(LLVMTypeConverter &converter,
487 RewritePatternSet &patterns,
488 OpenACCSupport &accSupport,
489 Region &globalSymbolRegion,
490 SymbolTable &symbolTable,
491 const ACCRuntimeCallConfig &config,
492 DeviceType clauseDeviceType) {
493 patterns.add<KernelEnvironmentLowering>(arg&: converter, args&: accSupport,
494 args&: globalSymbolRegion, args&: symbolTable,
495 args: config, args&: clauseDeviceType);
496}
497