1//===- ACCDataRuntime.cpp - Emit OpenACC data runtime arguments -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Builds the mapping argument arrays used by OpenACC data runtime entry points.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Conversion/OpenACCToLLVM/ACCDataRuntime.h"
14#include "mlir/Conversion/OpenACCToLLVM/ACCToLLVMUtils.h"
15
16#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
20#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
21#include "mlir/Dialect/OpenACC/OpenACC.h"
22#include "mlir/Dialect/OpenACC/OpenACCUtils.h"
23#include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h"
24#include "mlir/Dialect/OpenACC/OpenACCUtilsType.h"
25#include "mlir/Transforms/DialectConversion.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SmallVectorExtras.h"
28#include "llvm/Support/Debug.h"
29#include "llvm/Support/raw_ostream.h"
30
31#include <algorithm>
32
33#define DEBUG_TYPE "acc-data-runtime"
34
35using namespace mlir;
36using namespace mlir::acc;
37
38static Value getPointer(Location loc, Value value,
39 ConversionPatternRewriter &rewriter) {
40 Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext());
41 if (!value)
42 return LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrTy);
43 if (value.getType() == ptrTy)
44 return value;
45
46 if (isa<PointerLikeType>(Val: value.getType()))
47 return castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value, resultType: ptrTy);
48
49 Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: 1);
50 Value storage =
51 LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: value.getType(), arraySize: one);
52 LLVM::StoreOp::create(builder&: rewriter, location: loc, value, addr: storage);
53 return storage;
54}
55
56/// Returns the address of a slot holding \p value, so that the runtime reads
57/// the value from memory rather than receiving it directly.
58static Value getIndirectPointer(Location loc, Value value,
59 ConversionPatternRewriter &rewriter) {
60 Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext());
61 Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: 1);
62 Value storage =
63 LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: value.getType(), arraySize: one);
64 LLVM::StoreOp::create(builder&: rewriter, location: loc, value, addr: storage);
65 return storage;
66}
67
68Value mlir::createACCDataArray(Location loc, Type elementType,
69 ArrayRef<Value> values, RewriterBase &rewriter) {
70 Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext());
71 Type i32Ty = rewriter.getI32Type();
72 Value count = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: values.size());
73 Value array =
74 LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType, arraySize: count);
75 for (auto [index, value] : llvm::enumerate(First&: values)) {
76 Value indexValue = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: index);
77 Value element = LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType,
78 basePtr: array, indices: ValueRange{indexValue});
79 LLVM::StoreOp::create(builder&: rewriter, location: loc, value, addr: element);
80 }
81 return array;
82}
83
84void mlir::createACCMappedObjectArrays(Location loc,
85 ArrayRef<ACCMappedObjectArgs> objects,
86 ACCDataRuntimeArgs &runtimeArgs,
87 RewriterBase &rewriter) {
88 Type i32Ty = rewriter.getI32Type();
89 Type i64Ty = rewriter.getI64Type();
90 Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext());
91 auto createArray = [&](Type elementType, Value ACCMappedObjectArgs::*field) {
92 SmallVector<Value> values =
93 llvm::map_to_vector(C&: objects, F: [&](const ACCMappedObjectArgs &object) {
94 return object.*field;
95 });
96 return createACCDataArray(loc, elementType, values, rewriter);
97 };
98 // The data-call emitter states the count before the object values.
99 if (!runtimeArgs.argNum)
100 runtimeArgs.argNum =
101 LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: objects.size());
102 runtimeArgs.argBasePtrs = createArray(ptrTy, &ACCMappedObjectArgs::base);
103 runtimeArgs.argPtrs = createArray(ptrTy, &ACCMappedObjectArgs::pointer);
104 runtimeArgs.argSizes = createArray(i64Ty, &ACCMappedObjectArgs::size);
105 runtimeArgs.argTypes = createArray(i64Ty, &ACCMappedObjectArgs::type);
106 runtimeArgs.argNames = createArray(ptrTy, &ACCMappedObjectArgs::name);
107 if (!runtimeArgs.argMappers)
108 runtimeArgs.argMappers = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrTy);
109 runtimeArgs.argDescs = createArray(ptrTy, &ACCMappedObjectArgs::descriptor);
110}
111
112/// Privatized storage only ever exists on the device, so there is no host
113/// address to give. A device address names storage the runtime must not look
114/// up, and an object mapped by value has no address at all, so both are handed
115/// over in a slot the runtime reads them from. A pointer that no clause maps
116/// but the device can already access is a device address as well.
117Value mlir::createACCMappedObjectPointer(Location loc, Value operand,
118 Value converted, MapFlags mapFlags,
119 ConversionPatternRewriter &rewriter) {
120 Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext());
121 if (isa<PrivateType>(Val: operand.getType()))
122 return LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrTy);
123 if (converted && converted.getType() == ptrTy &&
124 (bitEnumContainsAny(bits: mapFlags, bit: MapFlags::devptr) ||
125 (!isDataEntryOp(op: operand.getDefiningOp()) &&
126 isDeviceAccessibleValue(val: operand))))
127 return getIndirectPointer(loc, value: converted, rewriter);
128 return getPointer(loc, value: converted, rewriter);
129}
130
131Value mlir::createACCMappedObjectBase(Location loc, Operation *mapOp,
132 MapFlags mapFlags,
133 ConversionPatternRewriter &rewriter) {
134 Value base;
135 if (mapOp) {
136 if (Value attach = acc::getVarPtrPtr(accDataClauseOp: mapOp))
137 base = remapOrSelf(value: attach, rewriter);
138 else if (bitEnumContainsAny(bits: mapFlags, bit: MapFlags::ptr_and_obj))
139 base = remapOrSelf(value: acc::getDesc(mapEntryOp: mapOp), rewriter);
140 }
141 return getPointer(loc, value: base, rewriter);
142}
143
144Value mlir::createACCMappedObjectName(Location loc, Value operand,
145 OpenACCSupport &accSupport,
146 Region &globalSymbolRegion,
147 SymbolTable *symbolTable,
148 ConversionPatternRewriter &rewriter) {
149 // The runtime resolves an object that lives on the device against the
150 // symbols of the binary by this name, so it is the name the object is
151 // emitted under rather than the one the source spells.
152 VariableNameConfig nameConfig;
153 nameConfig.preferDemangledName = false;
154 std::string name = accSupport.getVariableName(v: operand, config: nameConfig);
155 if (name.empty())
156 return LLVM::ZeroOp::create(
157 builder&: rewriter, location: loc, res: LLVM::LLVMPointerType::get(context: rewriter.getContext()));
158 return getOrCreateGlobalString(loc, builder&: rewriter,
159 name: getInternalGlobalName(kind: "var_name", detail: name), value: name,
160 globalSymbolRegion, symbolTable);
161}
162
163static std::optional<MapFlags> computePackedMapFlags(Operation *mapOp) {
164 if (auto mapInfo = dyn_cast<MapInfoOp>(Val: mapOp))
165 return mapInfo.getMapFlags();
166 if (isa<ACC_DATA_ENTRY_OPS>(Val: mapOp))
167 return computeDataClauseMapFlags(entryOp: mapOp, ptrAndObj: hasAttachPoint(mapEntryOp: mapOp));
168 return std::nullopt;
169}
170
171FailureOr<Value>
172mlir::createACCMappedObjectSize(Location loc, Value operand, Value converted,
173 MapFlags mapFlags, OpenACCSupport &accSupport,
174 ConversionPatternRewriter &rewriter) {
175 Operation *scope = rewriter.getInsertionBlock()->getParentOp();
176 auto module = dyn_cast<ModuleOp>(Val: scope);
177 if (!module)
178 module = scope->getParentOfType<ModuleOp>();
179 // An object mapped by value is passed as the argument itself, so its size is
180 // the size of what is passed and not of the object it was read from. The two
181 // differ when the argument holds one field of a decomposed aggregate.
182 if (bitEnumContainsAny(bits: mapFlags, bit: MapFlags::literal) && converted) {
183 auto sizeAndAlignment =
184 acc::getTypeSizeAndAlignment(ty: converted.getType(), module, support: &accSupport);
185 if (!sizeAndAlignment) {
186 std::string message;
187 llvm::raw_string_ostream os(message);
188 os << "unhandled type size computation for type: " << converted.getType();
189 (void)accSupport.emitNYI(loc, message);
190 return failure();
191 }
192 return constantI64(loc, value: sizeAndAlignment->first.getFixedValue(), builder&: rewriter);
193 }
194 Operation *mapOp = operand.getDefiningOp();
195 if (!isDataEntryOp(op: mapOp))
196 return constantI64(loc, value: 0, builder&: rewriter);
197 if (Value size = acc::getMapSize(mapEntryOp: mapOp))
198 return castToI64(loc, value: remapOrSelf(value: size, rewriter), builder&: rewriter);
199
200 Value var = acc::getVar(accDataClauseOp: mapOp);
201 if (!var)
202 return constantI64(loc, value: 0, builder&: rewriter);
203 Type varType = acc::getVarType(accDataClauseOp: mapOp);
204 if (!varType)
205 varType = var.getType();
206 // A memref is handed over with a descriptor stating its extents and element
207 // size, which is what states the size of such a mapping.
208 if (isa<MemRefType>(Val: var.getType()) || isa<MemRefType>(Val: varType))
209 return constantI64(loc, value: 0, builder&: rewriter);
210
211 // A data clause operation that was not turned into an `acc.map_info` has no
212 // size of its own, so it is sized the way that operation would have been.
213 // A size that is not statically known is left to the runtime, as an unsized
214 // mapping is.
215 int64_t size =
216 acc::computeMapInfoSizeBytes(var, varType, descKind: acc::getDataDescKind(mapEntryOp: mapOp),
217 bounds: acc::getBounds(accDataClauseOp: mapOp), support: &accSupport);
218 return constantI64(loc, value: std::max<int64_t>(a: size, b: 0), builder&: rewriter);
219}
220
221static Value getElementSize(Operation *mapOp, OpenACCSupport &accSupport,
222 ConversionPatternRewriter &rewriter) {
223 Location loc = mapOp->getLoc();
224 ModuleOp module = mapOp->getParentOfType<ModuleOp>();
225 if (std::optional<int64_t> size = acc::getMapElementSize(mapEntryOp: mapOp))
226 return constantI64(loc, value: *size, builder&: rewriter);
227
228 Type type = acc::getVarType(accDataClauseOp: mapOp);
229 if (auto shaped = dyn_cast<ShapedType>(Val&: type))
230 type = shaped.getElementType();
231 if (!type)
232 return constantI64(loc, value: 0, builder&: rewriter);
233 if (auto sizeAndAlignment =
234 acc::getTypeSizeAndAlignment(ty: type, module, support: &accSupport))
235 return constantI64(loc, value: sizeAndAlignment->first.getFixedValue(), builder&: rewriter);
236 return constantI64(loc, value: 0, builder&: rewriter);
237}
238
239static Value getBoundValue(Value value, Location loc,
240 ConversionPatternRewriter &rewriter) {
241 if (!value)
242 return constantI64(loc, value: 0, builder&: rewriter);
243 return castToI64(loc, value: remapOrSelf(value, rewriter), builder&: rewriter);
244}
245
246Value mlir::createACCDataDescriptor(Location loc, Value baseDescriptor,
247 Type baseDescriptorType, uint32_t version,
248 ValueRange bounds, Value elementSize,
249 ConversionPatternRewriter &rewriter) {
250 MLIRContext *context = rewriter.getContext();
251 Type i8Ty = rewriter.getI8Type();
252 Type i32Ty = rewriter.getI32Type();
253 Type i64Ty = rewriter.getI64Type();
254 Type ptrTy = LLVM::LLVMPointerType::get(context);
255
256 Value versionValue = LLVM::ConstantOp::create(
257 builder&: rewriter, location: loc, type: i32Ty,
258 value: version | static_cast<uint32_t>(
259 getDataDescriptorKind(desc: DataDescriptor::AccDataDescOpenACC)));
260 baseDescriptor = LLVM::InsertValueOp::create(
261 builder&: rewriter, location: loc, res: baseDescriptorType, container: baseDescriptor, value: versionValue,
262 position: ArrayRef<int64_t>{0});
263
264 Type descriptorType = getDataDescriptorType(
265 ctx: context, desc: DataDescriptor::AccDataDescOpenACC, baseType: baseDescriptorType);
266 Value descriptor = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: descriptorType);
267 Value rank = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i8Ty, value: bounds.size());
268
269 SmallVector<Value> lowerBounds;
270 SmallVector<Value> upperBounds;
271 SmallVector<Value> extents;
272 SmallVector<Value> strides;
273 SmallVector<Value> starts;
274 for (Value boundValue : bounds) {
275 auto bound = cast<DataBoundsOp>(Val: boundValue.getDefiningOp());
276 lowerBounds.push_back(
277 Elt: getBoundValue(value: bound.getLowerbound(), loc: bound.getLoc(), rewriter));
278 upperBounds.push_back(
279 Elt: getBoundValue(value: bound.getUpperbound(), loc: bound.getLoc(), rewriter));
280 Value sourceExtent =
281 bound.getSourceExtent() ? bound.getSourceExtent() : bound.getExtent();
282 extents.push_back(Elt: getBoundValue(value: sourceExtent, loc: bound.getLoc(), rewriter));
283 Value stride = getBoundValue(value: bound.getStride(), loc: bound.getLoc(), rewriter);
284 if (!bound.getStrideInBytes())
285 stride =
286 LLVM::MulOp::create(builder&: rewriter, location: bound.getLoc(), lhs: stride, rhs: elementSize);
287 strides.push_back(Elt: stride);
288 starts.push_back(
289 Elt: getBoundValue(value: bound.getStartIdx(), loc: bound.getLoc(), rewriter));
290 }
291
292 Value lowerBoundsArray =
293 createACCDataArray(loc, elementType: i64Ty, values: lowerBounds, rewriter);
294 Value upperBoundsArray =
295 createACCDataArray(loc, elementType: i64Ty, values: upperBounds, rewriter);
296 Value extentsArray = createACCDataArray(loc, elementType: i64Ty, values: extents, rewriter);
297 Value stridesArray = createACCDataArray(loc, elementType: i64Ty, values: strides, rewriter);
298 Value startsArray = createACCDataArray(loc, elementType: i64Ty, values: starts, rewriter);
299
300 using Field = AccDataDescOpenACCField;
301 auto insert = [&](Value value, Field field) {
302 descriptor = LLVM::InsertValueOp::create(
303 builder&: rewriter, location: loc, res: descriptorType, container: descriptor, value,
304 position: ArrayRef<int64_t>{getDataDescriptorFieldIndex(field)});
305 };
306 insert(baseDescriptor, Field::Base);
307 insert(rank, Field::Rank);
308 insert(elementSize, Field::ElementSize);
309 insert(lowerBoundsArray, Field::LowerBounds);
310 insert(upperBoundsArray, Field::UpperBounds);
311 insert(extentsArray, Field::Extents);
312 insert(stridesArray, Field::StridesInBytes);
313 insert(startsArray, Field::StartIndices);
314
315 Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: 1);
316 Value storage =
317 LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: descriptorType, arraySize: one);
318 LLVM::StoreOp::create(builder&: rewriter, location: loc, value: descriptor, addr: storage);
319 return storage;
320}
321
322Value mlir::createACCMemRefDescriptorWrapperArg(
323 Location loc, MemRefType memrefType, Value convertedMemref,
324 ValueRange bounds, Value elementSize, ConversionPatternRewriter &rewriter) {
325 MLIRContext *context = rewriter.getContext();
326 Type i8Ty = rewriter.getI8Type();
327 Type i32Ty = rewriter.getI32Type();
328 Type ptrTy = LLVM::LLVMPointerType::get(context);
329 using Field = AccDataDescMemRefField;
330 Type descriptorType =
331 getDataDescriptorType(ctx: context, desc: DataDescriptor::AccDataDescMemRef);
332 auto version = static_cast<uint32_t>(
333 getDataDescriptorKind(desc: DataDescriptor::AccDataDescMemRef));
334 Value descriptor = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: descriptorType);
335 Value rank =
336 LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i8Ty, value: memrefType.getRank());
337 auto insert = [&](Value value, Field field) {
338 descriptor = LLVM::InsertValueOp::create(
339 builder&: rewriter, location: loc, res: descriptorType, container: descriptor, value,
340 position: ArrayRef<int64_t>{getDataDescriptorFieldIndex(field)});
341 };
342 insert(rank, Field::Rank);
343 insert(elementSize, Field::ElementSize);
344 insert(getPointer(loc, value: convertedMemref, rewriter), Field::MemRefDescriptor);
345 if (!bounds.empty())
346 return createACCDataDescriptor(loc, baseDescriptor: descriptor, baseDescriptorType: descriptorType, version,
347 bounds, elementSize, rewriter);
348
349 insert(LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: version),
350 Field::Version);
351 Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: 1);
352 Value storage =
353 LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: descriptorType, arraySize: one);
354 LLVM::StoreOp::create(builder&: rewriter, location: loc, value: descriptor, addr: storage);
355 return storage;
356}
357
358Value mlir::createACCArgumentDescriptor(Operation *mapOp,
359 Value convertedOperand,
360 OpenACCSupport &accSupport,
361 ConversionPatternRewriter &rewriter) {
362 Location loc = mapOp->getLoc();
363 MLIRContext *context = rewriter.getContext();
364 Type i32Ty = rewriter.getI32Type();
365 Type ptrTy = LLVM::LLVMPointerType::get(context);
366 SmallVector<Value> bounds = acc::getBounds(accDataClauseOp: mapOp);
367 DataDescKind descKind = acc::getDataDescKind(mapEntryOp: mapOp);
368 // Only a descriptor stating bounds reads the element size, so it is
369 // materialized where it is used, leaving no constant behind for a mapping
370 // that needs no descriptor at all.
371 auto elementSize = [&] {
372 return getElementSize(mapOp, accSupport, rewriter);
373 };
374
375 Value var = acc::getVar(accDataClauseOp: mapOp);
376 if (var && isa<MemRefType>(Val: var.getType())) {
377 auto memrefType = cast<MemRefType>(Val: var.getType());
378 return createACCMemRefDescriptorWrapperArg(
379 loc, memrefType, convertedMemref: convertedOperand, bounds, elementSize: elementSize(), rewriter);
380 }
381
382 if (bitEnumContainsAny(bits: descKind, bit: DataDescKind::cfi)) {
383 using Field = AccDataDescCFIField;
384 Type descriptorType =
385 getDataDescriptorType(ctx: context, desc: DataDescriptor::AccDataDescCFI);
386 auto version = static_cast<uint32_t>(
387 getDataDescriptorKind(desc: DataDescriptor::AccDataDescCFI));
388 Value descriptor = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: descriptorType);
389 Value descriptorStorage = remapOrSelf(value: acc::getDesc(mapEntryOp: mapOp), rewriter);
390 if (!descriptorStorage)
391 descriptorStorage = convertedOperand;
392 descriptor = LLVM::InsertValueOp::create(
393 builder&: rewriter, location: loc, res: descriptorType, container: descriptor,
394 value: getPointer(loc, value: descriptorStorage, rewriter),
395 position: ArrayRef<int64_t>{getDataDescriptorFieldIndex(field: Field::CFIDescriptor)});
396 if (!bounds.empty())
397 return createACCDataDescriptor(loc, baseDescriptor: descriptor, baseDescriptorType: descriptorType, version,
398 bounds, elementSize: elementSize(), rewriter);
399
400 Value versionValue =
401 LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: version);
402 descriptor = LLVM::InsertValueOp::create(
403 builder&: rewriter, location: loc, res: descriptorType, container: descriptor, value: versionValue,
404 position: ArrayRef<int64_t>{getDataDescriptorFieldIndex(field: Field::Version)});
405 Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: 1);
406 Value storage =
407 LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: descriptorType, arraySize: one);
408 LLVM::StoreOp::create(builder&: rewriter, location: loc, value: descriptor, addr: storage);
409 return storage;
410 }
411
412 if (!bounds.empty()) {
413 Type descriptorType =
414 getDataDescriptorType(ctx: context, desc: DataDescriptor::AccDataDescGeneric);
415 auto version = static_cast<uint32_t>(
416 getDataDescriptorKind(desc: DataDescriptor::AccDataDescGeneric));
417 Value descriptor = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: descriptorType);
418 return createACCDataDescriptor(loc, baseDescriptor: descriptor, baseDescriptorType: descriptorType, version,
419 bounds, elementSize: elementSize(), rewriter);
420 }
421 return LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrTy);
422}
423
424SmallVector<Value> mlir::ACCDataRuntimeArgs::getCallArgs() const {
425 return {ident, flags, deviceType, argNum, argBasePtrs, argPtrs,
426 argSizes, argTypes, argNames, argMappers, argDescs};
427}
428
429LogicalResult mlir::emitACCDataRuntimeArgs(
430 Location loc, ValueRange mappingOperands, ValueRange convertedOperands,
431 ConversionPatternRewriter &rewriter, Region &globalSymbolRegion,
432 acc::OpenACCSupport &accSupport, const acc::ACCRuntimeCallConfig &config,
433 ACCDataRuntimeArgs &runtimeArgs, const ACCDataCallOptions &options,
434 SymbolTable *symbolTable) {
435 if (mappingOperands.size() != convertedOperands.size())
436 return failure();
437
438 runtimeArgs.ident =
439 createIdent(loc, functionName: getParentFunctionName(values: mappingOperands), builder&: rewriter,
440 globalSymbolRegion, config, symbolTable);
441 runtimeArgs.flags = constantI64(loc, value: options.flags, builder&: rewriter);
442 runtimeArgs.deviceType = constantI64(
443 loc, value: config.getDeviceTypeRuntimeValue(type: DeviceType::None), builder&: rewriter);
444 runtimeArgs.argMappers = LLVM::ZeroOp::create(
445 builder&: rewriter, location: loc, res: LLVM::LLVMPointerType::get(context: rewriter.getContext()));
446
447 // An aggregate passed by value is decomposed into one argument per field by
448 // the target ABI, so the runtime is told about one object per field as well.
449 // Each of them keeps the map entry of the aggregate it came from. The device
450 // address of an aggregate is returned in the slot of its first field.
451 struct MappedObject {
452 Value operand;
453 Value converted;
454 bool firstField;
455 };
456 SmallVector<MappedObject> mappedObjects;
457 for (auto [operand, converted] :
458 llvm::zip_equal(t&: mappingOperands, u&: convertedOperands)) {
459 auto structType = dyn_cast<LLVM::LLVMStructType>(Val: converted.getType());
460 if (structType && !structType.isIdentified() &&
461 structType.getBody().size() > 1) {
462 for (unsigned field = 0, fields = structType.getBody().size();
463 field != fields; ++field)
464 mappedObjects.push_back(
465 Elt: {.operand: operand,
466 .converted: LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: converted, position: field),
467 .firstField: field == 0});
468 continue;
469 }
470 mappedObjects.push_back(Elt: {.operand: operand, .converted: converted, .firstField: true});
471 }
472
473 // The count precedes the values of the objects, which is where a decomposed
474 // aggregate states how many fields it became.
475 runtimeArgs.argNum = LLVM::ConstantOp::create(
476 builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: mappedObjects.size());
477
478 constexpr unsigned noSlot = ~0u;
479 runtimeArgs.returnAddressSlots.assign(NumElts: options.returnAddressOperands.size(),
480 Elt: noSlot);
481 SmallVector<ACCMappedObjectArgs> objects;
482 for (auto [slot, object] : llvm::enumerate(First&: mappedObjects)) {
483 auto [operand, converted, firstField] = object;
484 Operation *mapOp = operand.getDefiningOp();
485 if (!isDataEntryOp(op: mapOp))
486 return failure();
487 std::optional<MapFlags> mapFlags = computePackedMapFlags(mapOp);
488 if (!mapFlags)
489 return failure();
490 *mapFlags = config.postProcessMapFlags(mapOp, flags: *mapFlags);
491 LLVM_DEBUG(llvm::dbgs() << "mapping " << *mapOp << "\n as "
492 << config.formatMapFlags(*mapFlags) << "\n");
493
494 Location mapLoc = mapOp->getLoc();
495 ACCMappedObjectArgs args;
496 args.pointer = createACCMappedObjectPointer(loc: mapLoc, operand, converted,
497 mapFlags: *mapFlags, rewriter);
498 // An object that only lives on the device has no host address to state as
499 // its base.
500 args.base =
501 bitEnumContainsAny(bits: *mapFlags, bit: MapFlags::device_resident)
502 ? getPointer(loc: mapLoc, value: Value(), rewriter)
503 : createACCMappedObjectBase(loc: mapLoc, mapOp, mapFlags: *mapFlags, rewriter);
504
505 // A local object that lives on the device is torn down when the region
506 // that declared it ends, which the runtime only does for a mapping that
507 // states it is deleted.
508 if (options.callKind == ACCDataCallKind::DataExit &&
509 bitEnumContainsAny(bits: *mapFlags, bit: MapFlags::device_resident)) {
510 Value var = acc::getVar(accDataClauseOp: mapOp);
511 if (var &&
512 !isa_and_nonnull<AddressOfGlobalOpInterface>(Val: var.getDefiningOp()))
513 *mapFlags = *mapFlags | MapFlags::delete_;
514 }
515
516 if (firstField) {
517 for (auto [index, returned] :
518 llvm::enumerate(First: options.returnAddressOperands)) {
519 if (returned != operand)
520 continue;
521 *mapFlags = *mapFlags | MapFlags::return_param;
522 runtimeArgs.returnAddressSlots[index] = slot;
523 }
524 }
525 int64_t mapType = config.getMapFlagsRuntimeValue(flags: *mapFlags);
526
527 FailureOr<Value> size = createACCMappedObjectSize(
528 loc: mapLoc, operand, converted, mapFlags: *mapFlags, accSupport, rewriter);
529 if (failed(Result: size))
530 return failure();
531 args.size = *size;
532 args.type = constantI64(loc: mapLoc, value: mapType, builder&: rewriter);
533 args.name = createACCMappedObjectName(
534 loc: mapLoc, operand, accSupport, globalSymbolRegion, symbolTable, rewriter);
535 args.descriptor =
536 createACCArgumentDescriptor(mapOp, convertedOperand: converted, accSupport, rewriter);
537 objects.push_back(Elt: args);
538 }
539 if (llvm::is_contained(Range&: runtimeArgs.returnAddressSlots, Element: noSlot))
540 return failure();
541
542 createACCMappedObjectArrays(loc, objects, runtimeArgs, rewriter);
543 return success();
544}
545
546FailureOr<SmallVector<Value>> mlir::emitACCDataRuntimeCall(
547 Location loc, RuntimeFunction fn, ValueRange mappingOperands,
548 ValueRange convertedOperands, ConversionPatternRewriter &rewriter,
549 OpenACCSupport &accSupport, Region &globalSymbolRegion,
550 SymbolTable &symbolTable, const ACCRuntimeCallConfig &config,
551 ArrayRef<Value> trailingArguments, const ACCDataCallOptions &options) {
552 SmallVector<Value> returnedAddresses;
553 if (mappingOperands.empty())
554 return returnedAddresses;
555
556 ACCDataRuntimeArgs runtimeArgs;
557 if (failed(Result: emitACCDataRuntimeArgs(
558 loc, mappingOperands, convertedOperands, rewriter, globalSymbolRegion,
559 accSupport, config, runtimeArgs, options, symbolTable: &symbolTable)))
560 return rewriter.notifyMatchFailure(
561 arg: mappingOperands.front().getDefiningOp(),
562 msg: "unsupported OpenACC data-clause operand");
563
564 SmallVector<Value> arguments = runtimeArgs.getCallArgs();
565 arguments.append(in_start: trailingArguments.begin(), in_end: trailingArguments.end());
566 if (failed(Result: createRuntimeCall(loc, builder&: rewriter, globalSymbolRegion, symbolTable,
567 fn, config, arguments)))
568 return failure();
569
570 Type i32Ty = rewriter.getI32Type();
571 Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext());
572 for (unsigned slot : runtimeArgs.returnAddressSlots) {
573 Value index = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: slot);
574 Value element =
575 LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: ptrTy,
576 basePtr: runtimeArgs.argBasePtrs, indices: ValueRange{index});
577 returnedAddresses.push_back(
578 Elt: LLVM::LoadOp::create(builder&: rewriter, location: loc, type: ptrTy, addr: element));
579 }
580 return returnedAddresses;
581}
582