| 1 | //===- ACCDataRuntime.cpp - Emit OpenACC data runtime arguments -*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Builds the mapping argument arrays used by OpenACC data runtime entry points. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Conversion/OpenACCToLLVM/ACCDataRuntime.h" |
| 14 | #include "mlir/Conversion/OpenACCToLLVM/ACCToLLVMUtils.h" |
| 15 | |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMArithOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 20 | #include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" |
| 21 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 22 | #include "mlir/Dialect/OpenACC/OpenACCUtils.h" |
| 23 | #include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h" |
| 24 | #include "mlir/Dialect/OpenACC/OpenACCUtilsType.h" |
| 25 | #include "mlir/Transforms/DialectConversion.h" |
| 26 | #include "llvm/ADT/STLExtras.h" |
| 27 | #include "llvm/ADT/SmallVectorExtras.h" |
| 28 | #include "llvm/Support/Debug.h" |
| 29 | #include "llvm/Support/raw_ostream.h" |
| 30 | |
| 31 | #include <algorithm> |
| 32 | |
| 33 | #define DEBUG_TYPE "acc-data-runtime" |
| 34 | |
| 35 | using namespace mlir; |
| 36 | using namespace mlir::acc; |
| 37 | |
| 38 | static Value getPointer(Location loc, Value value, |
| 39 | ConversionPatternRewriter &rewriter) { |
| 40 | Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 41 | if (!value) |
| 42 | return LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrTy); |
| 43 | if (value.getType() == ptrTy) |
| 44 | return value; |
| 45 | |
| 46 | if (isa<PointerLikeType>(Val: value.getType())) |
| 47 | return castPointerLikeTypeIfNeeded(builder&: rewriter, loc, value, resultType: ptrTy); |
| 48 | |
| 49 | Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: 1); |
| 50 | Value storage = |
| 51 | LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: value.getType(), arraySize: one); |
| 52 | LLVM::StoreOp::create(builder&: rewriter, location: loc, value, addr: storage); |
| 53 | return storage; |
| 54 | } |
| 55 | |
| 56 | /// Returns the address of a slot holding \p value, so that the runtime reads |
| 57 | /// the value from memory rather than receiving it directly. |
| 58 | static Value getIndirectPointer(Location loc, Value value, |
| 59 | ConversionPatternRewriter &rewriter) { |
| 60 | Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 61 | Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: 1); |
| 62 | Value storage = |
| 63 | LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: value.getType(), arraySize: one); |
| 64 | LLVM::StoreOp::create(builder&: rewriter, location: loc, value, addr: storage); |
| 65 | return storage; |
| 66 | } |
| 67 | |
| 68 | Value mlir::createACCDataArray(Location loc, Type elementType, |
| 69 | ArrayRef<Value> values, RewriterBase &rewriter) { |
| 70 | Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 71 | Type i32Ty = rewriter.getI32Type(); |
| 72 | Value count = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: values.size()); |
| 73 | Value array = |
| 74 | LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType, arraySize: count); |
| 75 | for (auto [index, value] : llvm::enumerate(First&: values)) { |
| 76 | Value indexValue = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: index); |
| 77 | Value element = LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType, |
| 78 | basePtr: array, indices: ValueRange{indexValue}); |
| 79 | LLVM::StoreOp::create(builder&: rewriter, location: loc, value, addr: element); |
| 80 | } |
| 81 | return array; |
| 82 | } |
| 83 | |
| 84 | void mlir::createACCMappedObjectArrays(Location loc, |
| 85 | ArrayRef<ACCMappedObjectArgs> objects, |
| 86 | ACCDataRuntimeArgs &runtimeArgs, |
| 87 | RewriterBase &rewriter) { |
| 88 | Type i32Ty = rewriter.getI32Type(); |
| 89 | Type i64Ty = rewriter.getI64Type(); |
| 90 | Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 91 | auto createArray = [&](Type elementType, Value ACCMappedObjectArgs::*field) { |
| 92 | SmallVector<Value> values = |
| 93 | llvm::map_to_vector(C&: objects, F: [&](const ACCMappedObjectArgs &object) { |
| 94 | return object.*field; |
| 95 | }); |
| 96 | return createACCDataArray(loc, elementType, values, rewriter); |
| 97 | }; |
| 98 | // The data-call emitter states the count before the object values. |
| 99 | if (!runtimeArgs.argNum) |
| 100 | runtimeArgs.argNum = |
| 101 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: objects.size()); |
| 102 | runtimeArgs.argBasePtrs = createArray(ptrTy, &ACCMappedObjectArgs::base); |
| 103 | runtimeArgs.argPtrs = createArray(ptrTy, &ACCMappedObjectArgs::pointer); |
| 104 | runtimeArgs.argSizes = createArray(i64Ty, &ACCMappedObjectArgs::size); |
| 105 | runtimeArgs.argTypes = createArray(i64Ty, &ACCMappedObjectArgs::type); |
| 106 | runtimeArgs.argNames = createArray(ptrTy, &ACCMappedObjectArgs::name); |
| 107 | if (!runtimeArgs.argMappers) |
| 108 | runtimeArgs.argMappers = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrTy); |
| 109 | runtimeArgs.argDescs = createArray(ptrTy, &ACCMappedObjectArgs::descriptor); |
| 110 | } |
| 111 | |
| 112 | /// Privatized storage only ever exists on the device, so there is no host |
| 113 | /// address to give. A device address names storage the runtime must not look |
| 114 | /// up, and an object mapped by value has no address at all, so both are handed |
| 115 | /// over in a slot the runtime reads them from. A pointer that no clause maps |
| 116 | /// but the device can already access is a device address as well. |
| 117 | Value mlir::createACCMappedObjectPointer(Location loc, Value operand, |
| 118 | Value converted, MapFlags mapFlags, |
| 119 | ConversionPatternRewriter &rewriter) { |
| 120 | Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 121 | if (isa<PrivateType>(Val: operand.getType())) |
| 122 | return LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrTy); |
| 123 | if (converted && converted.getType() == ptrTy && |
| 124 | (bitEnumContainsAny(bits: mapFlags, bit: MapFlags::devptr) || |
| 125 | (!isDataEntryOp(op: operand.getDefiningOp()) && |
| 126 | isDeviceAccessibleValue(val: operand)))) |
| 127 | return getIndirectPointer(loc, value: converted, rewriter); |
| 128 | return getPointer(loc, value: converted, rewriter); |
| 129 | } |
| 130 | |
| 131 | Value mlir::createACCMappedObjectBase(Location loc, Operation *mapOp, |
| 132 | MapFlags mapFlags, |
| 133 | ConversionPatternRewriter &rewriter) { |
| 134 | Value base; |
| 135 | if (mapOp) { |
| 136 | if (Value attach = acc::getVarPtrPtr(accDataClauseOp: mapOp)) |
| 137 | base = remapOrSelf(value: attach, rewriter); |
| 138 | else if (bitEnumContainsAny(bits: mapFlags, bit: MapFlags::ptr_and_obj)) |
| 139 | base = remapOrSelf(value: acc::getDesc(mapEntryOp: mapOp), rewriter); |
| 140 | } |
| 141 | return getPointer(loc, value: base, rewriter); |
| 142 | } |
| 143 | |
| 144 | Value mlir::createACCMappedObjectName(Location loc, Value operand, |
| 145 | OpenACCSupport &accSupport, |
| 146 | Region &globalSymbolRegion, |
| 147 | SymbolTable *symbolTable, |
| 148 | ConversionPatternRewriter &rewriter) { |
| 149 | // The runtime resolves an object that lives on the device against the |
| 150 | // symbols of the binary by this name, so it is the name the object is |
| 151 | // emitted under rather than the one the source spells. |
| 152 | VariableNameConfig nameConfig; |
| 153 | nameConfig.preferDemangledName = false; |
| 154 | std::string name = accSupport.getVariableName(v: operand, config: nameConfig); |
| 155 | if (name.empty()) |
| 156 | return LLVM::ZeroOp::create( |
| 157 | builder&: rewriter, location: loc, res: LLVM::LLVMPointerType::get(context: rewriter.getContext())); |
| 158 | return getOrCreateGlobalString(loc, builder&: rewriter, |
| 159 | name: getInternalGlobalName(kind: "var_name" , detail: name), value: name, |
| 160 | globalSymbolRegion, symbolTable); |
| 161 | } |
| 162 | |
| 163 | static std::optional<MapFlags> computePackedMapFlags(Operation *mapOp) { |
| 164 | if (auto mapInfo = dyn_cast<MapInfoOp>(Val: mapOp)) |
| 165 | return mapInfo.getMapFlags(); |
| 166 | if (isa<ACC_DATA_ENTRY_OPS>(Val: mapOp)) |
| 167 | return computeDataClauseMapFlags(entryOp: mapOp, ptrAndObj: hasAttachPoint(mapEntryOp: mapOp)); |
| 168 | return std::nullopt; |
| 169 | } |
| 170 | |
| 171 | FailureOr<Value> |
| 172 | mlir::createACCMappedObjectSize(Location loc, Value operand, Value converted, |
| 173 | MapFlags mapFlags, OpenACCSupport &accSupport, |
| 174 | ConversionPatternRewriter &rewriter) { |
| 175 | Operation *scope = rewriter.getInsertionBlock()->getParentOp(); |
| 176 | auto module = dyn_cast<ModuleOp>(Val: scope); |
| 177 | if (!module) |
| 178 | module = scope->getParentOfType<ModuleOp>(); |
| 179 | // An object mapped by value is passed as the argument itself, so its size is |
| 180 | // the size of what is passed and not of the object it was read from. The two |
| 181 | // differ when the argument holds one field of a decomposed aggregate. |
| 182 | if (bitEnumContainsAny(bits: mapFlags, bit: MapFlags::literal) && converted) { |
| 183 | auto sizeAndAlignment = |
| 184 | acc::getTypeSizeAndAlignment(ty: converted.getType(), module, support: &accSupport); |
| 185 | if (!sizeAndAlignment) { |
| 186 | std::string message; |
| 187 | llvm::raw_string_ostream os(message); |
| 188 | os << "unhandled type size computation for type: " << converted.getType(); |
| 189 | (void)accSupport.emitNYI(loc, message); |
| 190 | return failure(); |
| 191 | } |
| 192 | return constantI64(loc, value: sizeAndAlignment->first.getFixedValue(), builder&: rewriter); |
| 193 | } |
| 194 | Operation *mapOp = operand.getDefiningOp(); |
| 195 | if (!isDataEntryOp(op: mapOp)) |
| 196 | return constantI64(loc, value: 0, builder&: rewriter); |
| 197 | if (Value size = acc::getMapSize(mapEntryOp: mapOp)) |
| 198 | return castToI64(loc, value: remapOrSelf(value: size, rewriter), builder&: rewriter); |
| 199 | |
| 200 | Value var = acc::getVar(accDataClauseOp: mapOp); |
| 201 | if (!var) |
| 202 | return constantI64(loc, value: 0, builder&: rewriter); |
| 203 | Type varType = acc::getVarType(accDataClauseOp: mapOp); |
| 204 | if (!varType) |
| 205 | varType = var.getType(); |
| 206 | // A memref is handed over with a descriptor stating its extents and element |
| 207 | // size, which is what states the size of such a mapping. |
| 208 | if (isa<MemRefType>(Val: var.getType()) || isa<MemRefType>(Val: varType)) |
| 209 | return constantI64(loc, value: 0, builder&: rewriter); |
| 210 | |
| 211 | // A data clause operation that was not turned into an `acc.map_info` has no |
| 212 | // size of its own, so it is sized the way that operation would have been. |
| 213 | // A size that is not statically known is left to the runtime, as an unsized |
| 214 | // mapping is. |
| 215 | int64_t size = |
| 216 | acc::computeMapInfoSizeBytes(var, varType, descKind: acc::getDataDescKind(mapEntryOp: mapOp), |
| 217 | bounds: acc::getBounds(accDataClauseOp: mapOp), support: &accSupport); |
| 218 | return constantI64(loc, value: std::max<int64_t>(a: size, b: 0), builder&: rewriter); |
| 219 | } |
| 220 | |
| 221 | static Value getElementSize(Operation *mapOp, OpenACCSupport &accSupport, |
| 222 | ConversionPatternRewriter &rewriter) { |
| 223 | Location loc = mapOp->getLoc(); |
| 224 | ModuleOp module = mapOp->getParentOfType<ModuleOp>(); |
| 225 | if (std::optional<int64_t> size = acc::getMapElementSize(mapEntryOp: mapOp)) |
| 226 | return constantI64(loc, value: *size, builder&: rewriter); |
| 227 | |
| 228 | Type type = acc::getVarType(accDataClauseOp: mapOp); |
| 229 | if (auto shaped = dyn_cast<ShapedType>(Val&: type)) |
| 230 | type = shaped.getElementType(); |
| 231 | if (!type) |
| 232 | return constantI64(loc, value: 0, builder&: rewriter); |
| 233 | if (auto sizeAndAlignment = |
| 234 | acc::getTypeSizeAndAlignment(ty: type, module, support: &accSupport)) |
| 235 | return constantI64(loc, value: sizeAndAlignment->first.getFixedValue(), builder&: rewriter); |
| 236 | return constantI64(loc, value: 0, builder&: rewriter); |
| 237 | } |
| 238 | |
| 239 | static Value getBoundValue(Value value, Location loc, |
| 240 | ConversionPatternRewriter &rewriter) { |
| 241 | if (!value) |
| 242 | return constantI64(loc, value: 0, builder&: rewriter); |
| 243 | return castToI64(loc, value: remapOrSelf(value, rewriter), builder&: rewriter); |
| 244 | } |
| 245 | |
| 246 | Value mlir::createACCDataDescriptor(Location loc, Value baseDescriptor, |
| 247 | Type baseDescriptorType, uint32_t version, |
| 248 | ValueRange bounds, Value elementSize, |
| 249 | ConversionPatternRewriter &rewriter) { |
| 250 | MLIRContext *context = rewriter.getContext(); |
| 251 | Type i8Ty = rewriter.getI8Type(); |
| 252 | Type i32Ty = rewriter.getI32Type(); |
| 253 | Type i64Ty = rewriter.getI64Type(); |
| 254 | Type ptrTy = LLVM::LLVMPointerType::get(context); |
| 255 | |
| 256 | Value versionValue = LLVM::ConstantOp::create( |
| 257 | builder&: rewriter, location: loc, type: i32Ty, |
| 258 | value: version | static_cast<uint32_t>( |
| 259 | getDataDescriptorKind(desc: DataDescriptor::AccDataDescOpenACC))); |
| 260 | baseDescriptor = LLVM::InsertValueOp::create( |
| 261 | builder&: rewriter, location: loc, res: baseDescriptorType, container: baseDescriptor, value: versionValue, |
| 262 | position: ArrayRef<int64_t>{0}); |
| 263 | |
| 264 | Type descriptorType = getDataDescriptorType( |
| 265 | ctx: context, desc: DataDescriptor::AccDataDescOpenACC, baseType: baseDescriptorType); |
| 266 | Value descriptor = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: descriptorType); |
| 267 | Value rank = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i8Ty, value: bounds.size()); |
| 268 | |
| 269 | SmallVector<Value> lowerBounds; |
| 270 | SmallVector<Value> upperBounds; |
| 271 | SmallVector<Value> extents; |
| 272 | SmallVector<Value> strides; |
| 273 | SmallVector<Value> starts; |
| 274 | for (Value boundValue : bounds) { |
| 275 | auto bound = cast<DataBoundsOp>(Val: boundValue.getDefiningOp()); |
| 276 | lowerBounds.push_back( |
| 277 | Elt: getBoundValue(value: bound.getLowerbound(), loc: bound.getLoc(), rewriter)); |
| 278 | upperBounds.push_back( |
| 279 | Elt: getBoundValue(value: bound.getUpperbound(), loc: bound.getLoc(), rewriter)); |
| 280 | Value sourceExtent = |
| 281 | bound.getSourceExtent() ? bound.getSourceExtent() : bound.getExtent(); |
| 282 | extents.push_back(Elt: getBoundValue(value: sourceExtent, loc: bound.getLoc(), rewriter)); |
| 283 | Value stride = getBoundValue(value: bound.getStride(), loc: bound.getLoc(), rewriter); |
| 284 | if (!bound.getStrideInBytes()) |
| 285 | stride = |
| 286 | LLVM::MulOp::create(builder&: rewriter, location: bound.getLoc(), lhs: stride, rhs: elementSize); |
| 287 | strides.push_back(Elt: stride); |
| 288 | starts.push_back( |
| 289 | Elt: getBoundValue(value: bound.getStartIdx(), loc: bound.getLoc(), rewriter)); |
| 290 | } |
| 291 | |
| 292 | Value lowerBoundsArray = |
| 293 | createACCDataArray(loc, elementType: i64Ty, values: lowerBounds, rewriter); |
| 294 | Value upperBoundsArray = |
| 295 | createACCDataArray(loc, elementType: i64Ty, values: upperBounds, rewriter); |
| 296 | Value extentsArray = createACCDataArray(loc, elementType: i64Ty, values: extents, rewriter); |
| 297 | Value stridesArray = createACCDataArray(loc, elementType: i64Ty, values: strides, rewriter); |
| 298 | Value startsArray = createACCDataArray(loc, elementType: i64Ty, values: starts, rewriter); |
| 299 | |
| 300 | using Field = AccDataDescOpenACCField; |
| 301 | auto insert = [&](Value value, Field field) { |
| 302 | descriptor = LLVM::InsertValueOp::create( |
| 303 | builder&: rewriter, location: loc, res: descriptorType, container: descriptor, value, |
| 304 | position: ArrayRef<int64_t>{getDataDescriptorFieldIndex(field)}); |
| 305 | }; |
| 306 | insert(baseDescriptor, Field::Base); |
| 307 | insert(rank, Field::Rank); |
| 308 | insert(elementSize, Field::ElementSize); |
| 309 | insert(lowerBoundsArray, Field::LowerBounds); |
| 310 | insert(upperBoundsArray, Field::UpperBounds); |
| 311 | insert(extentsArray, Field::Extents); |
| 312 | insert(stridesArray, Field::StridesInBytes); |
| 313 | insert(startsArray, Field::StartIndices); |
| 314 | |
| 315 | Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: 1); |
| 316 | Value storage = |
| 317 | LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: descriptorType, arraySize: one); |
| 318 | LLVM::StoreOp::create(builder&: rewriter, location: loc, value: descriptor, addr: storage); |
| 319 | return storage; |
| 320 | } |
| 321 | |
| 322 | Value mlir::createACCMemRefDescriptorWrapperArg( |
| 323 | Location loc, MemRefType memrefType, Value convertedMemref, |
| 324 | ValueRange bounds, Value elementSize, ConversionPatternRewriter &rewriter) { |
| 325 | MLIRContext *context = rewriter.getContext(); |
| 326 | Type i8Ty = rewriter.getI8Type(); |
| 327 | Type i32Ty = rewriter.getI32Type(); |
| 328 | Type ptrTy = LLVM::LLVMPointerType::get(context); |
| 329 | using Field = AccDataDescMemRefField; |
| 330 | Type descriptorType = |
| 331 | getDataDescriptorType(ctx: context, desc: DataDescriptor::AccDataDescMemRef); |
| 332 | auto version = static_cast<uint32_t>( |
| 333 | getDataDescriptorKind(desc: DataDescriptor::AccDataDescMemRef)); |
| 334 | Value descriptor = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: descriptorType); |
| 335 | Value rank = |
| 336 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i8Ty, value: memrefType.getRank()); |
| 337 | auto insert = [&](Value value, Field field) { |
| 338 | descriptor = LLVM::InsertValueOp::create( |
| 339 | builder&: rewriter, location: loc, res: descriptorType, container: descriptor, value, |
| 340 | position: ArrayRef<int64_t>{getDataDescriptorFieldIndex(field)}); |
| 341 | }; |
| 342 | insert(rank, Field::Rank); |
| 343 | insert(elementSize, Field::ElementSize); |
| 344 | insert(getPointer(loc, value: convertedMemref, rewriter), Field::MemRefDescriptor); |
| 345 | if (!bounds.empty()) |
| 346 | return createACCDataDescriptor(loc, baseDescriptor: descriptor, baseDescriptorType: descriptorType, version, |
| 347 | bounds, elementSize, rewriter); |
| 348 | |
| 349 | insert(LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: version), |
| 350 | Field::Version); |
| 351 | Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: 1); |
| 352 | Value storage = |
| 353 | LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: descriptorType, arraySize: one); |
| 354 | LLVM::StoreOp::create(builder&: rewriter, location: loc, value: descriptor, addr: storage); |
| 355 | return storage; |
| 356 | } |
| 357 | |
| 358 | Value mlir::createACCArgumentDescriptor(Operation *mapOp, |
| 359 | Value convertedOperand, |
| 360 | OpenACCSupport &accSupport, |
| 361 | ConversionPatternRewriter &rewriter) { |
| 362 | Location loc = mapOp->getLoc(); |
| 363 | MLIRContext *context = rewriter.getContext(); |
| 364 | Type i32Ty = rewriter.getI32Type(); |
| 365 | Type ptrTy = LLVM::LLVMPointerType::get(context); |
| 366 | SmallVector<Value> bounds = acc::getBounds(accDataClauseOp: mapOp); |
| 367 | DataDescKind descKind = acc::getDataDescKind(mapEntryOp: mapOp); |
| 368 | // Only a descriptor stating bounds reads the element size, so it is |
| 369 | // materialized where it is used, leaving no constant behind for a mapping |
| 370 | // that needs no descriptor at all. |
| 371 | auto elementSize = [&] { |
| 372 | return getElementSize(mapOp, accSupport, rewriter); |
| 373 | }; |
| 374 | |
| 375 | Value var = acc::getVar(accDataClauseOp: mapOp); |
| 376 | if (var && isa<MemRefType>(Val: var.getType())) { |
| 377 | auto memrefType = cast<MemRefType>(Val: var.getType()); |
| 378 | return createACCMemRefDescriptorWrapperArg( |
| 379 | loc, memrefType, convertedMemref: convertedOperand, bounds, elementSize: elementSize(), rewriter); |
| 380 | } |
| 381 | |
| 382 | if (bitEnumContainsAny(bits: descKind, bit: DataDescKind::cfi)) { |
| 383 | using Field = AccDataDescCFIField; |
| 384 | Type descriptorType = |
| 385 | getDataDescriptorType(ctx: context, desc: DataDescriptor::AccDataDescCFI); |
| 386 | auto version = static_cast<uint32_t>( |
| 387 | getDataDescriptorKind(desc: DataDescriptor::AccDataDescCFI)); |
| 388 | Value descriptor = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: descriptorType); |
| 389 | Value descriptorStorage = remapOrSelf(value: acc::getDesc(mapEntryOp: mapOp), rewriter); |
| 390 | if (!descriptorStorage) |
| 391 | descriptorStorage = convertedOperand; |
| 392 | descriptor = LLVM::InsertValueOp::create( |
| 393 | builder&: rewriter, location: loc, res: descriptorType, container: descriptor, |
| 394 | value: getPointer(loc, value: descriptorStorage, rewriter), |
| 395 | position: ArrayRef<int64_t>{getDataDescriptorFieldIndex(field: Field::CFIDescriptor)}); |
| 396 | if (!bounds.empty()) |
| 397 | return createACCDataDescriptor(loc, baseDescriptor: descriptor, baseDescriptorType: descriptorType, version, |
| 398 | bounds, elementSize: elementSize(), rewriter); |
| 399 | |
| 400 | Value versionValue = |
| 401 | LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: version); |
| 402 | descriptor = LLVM::InsertValueOp::create( |
| 403 | builder&: rewriter, location: loc, res: descriptorType, container: descriptor, value: versionValue, |
| 404 | position: ArrayRef<int64_t>{getDataDescriptorFieldIndex(field: Field::Version)}); |
| 405 | Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: 1); |
| 406 | Value storage = |
| 407 | LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: descriptorType, arraySize: one); |
| 408 | LLVM::StoreOp::create(builder&: rewriter, location: loc, value: descriptor, addr: storage); |
| 409 | return storage; |
| 410 | } |
| 411 | |
| 412 | if (!bounds.empty()) { |
| 413 | Type descriptorType = |
| 414 | getDataDescriptorType(ctx: context, desc: DataDescriptor::AccDataDescGeneric); |
| 415 | auto version = static_cast<uint32_t>( |
| 416 | getDataDescriptorKind(desc: DataDescriptor::AccDataDescGeneric)); |
| 417 | Value descriptor = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: descriptorType); |
| 418 | return createACCDataDescriptor(loc, baseDescriptor: descriptor, baseDescriptorType: descriptorType, version, |
| 419 | bounds, elementSize: elementSize(), rewriter); |
| 420 | } |
| 421 | return LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: ptrTy); |
| 422 | } |
| 423 | |
| 424 | SmallVector<Value> mlir::ACCDataRuntimeArgs::getCallArgs() const { |
| 425 | return {ident, flags, deviceType, argNum, argBasePtrs, argPtrs, |
| 426 | argSizes, argTypes, argNames, argMappers, argDescs}; |
| 427 | } |
| 428 | |
| 429 | LogicalResult mlir::emitACCDataRuntimeArgs( |
| 430 | Location loc, ValueRange mappingOperands, ValueRange convertedOperands, |
| 431 | ConversionPatternRewriter &rewriter, Region &globalSymbolRegion, |
| 432 | acc::OpenACCSupport &accSupport, const acc::ACCRuntimeCallConfig &config, |
| 433 | ACCDataRuntimeArgs &runtimeArgs, const ACCDataCallOptions &options, |
| 434 | SymbolTable *symbolTable) { |
| 435 | if (mappingOperands.size() != convertedOperands.size()) |
| 436 | return failure(); |
| 437 | |
| 438 | runtimeArgs.ident = |
| 439 | createIdent(loc, functionName: getParentFunctionName(values: mappingOperands), builder&: rewriter, |
| 440 | globalSymbolRegion, config, symbolTable); |
| 441 | runtimeArgs.flags = constantI64(loc, value: options.flags, builder&: rewriter); |
| 442 | runtimeArgs.deviceType = constantI64( |
| 443 | loc, value: config.getDeviceTypeRuntimeValue(type: DeviceType::None), builder&: rewriter); |
| 444 | runtimeArgs.argMappers = LLVM::ZeroOp::create( |
| 445 | builder&: rewriter, location: loc, res: LLVM::LLVMPointerType::get(context: rewriter.getContext())); |
| 446 | |
| 447 | // An aggregate passed by value is decomposed into one argument per field by |
| 448 | // the target ABI, so the runtime is told about one object per field as well. |
| 449 | // Each of them keeps the map entry of the aggregate it came from. The device |
| 450 | // address of an aggregate is returned in the slot of its first field. |
| 451 | struct MappedObject { |
| 452 | Value operand; |
| 453 | Value converted; |
| 454 | bool firstField; |
| 455 | }; |
| 456 | SmallVector<MappedObject> mappedObjects; |
| 457 | for (auto [operand, converted] : |
| 458 | llvm::zip_equal(t&: mappingOperands, u&: convertedOperands)) { |
| 459 | auto structType = dyn_cast<LLVM::LLVMStructType>(Val: converted.getType()); |
| 460 | if (structType && !structType.isIdentified() && |
| 461 | structType.getBody().size() > 1) { |
| 462 | for (unsigned field = 0, fields = structType.getBody().size(); |
| 463 | field != fields; ++field) |
| 464 | mappedObjects.push_back( |
| 465 | Elt: {.operand: operand, |
| 466 | .converted: LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: converted, position: field), |
| 467 | .firstField: field == 0}); |
| 468 | continue; |
| 469 | } |
| 470 | mappedObjects.push_back(Elt: {.operand: operand, .converted: converted, .firstField: true}); |
| 471 | } |
| 472 | |
| 473 | // The count precedes the values of the objects, which is where a decomposed |
| 474 | // aggregate states how many fields it became. |
| 475 | runtimeArgs.argNum = LLVM::ConstantOp::create( |
| 476 | builder&: rewriter, location: loc, type: rewriter.getI32Type(), value: mappedObjects.size()); |
| 477 | |
| 478 | constexpr unsigned noSlot = ~0u; |
| 479 | runtimeArgs.returnAddressSlots.assign(NumElts: options.returnAddressOperands.size(), |
| 480 | Elt: noSlot); |
| 481 | SmallVector<ACCMappedObjectArgs> objects; |
| 482 | for (auto [slot, object] : llvm::enumerate(First&: mappedObjects)) { |
| 483 | auto [operand, converted, firstField] = object; |
| 484 | Operation *mapOp = operand.getDefiningOp(); |
| 485 | if (!isDataEntryOp(op: mapOp)) |
| 486 | return failure(); |
| 487 | std::optional<MapFlags> mapFlags = computePackedMapFlags(mapOp); |
| 488 | if (!mapFlags) |
| 489 | return failure(); |
| 490 | *mapFlags = config.postProcessMapFlags(mapOp, flags: *mapFlags); |
| 491 | LLVM_DEBUG(llvm::dbgs() << "mapping " << *mapOp << "\n as " |
| 492 | << config.formatMapFlags(*mapFlags) << "\n" ); |
| 493 | |
| 494 | Location mapLoc = mapOp->getLoc(); |
| 495 | ACCMappedObjectArgs args; |
| 496 | args.pointer = createACCMappedObjectPointer(loc: mapLoc, operand, converted, |
| 497 | mapFlags: *mapFlags, rewriter); |
| 498 | // An object that only lives on the device has no host address to state as |
| 499 | // its base. |
| 500 | args.base = |
| 501 | bitEnumContainsAny(bits: *mapFlags, bit: MapFlags::device_resident) |
| 502 | ? getPointer(loc: mapLoc, value: Value(), rewriter) |
| 503 | : createACCMappedObjectBase(loc: mapLoc, mapOp, mapFlags: *mapFlags, rewriter); |
| 504 | |
| 505 | // A local object that lives on the device is torn down when the region |
| 506 | // that declared it ends, which the runtime only does for a mapping that |
| 507 | // states it is deleted. |
| 508 | if (options.callKind == ACCDataCallKind::DataExit && |
| 509 | bitEnumContainsAny(bits: *mapFlags, bit: MapFlags::device_resident)) { |
| 510 | Value var = acc::getVar(accDataClauseOp: mapOp); |
| 511 | if (var && |
| 512 | !isa_and_nonnull<AddressOfGlobalOpInterface>(Val: var.getDefiningOp())) |
| 513 | *mapFlags = *mapFlags | MapFlags::delete_; |
| 514 | } |
| 515 | |
| 516 | if (firstField) { |
| 517 | for (auto [index, returned] : |
| 518 | llvm::enumerate(First: options.returnAddressOperands)) { |
| 519 | if (returned != operand) |
| 520 | continue; |
| 521 | *mapFlags = *mapFlags | MapFlags::return_param; |
| 522 | runtimeArgs.returnAddressSlots[index] = slot; |
| 523 | } |
| 524 | } |
| 525 | int64_t mapType = config.getMapFlagsRuntimeValue(flags: *mapFlags); |
| 526 | |
| 527 | FailureOr<Value> size = createACCMappedObjectSize( |
| 528 | loc: mapLoc, operand, converted, mapFlags: *mapFlags, accSupport, rewriter); |
| 529 | if (failed(Result: size)) |
| 530 | return failure(); |
| 531 | args.size = *size; |
| 532 | args.type = constantI64(loc: mapLoc, value: mapType, builder&: rewriter); |
| 533 | args.name = createACCMappedObjectName( |
| 534 | loc: mapLoc, operand, accSupport, globalSymbolRegion, symbolTable, rewriter); |
| 535 | args.descriptor = |
| 536 | createACCArgumentDescriptor(mapOp, convertedOperand: converted, accSupport, rewriter); |
| 537 | objects.push_back(Elt: args); |
| 538 | } |
| 539 | if (llvm::is_contained(Range&: runtimeArgs.returnAddressSlots, Element: noSlot)) |
| 540 | return failure(); |
| 541 | |
| 542 | createACCMappedObjectArrays(loc, objects, runtimeArgs, rewriter); |
| 543 | return success(); |
| 544 | } |
| 545 | |
| 546 | FailureOr<SmallVector<Value>> mlir::emitACCDataRuntimeCall( |
| 547 | Location loc, RuntimeFunction fn, ValueRange mappingOperands, |
| 548 | ValueRange convertedOperands, ConversionPatternRewriter &rewriter, |
| 549 | OpenACCSupport &accSupport, Region &globalSymbolRegion, |
| 550 | SymbolTable &symbolTable, const ACCRuntimeCallConfig &config, |
| 551 | ArrayRef<Value> trailingArguments, const ACCDataCallOptions &options) { |
| 552 | SmallVector<Value> returnedAddresses; |
| 553 | if (mappingOperands.empty()) |
| 554 | return returnedAddresses; |
| 555 | |
| 556 | ACCDataRuntimeArgs runtimeArgs; |
| 557 | if (failed(Result: emitACCDataRuntimeArgs( |
| 558 | loc, mappingOperands, convertedOperands, rewriter, globalSymbolRegion, |
| 559 | accSupport, config, runtimeArgs, options, symbolTable: &symbolTable))) |
| 560 | return rewriter.notifyMatchFailure( |
| 561 | arg: mappingOperands.front().getDefiningOp(), |
| 562 | msg: "unsupported OpenACC data-clause operand" ); |
| 563 | |
| 564 | SmallVector<Value> arguments = runtimeArgs.getCallArgs(); |
| 565 | arguments.append(in_start: trailingArguments.begin(), in_end: trailingArguments.end()); |
| 566 | if (failed(Result: createRuntimeCall(loc, builder&: rewriter, globalSymbolRegion, symbolTable, |
| 567 | fn, config, arguments))) |
| 568 | return failure(); |
| 569 | |
| 570 | Type i32Ty = rewriter.getI32Type(); |
| 571 | Type ptrTy = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 572 | for (unsigned slot : runtimeArgs.returnAddressSlots) { |
| 573 | Value index = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: i32Ty, value: slot); |
| 574 | Value element = |
| 575 | LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrTy, elementType: ptrTy, |
| 576 | basePtr: runtimeArgs.argBasePtrs, indices: ValueRange{index}); |
| 577 | returnedAddresses.push_back( |
| 578 | Elt: LLVM::LoadOp::create(builder&: rewriter, location: loc, type: ptrTy, addr: element)); |
| 579 | } |
| 580 | return returnedAddresses; |
| 581 | } |
| 582 | |