| 1 | //===- MemRefInterfaces.cpp - OpenACC external models ---------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/OpenACC/Extensions/MemRefInterfaces.h" |
| 10 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 11 | #include "mlir/Dialect/GPU/IR/GPUBase.h" |
| 12 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 13 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 14 | #include "mlir/IR/BuiltinOps.h" |
| 15 | #include "mlir/IR/DialectRegistry.h" |
| 16 | |
| 17 | using namespace mlir; |
| 18 | using namespace mlir::acc; |
| 19 | |
| 20 | namespace { |
| 21 | static bool isScalarLikeType(Type type) { |
| 22 | return type.isIntOrIndexOrFloat() || isa<ComplexType>(Val: type); |
| 23 | } |
| 24 | |
| 25 | /// Helper function to attach the `VarName` attribute to an operation |
| 26 | /// if a variable name is provided. |
| 27 | static void attachVarNameAttr(Operation *op, OpBuilder &builder, |
| 28 | StringRef varName) { |
| 29 | if (!varName.empty()) { |
| 30 | auto varNameAttr = acc::VarNameAttr::get(context: builder.getContext(), name: varName); |
| 31 | op->setDiscardableAttr(name: acc::getVarNameAttrName(), value: varNameAttr); |
| 32 | } |
| 33 | } |
| 34 | |
| 35 | template <typename T> |
| 36 | struct MemRefPointerLikeModel |
| 37 | : public PointerLikeType::ExternalModel<MemRefPointerLikeModel<T>, T> { |
| 38 | Type getElementType(Type pointer) const { |
| 39 | return cast<T>(pointer).getElementType(); |
| 40 | } |
| 41 | |
| 42 | mlir::acc::VariableTypeCategory |
| 43 | getPointeeTypeCategory(Type pointer, TypedValue<PointerLikeType> varPtr, |
| 44 | Type varType) const { |
| 45 | if (auto mappableTy = dyn_cast<MappableType>(Val&: varType)) { |
| 46 | return mappableTy.getTypeCategory(var: varPtr); |
| 47 | } |
| 48 | auto memrefTy = cast<T>(pointer); |
| 49 | if (!memrefTy.hasRank()) { |
| 50 | // This memref is unranked - aka it could have any rank, including a |
| 51 | // rank of 0 which could mean scalar. For now, return uncategorized. |
| 52 | return mlir::acc::VariableTypeCategory::uncategorized; |
| 53 | } |
| 54 | |
| 55 | if (memrefTy.getRank() == 0) { |
| 56 | if (isScalarLikeType(memrefTy.getElementType())) { |
| 57 | return mlir::acc::VariableTypeCategory::scalar; |
| 58 | } |
| 59 | // Zero-rank non-scalar - need further analysis to determine the type |
| 60 | // category. For now, return uncategorized. |
| 61 | return mlir::acc::VariableTypeCategory::uncategorized; |
| 62 | } |
| 63 | |
| 64 | // It has a rank - must be an array. |
| 65 | assert(memrefTy.getRank() > 0 && "rank expected to be positive" ); |
| 66 | return mlir::acc::VariableTypeCategory::array; |
| 67 | } |
| 68 | |
| 69 | mlir::Value genAllocate(Type pointer, OpBuilder &builder, Location loc, |
| 70 | StringRef varName, Type varType, Value originalVar, |
| 71 | bool &needsFree) const { |
| 72 | auto memrefTy = cast<MemRefType>(Val&: pointer); |
| 73 | |
| 74 | // Check if this is a static memref (all dimensions are known) - if yes |
| 75 | // then we can generate an alloca operation. |
| 76 | if (memrefTy.hasStaticShape()) { |
| 77 | needsFree = false; // alloca doesn't need deallocation |
| 78 | auto allocaOp = memref::AllocaOp::create(builder, location: loc, memrefType: memrefTy); |
| 79 | attachVarNameAttr(op: allocaOp, builder, varName); |
| 80 | return allocaOp.getResult(); |
| 81 | } |
| 82 | |
| 83 | // For dynamic memrefs, extract sizes from the original variable if |
| 84 | // provided. Otherwise they cannot be handled. |
| 85 | if (originalVar && originalVar.getType() == memrefTy && |
| 86 | memrefTy.hasRank()) { |
| 87 | SmallVector<Value> dynamicSizes; |
| 88 | for (int64_t i = 0; i < memrefTy.getRank(); ++i) { |
| 89 | if (memrefTy.isDynamicDim(idx: i)) { |
| 90 | // Extract the size of dimension i from the original variable |
| 91 | auto indexValue = arith::ConstantIndexOp::create(builder, location: loc, value: i); |
| 92 | auto dimSize = |
| 93 | memref::DimOp::create(builder, location: loc, source: originalVar, index: indexValue); |
| 94 | dynamicSizes.push_back(Elt: dimSize); |
| 95 | } |
| 96 | // Note: We only add dynamic sizes to the dynamicSizes array |
| 97 | // Static dimensions are handled automatically by AllocOp |
| 98 | } |
| 99 | needsFree = true; // alloc needs deallocation |
| 100 | auto allocOp = |
| 101 | memref::AllocOp::create(builder, location: loc, memrefType: memrefTy, dynamicSizes); |
| 102 | attachVarNameAttr(op: allocOp, builder, varName); |
| 103 | return allocOp.getResult(); |
| 104 | } |
| 105 | |
| 106 | // TODO: Unranked not yet supported. |
| 107 | return {}; |
| 108 | } |
| 109 | |
| 110 | bool genFree(Type pointer, OpBuilder &builder, Location loc, |
| 111 | TypedValue<PointerLikeType> varToFree, Value allocRes, |
| 112 | Type varType) const { |
| 113 | if (auto memrefValue = dyn_cast<TypedValue<MemRefType>>(Val&: varToFree)) { |
| 114 | // Use allocRes if provided to determine the allocation type |
| 115 | Value valueToInspect = allocRes ? allocRes : memrefValue; |
| 116 | |
| 117 | // Walk through casts to find the original allocation |
| 118 | Value currentValue = valueToInspect; |
| 119 | Operation *originalAlloc = nullptr; |
| 120 | |
| 121 | // Follow the chain of operations to find the original allocation |
| 122 | // even if a casted result is provided. |
| 123 | while (currentValue) { |
| 124 | if (auto *definingOp = currentValue.getDefiningOp()) { |
| 125 | // Check if this is an allocation operation |
| 126 | if (isa<memref::AllocOp, memref::AllocaOp>(Val: definingOp)) { |
| 127 | originalAlloc = definingOp; |
| 128 | break; |
| 129 | } |
| 130 | |
| 131 | // Check if this is a cast operation we can look through |
| 132 | if (auto castOp = dyn_cast<memref::CastOp>(Val: definingOp)) { |
| 133 | currentValue = castOp.getSource(); |
| 134 | continue; |
| 135 | } |
| 136 | |
| 137 | // Check for other cast-like operations |
| 138 | if (auto reinterpretCastOp = |
| 139 | dyn_cast<memref::ReinterpretCastOp>(Val: definingOp)) { |
| 140 | currentValue = reinterpretCastOp.getSource(); |
| 141 | continue; |
| 142 | } |
| 143 | |
| 144 | // If we can't look through this operation, stop |
| 145 | break; |
| 146 | } |
| 147 | // This is a block argument or similar - can't trace further. |
| 148 | break; |
| 149 | } |
| 150 | |
| 151 | if (originalAlloc) { |
| 152 | if (isa<memref::AllocaOp>(Val: originalAlloc)) { |
| 153 | // This is an alloca - no dealloc needed, but return true (success) |
| 154 | return true; |
| 155 | } |
| 156 | if (isa<memref::AllocOp>(Val: originalAlloc)) { |
| 157 | // This is an alloc - generate dealloc on varToFree |
| 158 | memref::DeallocOp::create(builder, location: loc, memref: memrefValue); |
| 159 | return true; |
| 160 | } |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | return false; |
| 165 | } |
| 166 | |
| 167 | bool genCopy(Type pointer, OpBuilder &builder, Location loc, |
| 168 | TypedValue<PointerLikeType> destination, |
| 169 | TypedValue<PointerLikeType> source, Type varType) const { |
| 170 | // Generate a copy operation between two memrefs |
| 171 | auto destMemref = dyn_cast_if_present<TypedValue<MemRefType>>(Val&: destination); |
| 172 | auto srcMemref = dyn_cast_if_present<TypedValue<MemRefType>>(Val&: source); |
| 173 | |
| 174 | // As per memref documentation, source and destination must have same |
| 175 | // element type and shape in order to be compatible. We do not want to fail |
| 176 | // with an IR verification error - thus check that before generating the |
| 177 | // copy operation. |
| 178 | if (destMemref && srcMemref && |
| 179 | destMemref.getType().getElementType() == |
| 180 | srcMemref.getType().getElementType() && |
| 181 | destMemref.getType().getShape() == srcMemref.getType().getShape()) { |
| 182 | memref::CopyOp::create(builder, location: loc, source: srcMemref, target: destMemref); |
| 183 | return true; |
| 184 | } |
| 185 | |
| 186 | return false; |
| 187 | } |
| 188 | |
| 189 | mlir::Value genLoad(Type pointer, OpBuilder &builder, Location loc, |
| 190 | TypedValue<PointerLikeType> srcPtr, |
| 191 | Type valueType) const { |
| 192 | // Load from a memref - only valid for scalar memrefs (rank 0). |
| 193 | // This is because the address computation for memrefs is part of the load |
| 194 | // (and not computed separately), but the API does not have arguments for |
| 195 | // indexing. |
| 196 | auto memrefValue = dyn_cast_if_present<TypedValue<MemRefType>>(Val&: srcPtr); |
| 197 | if (!memrefValue) |
| 198 | return {}; |
| 199 | |
| 200 | auto memrefTy = memrefValue.getType(); |
| 201 | |
| 202 | // Only load from scalar memrefs (rank 0) |
| 203 | if (memrefTy.getRank() != 0) |
| 204 | return {}; |
| 205 | |
| 206 | return memref::LoadOp::create(builder, location: loc, memref: memrefValue, indices: ValueRange{}); |
| 207 | } |
| 208 | |
| 209 | bool genStore(Type pointer, OpBuilder &builder, Location loc, |
| 210 | Value valueToStore, TypedValue<PointerLikeType> destPtr) const { |
| 211 | // Store to a memref - only valid for scalar memrefs (rank 0) |
| 212 | // This is because the address computation for memrefs is part of the store |
| 213 | // (and not computed separately), but the API does not have arguments for |
| 214 | // indexing. |
| 215 | auto memrefValue = dyn_cast_if_present<TypedValue<MemRefType>>(Val&: destPtr); |
| 216 | if (!memrefValue) |
| 217 | return false; |
| 218 | |
| 219 | auto memrefTy = memrefValue.getType(); |
| 220 | |
| 221 | // Only store to scalar memrefs (rank 0) |
| 222 | if (memrefTy.getRank() != 0) |
| 223 | return false; |
| 224 | |
| 225 | memref::StoreOp::create(builder, location: loc, valueToStore, memref: memrefValue); |
| 226 | return true; |
| 227 | } |
| 228 | |
| 229 | Value genCast(Type, OpBuilder &builder, Location loc, Value value, |
| 230 | Type resultType) const { |
| 231 | if (value.getType() == resultType) |
| 232 | return value; |
| 233 | |
| 234 | if (isa<BaseMemRefType>(Val: value.getType()) && |
| 235 | isa<BaseMemRefType>(Val: resultType)) { |
| 236 | if (memref::CastOp::areCastCompatible(inputs: TypeRange(value.getType()), |
| 237 | outputs: TypeRange(resultType))) |
| 238 | return memref::CastOp::create(builder, location: loc, dest: resultType, source: value); |
| 239 | if (memref::MemorySpaceCastOp::areCastCompatible( |
| 240 | inputs: TypeRange(value.getType()), outputs: TypeRange(resultType))) |
| 241 | return memref::MemorySpaceCastOp::create(builder, location: loc, dest: resultType, |
| 242 | source: value); |
| 243 | } |
| 244 | |
| 245 | // If one side is not a memref, try the other type's `PointerLikeType` |
| 246 | // implementation (since it may be an out-of-tree reference type that |
| 247 | // we cannot generate here). |
| 248 | if (auto resPtrLike = dyn_cast<PointerLikeType>(Val&: resultType)) |
| 249 | if (!isa<BaseMemRefType>(Val: resPtrLike)) |
| 250 | if (Value v = resPtrLike.genCast(builder, loc, value, resultType)) |
| 251 | return v; |
| 252 | if (auto valPtrLike = dyn_cast<PointerLikeType>(Val: value.getType())) |
| 253 | if (!isa<BaseMemRefType>(Val: valPtrLike)) |
| 254 | if (Value v = valPtrLike.genCast(builder, loc, value, resultType)) |
| 255 | return v; |
| 256 | |
| 257 | return {}; |
| 258 | } |
| 259 | |
| 260 | bool isDeviceAccessible(Type pointer, Value var, |
| 261 | SymbolTableCollection *symbolTables) const { |
| 262 | auto memrefTy = cast<T>(pointer); |
| 263 | Attribute memSpace = memrefTy.getMemorySpace(); |
| 264 | return isa_and_nonnull<gpu::AddressSpaceAttr>(Val: memSpace); |
| 265 | } |
| 266 | |
| 267 | MemRefType getAsMemRefType(Type pointer, ModuleOp module) const { |
| 268 | (void)module; |
| 269 | return dyn_cast<MemRefType>(Val&: pointer); |
| 270 | } |
| 271 | }; |
| 272 | |
| 273 | struct MemrefAddressOfGlobalModel |
| 274 | : public AddressOfGlobalOpInterface::ExternalModel< |
| 275 | MemrefAddressOfGlobalModel, memref::GetGlobalOp> { |
| 276 | SymbolRefAttr getSymbol(Operation *op) const { |
| 277 | auto getGlobalOp = cast<memref::GetGlobalOp>(Val: op); |
| 278 | return getGlobalOp.getNameAttr(); |
| 279 | } |
| 280 | }; |
| 281 | |
| 282 | struct MemrefGlobalVariableModel |
| 283 | : public GlobalVariableOpInterface::ExternalModel<MemrefGlobalVariableModel, |
| 284 | memref::GlobalOp> { |
| 285 | bool isConstant(Operation *op) const { |
| 286 | auto globalOp = cast<memref::GlobalOp>(Val: op); |
| 287 | return globalOp.getConstant(); |
| 288 | } |
| 289 | |
| 290 | bool hasInitializer(Operation *op) const { |
| 291 | auto globalOp = cast<memref::GlobalOp>(Val: op); |
| 292 | return globalOp.getInitialValue().has_value(); |
| 293 | } |
| 294 | |
| 295 | Region *getInitRegion(Operation *op) const { |
| 296 | // GlobalOp uses attributes for initialization, not regions |
| 297 | return nullptr; |
| 298 | } |
| 299 | |
| 300 | bool isDeviceAccessible(Operation *op) const { |
| 301 | auto globalOp = cast<memref::GlobalOp>(Val: op); |
| 302 | Attribute memSpace = globalOp.getType().getMemorySpace(); |
| 303 | return isa_and_nonnull<gpu::AddressSpaceAttr>(Val: memSpace); |
| 304 | } |
| 305 | |
| 306 | bool isInDeviceMemory(Operation *op) const { |
| 307 | // A memref address space models storage that is physically resident on the |
| 308 | // device, so a device-accessible global is also in device memory. (There |
| 309 | // is no host-shared/migratable address space to exclude here.) |
| 310 | return isDeviceAccessible(op); |
| 311 | } |
| 312 | |
| 313 | bool isCompilerGenerated(Operation *op) const { return false; } |
| 314 | }; |
| 315 | } // namespace |
| 316 | |
| 317 | void mlir::acc::registerMemRefInterfaces(DialectRegistry ®istry) { |
| 318 | // MemRef types belong to BuiltinDialect, so IR can use them without any |
| 319 | // memref operations. Install their models as soon as OpenACC is loaded. |
| 320 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, OpenACCDialect *) { |
| 321 | ctx->getOrLoadDialect<memref::MemRefDialect>(); |
| 322 | ctx->getOrLoadDialect<arith::ArithDialect>(); |
| 323 | MemRefType::attachInterface<MemRefPointerLikeModel<MemRefType>>(context&: *ctx); |
| 324 | UnrankedMemRefType::attachInterface< |
| 325 | MemRefPointerLikeModel<UnrankedMemRefType>>(context&: *ctx); |
| 326 | memref::GetGlobalOp::attachInterface<MemrefAddressOfGlobalModel>(context&: *ctx); |
| 327 | memref::GlobalOp::attachInterface<MemrefGlobalVariableModel>(context&: *ctx); |
| 328 | }); |
| 329 | } |
| 330 | |