| 1 | //===- GPUOpsLowering.cpp - GPU FuncOp / ReturnOp lowering ----------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "GPUOpsLowering.h" |
| 10 | |
| 11 | #include "mlir/Conversion/GPUCommon/GPUCommonPass.h" |
| 12 | #include "mlir/Conversion/LLVMCommon/VectorPattern.h" |
| 13 | #include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h" |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h" |
| 20 | #include "mlir/Dialect/LLVMIR/LLVMVectorOps.h" |
| 21 | #include "mlir/IR/Attributes.h" |
| 22 | #include "mlir/IR/Builders.h" |
| 23 | #include "mlir/IR/BuiltinTypes.h" |
| 24 | #include "mlir/IR/SymbolTable.h" |
| 25 | #include "llvm/ADT/SmallVectorExtras.h" |
| 26 | #include "llvm/ADT/StringSet.h" |
| 27 | #include "llvm/Support/DebugLog.h" |
| 28 | #include "llvm/Support/FormatVariadic.h" |
| 29 | |
| 30 | #define DEBUG_TYPE "gpu-lowering" |
| 31 | |
| 32 | using namespace mlir; |
| 33 | |
| 34 | LLVM::LLVMFuncOp mlir::getOrDefineFunction(Operation *moduleOp, Location loc, |
| 35 | OpBuilder &b, StringRef name, |
| 36 | LLVM::LLVMFunctionType type) { |
| 37 | auto existing = dyn_cast_or_null<LLVM::LLVMFuncOp>( |
| 38 | Val: SymbolTable::lookupSymbolIn(op: moduleOp, symbol: name)); |
| 39 | if (existing) |
| 40 | return existing; |
| 41 | |
| 42 | OpBuilder::InsertionGuard guard(b); |
| 43 | b.setInsertionPointToStart(&moduleOp->getRegion(index: 0).front()); |
| 44 | return LLVM::LLVMFuncOp::create(builder&: b, location: loc, name, type, linkage: LLVM::Linkage::External); |
| 45 | } |
| 46 | |
| 47 | static SmallString<16> getUniqueSymbolName(Operation *moduleOp, |
| 48 | StringRef prefix) { |
| 49 | // Get a unique global name. |
| 50 | unsigned stringNumber = 0; |
| 51 | SmallString<16> stringConstName; |
| 52 | do { |
| 53 | stringConstName.clear(); |
| 54 | (prefix + Twine(stringNumber++)).toStringRef(Out&: stringConstName); |
| 55 | } while (SymbolTable::lookupSymbolIn(op: moduleOp, symbol: stringConstName)); |
| 56 | return stringConstName; |
| 57 | } |
| 58 | |
| 59 | LLVM::GlobalOp mlir::getOrCreateStringConstant(OpBuilder &b, Location loc, |
| 60 | Operation *moduleOp, Type llvmI8, |
| 61 | StringRef namePrefix, |
| 62 | StringRef str, |
| 63 | uint64_t alignment, |
| 64 | unsigned addrSpace) { |
| 65 | llvm::SmallString<20> nullTermStr(str); |
| 66 | nullTermStr.push_back(Elt: '\0'); // Null terminate for C |
| 67 | auto globalType = |
| 68 | LLVM::LLVMArrayType::get(elementType: llvmI8, numElements: nullTermStr.size_in_bytes()); |
| 69 | StringAttr attr = b.getStringAttr(bytes: nullTermStr); |
| 70 | |
| 71 | // Try to find existing global. |
| 72 | for (auto globalOp : moduleOp->getRegion(index: 0).getOps<LLVM::GlobalOp>()) |
| 73 | if (globalOp.getGlobalType() == globalType && globalOp.getConstant() && |
| 74 | globalOp.getValueAttr() == attr && |
| 75 | globalOp.getAlignment().value_or(u: 0) == alignment && |
| 76 | globalOp.getAddrSpace() == addrSpace) |
| 77 | return globalOp; |
| 78 | |
| 79 | // Not found: create new global. |
| 80 | OpBuilder::InsertionGuard guard(b); |
| 81 | b.setInsertionPointToStart(&moduleOp->getRegion(index: 0).front()); |
| 82 | SmallString<16> name = getUniqueSymbolName(moduleOp, prefix: namePrefix); |
| 83 | return LLVM::GlobalOp::create(builder&: b, location: loc, type: globalType, |
| 84 | /*isConstant=*/true, linkage: LLVM::Linkage::Internal, |
| 85 | name, value: attr, alignment, addrSpace); |
| 86 | } |
| 87 | |
| 88 | FailureOr<LoweredLLVMFuncAttrs> GPUFuncOpLowering::buildLoweredGPULLVMFuncAttrs( |
| 89 | gpu::GPUFuncOp gpuFuncOp, Type llvmFuncType, OpBuilder &rewriter) const { |
| 90 | FailureOr<LoweredLLVMFuncAttrs> loweredAttrs = |
| 91 | lowerDiscardableAttrsForLLVMFunc(funcOp: gpuFuncOp, llvmFuncType); |
| 92 | if (failed(Result: loweredAttrs)) |
| 93 | return failure(); |
| 94 | |
| 95 | MLIRContext *ctx = rewriter.getContext(); |
| 96 | LLVM::LLVMFuncOp::Properties &props = loweredAttrs->properties; |
| 97 | props.sym_name = rewriter.getStringAttr(bytes: gpuFuncOp.getName()); |
| 98 | props.function_type = TypeAttr::get(type: llvmFuncType); |
| 99 | const bool isKernelFunc = gpuFuncOp.isKernel(); |
| 100 | props.setCConv(LLVM::CConvAttr::get(context: ctx, CallingConv: isKernelFunc |
| 101 | ? kernelCallingConvention |
| 102 | : nonKernelCallingConvention)); |
| 103 | |
| 104 | NamedAttrList &discardable = loweredAttrs->discardableAttrs; |
| 105 | auto *gpuDialect = cast<gpu::GPUDialect>(Val: gpuFuncOp->getDialect()); |
| 106 | |
| 107 | auto appendIfNameAndValue = [&](StringAttr name, Attribute value) { |
| 108 | if (name && value) |
| 109 | discardable.append(name, attr: value); |
| 110 | }; |
| 111 | |
| 112 | DenseI32ArrayAttr knownBlockSize = gpuFuncOp.getKnownBlockSizeAttr(); |
| 113 | DenseI32ArrayAttr knownGridSize = gpuFuncOp.getKnownGridSizeAttr(); |
| 114 | DenseI32ArrayAttr knownClusterSize = gpuFuncOp.getKnownClusterSizeAttr(); |
| 115 | |
| 116 | appendIfNameAndValue(gpuDialect->getKnownBlockSizeAttrHelper().getName(), |
| 117 | knownBlockSize); |
| 118 | appendIfNameAndValue(gpuDialect->getKnownGridSizeAttrHelper().getName(), |
| 119 | knownGridSize); |
| 120 | appendIfNameAndValue(gpuDialect->getKnownClusterSizeAttrHelper().getName(), |
| 121 | knownClusterSize); |
| 122 | |
| 123 | if (isKernelFunc) { |
| 124 | discardable.append(name: gpuDialect->getKernelFuncAttrName(), |
| 125 | attr: rewriter.getUnitAttr()); |
| 126 | // Add a dialect specific kernel attribute in addition to GPU kernel |
| 127 | // attribute. The former is necessary for further translation while the |
| 128 | // latter is expected by gpu.launch_func. |
| 129 | appendIfNameAndValue(kernelAttributeName, rewriter.getUnitAttr()); |
| 130 | appendIfNameAndValue(kernelBlockSizeAttributeName, knownBlockSize); |
| 131 | appendIfNameAndValue(kernelClusterSizeAttributeName, knownClusterSize); |
| 132 | } |
| 133 | |
| 134 | return loweredAttrs; |
| 135 | } |
| 136 | |
| 137 | LogicalResult |
| 138 | GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor, |
| 139 | ConversionPatternRewriter &rewriter) const { |
| 140 | Location loc = gpuFuncOp.getLoc(); |
| 141 | |
| 142 | SmallVector<LLVM::GlobalOp, 3> workgroupBuffers; |
| 143 | if (encodeWorkgroupAttributionsAsArguments) { |
| 144 | // Append an `llvm.ptr` argument to the function signature to encode |
| 145 | // workgroup attributions. |
| 146 | |
| 147 | ArrayRef<BlockArgument> workgroupAttributions = |
| 148 | gpuFuncOp.getWorkgroupAttributionBBArgs(); |
| 149 | size_t numAttributions = workgroupAttributions.size(); |
| 150 | |
| 151 | // Insert all arguments at the end. |
| 152 | unsigned index = gpuFuncOp.getNumArguments(); |
| 153 | SmallVector<unsigned> argIndices(numAttributions, index); |
| 154 | |
| 155 | // New arguments will simply be `llvm.ptr` with the correct address space |
| 156 | Type workgroupPtrType = |
| 157 | rewriter.getType<LLVM::LLVMPointerType>(args: workgroupAddrSpace); |
| 158 | Repeated<Type> argTypes(numAttributions, workgroupPtrType); |
| 159 | |
| 160 | // Attributes: noalias, llvm.mlir.workgroup_attribution(<size>, <type>) |
| 161 | std::array attrs{ |
| 162 | rewriter.getNamedAttr(name: LLVM::LLVMDialect::getNoAliasAttrName(), |
| 163 | val: rewriter.getUnitAttr()), |
| 164 | rewriter.getNamedAttr( |
| 165 | name: getDialect().getWorkgroupAttributionAttrHelper().getName(), |
| 166 | val: rewriter.getUnitAttr()), |
| 167 | }; |
| 168 | SmallVector<DictionaryAttr> argAttrs; |
| 169 | for (BlockArgument attribution : workgroupAttributions) { |
| 170 | auto attributionType = cast<MemRefType>(Val: attribution.getType()); |
| 171 | IntegerAttr numElements = |
| 172 | rewriter.getI64IntegerAttr(value: attributionType.getNumElements()); |
| 173 | Type llvmElementType = |
| 174 | getTypeConverter()->convertType(t: attributionType.getElementType()); |
| 175 | if (!llvmElementType) |
| 176 | return failure(); |
| 177 | TypeAttr type = TypeAttr::get(type: llvmElementType); |
| 178 | attrs.back().setValue( |
| 179 | rewriter.getAttr<LLVM::WorkgroupAttributionAttr>(args&: numElements, args&: type)); |
| 180 | argAttrs.push_back(Elt: rewriter.getDictionaryAttr(value: attrs)); |
| 181 | } |
| 182 | |
| 183 | // Location match function location |
| 184 | SmallVector<Location> argLocs(numAttributions, gpuFuncOp.getLoc()); |
| 185 | |
| 186 | // Perform signature modification |
| 187 | rewriter.modifyOpInPlace( |
| 188 | root: gpuFuncOp, callable: [gpuFuncOp, &argIndices, &argTypes, &argAttrs, &argLocs]() { |
| 189 | LogicalResult inserted = |
| 190 | static_cast<FunctionOpInterface>(gpuFuncOp).insertArguments( |
| 191 | argIndices, argTypes, argAttrs, argLocs); |
| 192 | (void)inserted; |
| 193 | assert(succeeded(inserted) && |
| 194 | "expected GPU funcs to support inserting any argument" ); |
| 195 | }); |
| 196 | } else { |
| 197 | workgroupBuffers.reserve(N: gpuFuncOp.getNumWorkgroupAttributions()); |
| 198 | for (auto [idx, attribution] : |
| 199 | llvm::enumerate(First: gpuFuncOp.getWorkgroupAttributionBBArgs())) { |
| 200 | auto type = dyn_cast<MemRefType>(Val: attribution.getType()); |
| 201 | assert(type && type.hasStaticShape() && "unexpected type in attribution" ); |
| 202 | |
| 203 | uint64_t numElements = type.getNumElements(); |
| 204 | |
| 205 | auto elementType = |
| 206 | cast<Type>(Val: typeConverter->convertType(t: type.getElementType())); |
| 207 | auto arrayType = LLVM::LLVMArrayType::get(elementType, numElements); |
| 208 | std::string name = |
| 209 | std::string(llvm::formatv(Fmt: "__wg_{0}_{1}" , Vals: gpuFuncOp.getName(), Vals&: idx)); |
| 210 | uint64_t alignment = 0; |
| 211 | if (auto alignAttr = dyn_cast_or_null<IntegerAttr>( |
| 212 | Val: gpuFuncOp.getWorkgroupAttributionAttr( |
| 213 | index: idx, name: LLVM::LLVMDialect::getAlignAttrName()))) |
| 214 | alignment = alignAttr.getInt(); |
| 215 | auto globalOp = LLVM::GlobalOp::create( |
| 216 | builder&: rewriter, location: gpuFuncOp.getLoc(), type: arrayType, /*isConstant=*/false, |
| 217 | linkage: LLVM::Linkage::Internal, name, /*value=*/Attribute(), alignment, |
| 218 | addrSpace: workgroupAddrSpace); |
| 219 | workgroupBuffers.push_back(Elt: globalOp); |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | // Remap proper input types. |
| 224 | TypeConverter::SignatureConversion signatureConversion( |
| 225 | gpuFuncOp.front().getNumArguments()); |
| 226 | |
| 227 | Type funcType = getTypeConverter()->convertFunctionSignature( |
| 228 | funcTy: gpuFuncOp.getFunctionType(), /*isVariadic=*/false, |
| 229 | useBarePtrCallConv: getTypeConverter()->getOptions().useBarePtrCallConv, result&: signatureConversion); |
| 230 | if (!funcType) { |
| 231 | return rewriter.notifyMatchFailure(op: gpuFuncOp, reasonCallback: [&](Diagnostic &diag) { |
| 232 | diag << "failed to convert function signature type for: " |
| 233 | << gpuFuncOp.getFunctionType(); |
| 234 | }); |
| 235 | } |
| 236 | |
| 237 | ArrayAttr argAttrs = gpuFuncOp.getArgAttrsAttr(); |
| 238 | |
| 239 | FailureOr<LoweredLLVMFuncAttrs> loweredAttrs = |
| 240 | buildLoweredGPULLVMFuncAttrs(gpuFuncOp, llvmFuncType: funcType, rewriter); |
| 241 | if (failed(Result: loweredAttrs)) |
| 242 | return rewriter.notifyMatchFailure(arg&: gpuFuncOp, |
| 243 | msg: "failed to lower func attributes" ); |
| 244 | |
| 245 | auto llvmFuncOp = LLVM::LLVMFuncOp::create(builder&: rewriter, location: gpuFuncOp.getLoc(), |
| 246 | properties: loweredAttrs->properties, |
| 247 | discardableAttributes: loweredAttrs->discardableAttrs); |
| 248 | |
| 249 | { |
| 250 | // Insert operations that correspond to converted workgroup and private |
| 251 | // memory attributions to the body of the function. This must operate on |
| 252 | // the original function, before the body region is inlined in the new |
| 253 | // function to maintain the relation between block arguments and the |
| 254 | // parent operation that assigns their semantics. |
| 255 | OpBuilder::InsertionGuard guard(rewriter); |
| 256 | |
| 257 | // Rewrite workgroup memory attributions to addresses of global buffers. |
| 258 | rewriter.setInsertionPointToStart(&gpuFuncOp.front()); |
| 259 | unsigned numProperArguments = gpuFuncOp.getNumArguments(); |
| 260 | |
| 261 | if (encodeWorkgroupAttributionsAsArguments) { |
| 262 | // Build a MemRefDescriptor with each of the arguments added above. |
| 263 | |
| 264 | unsigned numAttributions = gpuFuncOp.getNumWorkgroupAttributions(); |
| 265 | assert(numProperArguments >= numAttributions && |
| 266 | "Expecting attributions to be encoded as arguments already" ); |
| 267 | |
| 268 | // Arguments encoding workgroup attributions will be in positions |
| 269 | // [numProperArguments, numProperArguments+numAttributions) |
| 270 | ArrayRef<BlockArgument> attributionArguments = |
| 271 | gpuFuncOp.getArguments().slice(N: numProperArguments - numAttributions, |
| 272 | M: numAttributions); |
| 273 | for (auto [idx, vals] : llvm::enumerate( |
| 274 | First: llvm::zip_equal(t: gpuFuncOp.getWorkgroupAttributionBBArgs(), |
| 275 | u&: attributionArguments))) { |
| 276 | auto [attribution, arg] = vals; |
| 277 | auto type = cast<MemRefType>(Val: attribution.getType()); |
| 278 | |
| 279 | // Arguments are of llvm.ptr type and attributions are of memref type: |
| 280 | // we need to wrap them in memref descriptors. |
| 281 | Value descr = MemRefDescriptor::fromStaticShape( |
| 282 | builder&: rewriter, loc, typeConverter: *getTypeConverter(), type, memory: arg); |
| 283 | |
| 284 | // And remap the arguments |
| 285 | signatureConversion.remapInput(origInputNo: numProperArguments + idx, replacements: descr); |
| 286 | } |
| 287 | } else { |
| 288 | for (const auto [idx, global] : llvm::enumerate(First&: workgroupBuffers)) { |
| 289 | auto ptrType = LLVM::LLVMPointerType::get(context: rewriter.getContext(), |
| 290 | addressSpace: global.getAddrSpace()); |
| 291 | Value address = LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: ptrType, |
| 292 | global_name: global.getSymNameAttr()); |
| 293 | Value memory = |
| 294 | LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: global.getType(), |
| 295 | basePtr: address, indices: ArrayRef<LLVM::GEPArg>{0, 0}); |
| 296 | |
| 297 | // Build a memref descriptor pointing to the buffer to plug with the |
| 298 | // existing memref infrastructure. This may use more registers than |
| 299 | // otherwise necessary given that memref sizes are fixed, but we can try |
| 300 | // and canonicalize that away later. |
| 301 | Value attribution = gpuFuncOp.getWorkgroupAttributionBBArgs()[idx]; |
| 302 | auto type = cast<MemRefType>(Val: attribution.getType()); |
| 303 | Value descr = MemRefDescriptor::fromStaticShape( |
| 304 | builder&: rewriter, loc, typeConverter: *getTypeConverter(), type, memory); |
| 305 | signatureConversion.remapInput(origInputNo: numProperArguments + idx, replacements: descr); |
| 306 | } |
| 307 | } |
| 308 | |
| 309 | // Rewrite private memory attributions to alloca'ed buffers. |
| 310 | unsigned numWorkgroupAttributions = gpuFuncOp.getNumWorkgroupAttributions(); |
| 311 | auto int64Ty = IntegerType::get(context: rewriter.getContext(), width: 64); |
| 312 | for (const auto [idx, attribution] : |
| 313 | llvm::enumerate(First: gpuFuncOp.getPrivateAttributions())) { |
| 314 | auto type = cast<MemRefType>(Val: attribution.getType()); |
| 315 | assert(type && type.hasStaticShape() && "unexpected type in attribution" ); |
| 316 | |
| 317 | // Explicitly drop memory space when lowering private memory |
| 318 | // attributions since NVVM models it as `alloca`s in the default |
| 319 | // memory space and does not support `alloca`s with addrspace(5). |
| 320 | Type elementType = typeConverter->convertType(t: type.getElementType()); |
| 321 | auto ptrType = |
| 322 | LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: allocaAddrSpace); |
| 323 | Value numElements = LLVM::ConstantOp::create( |
| 324 | builder&: rewriter, location: gpuFuncOp.getLoc(), type: int64Ty, value: type.getNumElements()); |
| 325 | uint64_t alignment = 0; |
| 326 | if (auto alignAttr = |
| 327 | dyn_cast_or_null<IntegerAttr>(Val: gpuFuncOp.getPrivateAttributionAttr( |
| 328 | index: idx, name: LLVM::LLVMDialect::getAlignAttrName()))) |
| 329 | alignment = alignAttr.getInt(); |
| 330 | Value allocated = |
| 331 | LLVM::AllocaOp::create(builder&: rewriter, location: gpuFuncOp.getLoc(), resultType: ptrType, |
| 332 | elementType, arraySize: numElements, alignment); |
| 333 | Value descr = MemRefDescriptor::fromStaticShape( |
| 334 | builder&: rewriter, loc, typeConverter: *getTypeConverter(), type, memory: allocated); |
| 335 | signatureConversion.remapInput( |
| 336 | origInputNo: numProperArguments + numWorkgroupAttributions + idx, replacements: descr); |
| 337 | } |
| 338 | } |
| 339 | |
| 340 | // Move the region to the new function, update the entry block signature. |
| 341 | rewriter.inlineRegionBefore(region&: gpuFuncOp.getBody(), parent&: llvmFuncOp.getBody(), |
| 342 | before: llvmFuncOp.end()); |
| 343 | if (failed(Result: rewriter.convertRegionTypes(region: &llvmFuncOp.getBody(), converter: *typeConverter, |
| 344 | entryConversion: &signatureConversion))) |
| 345 | return failure(); |
| 346 | |
| 347 | // Get memref type from function arguments and set the noalias to |
| 348 | // pointer arguments. |
| 349 | for (const auto [idx, argTy] : |
| 350 | llvm::enumerate(First: gpuFuncOp.getArgumentTypes())) { |
| 351 | auto remapping = signatureConversion.getInputMapping(input: idx); |
| 352 | NamedAttrList argAttr = |
| 353 | argAttrs ? cast<DictionaryAttr>(Val: argAttrs[idx]) : NamedAttrList(); |
| 354 | auto copyAttribute = [&](StringRef attrName) { |
| 355 | Attribute attr = argAttr.erase(name: attrName); |
| 356 | if (!attr) |
| 357 | return; |
| 358 | for (size_t i = 0, e = remapping->size; i < e; ++i) |
| 359 | llvmFuncOp.setArgAttr(index: remapping->inputNo + i, name: attrName, value: attr); |
| 360 | }; |
| 361 | auto copyPointerAttribute = [&](StringRef attrName) { |
| 362 | Attribute attr = argAttr.erase(name: attrName); |
| 363 | |
| 364 | if (!attr) |
| 365 | return; |
| 366 | if (remapping->size > 1 && |
| 367 | attrName == LLVM::LLVMDialect::getNoAliasAttrName()) { |
| 368 | emitWarning(loc: llvmFuncOp.getLoc(), |
| 369 | message: "Cannot copy noalias with non-bare pointers.\n" ); |
| 370 | return; |
| 371 | } |
| 372 | for (size_t i = 0, e = remapping->size; i < e; ++i) { |
| 373 | if (isa<LLVM::LLVMPointerType>( |
| 374 | Val: llvmFuncOp.getArgument(idx: remapping->inputNo + i).getType())) { |
| 375 | llvmFuncOp.setArgAttr(index: remapping->inputNo + i, name: attrName, value: attr); |
| 376 | } |
| 377 | } |
| 378 | }; |
| 379 | |
| 380 | if (argAttr.empty()) |
| 381 | continue; |
| 382 | |
| 383 | copyAttribute(LLVM::LLVMDialect::getReturnedAttrName()); |
| 384 | copyAttribute(LLVM::LLVMDialect::getNoUndefAttrName()); |
| 385 | copyAttribute(LLVM::LLVMDialect::getInRegAttrName()); |
| 386 | bool lowersToPointer = false; |
| 387 | for (size_t i = 0, e = remapping->size; i < e; ++i) { |
| 388 | lowersToPointer |= isa<LLVM::LLVMPointerType>( |
| 389 | Val: llvmFuncOp.getArgument(idx: remapping->inputNo + i).getType()); |
| 390 | } |
| 391 | |
| 392 | if (lowersToPointer) { |
| 393 | copyPointerAttribute(LLVM::LLVMDialect::getNoAliasAttrName()); |
| 394 | copyPointerAttribute(LLVM::LLVMDialect::getNoCaptureAttrName()); |
| 395 | copyPointerAttribute(LLVM::LLVMDialect::getNoFreeAttrName()); |
| 396 | copyPointerAttribute(LLVM::LLVMDialect::getAlignAttrName()); |
| 397 | copyPointerAttribute(LLVM::LLVMDialect::getReadonlyAttrName()); |
| 398 | copyPointerAttribute(LLVM::LLVMDialect::getWriteOnlyAttrName()); |
| 399 | copyPointerAttribute(LLVM::LLVMDialect::getReadnoneAttrName()); |
| 400 | copyPointerAttribute(LLVM::LLVMDialect::getNonNullAttrName()); |
| 401 | copyPointerAttribute(LLVM::LLVMDialect::getDereferenceableAttrName()); |
| 402 | copyPointerAttribute( |
| 403 | LLVM::LLVMDialect::getDereferenceableOrNullAttrName()); |
| 404 | copyPointerAttribute( |
| 405 | LLVM::LLVMDialect::WorkgroupAttributionAttrHelper::getNameStr()); |
| 406 | } |
| 407 | } |
| 408 | rewriter.eraseOp(op: gpuFuncOp); |
| 409 | return success(); |
| 410 | } |
| 411 | |
| 412 | LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite( |
| 413 | gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor, |
| 414 | ConversionPatternRewriter &rewriter) const { |
| 415 | Location loc = gpuPrintfOp->getLoc(); |
| 416 | |
| 417 | mlir::Type llvmI8 = typeConverter->convertType(t: rewriter.getI8Type()); |
| 418 | auto ptrType = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 419 | mlir::Type llvmI32 = typeConverter->convertType(t: rewriter.getI32Type()); |
| 420 | mlir::Type llvmI64 = typeConverter->convertType(t: rewriter.getI64Type()); |
| 421 | |
| 422 | Operation *moduleOp = gpuPrintfOp->getParentWithTrait<OpTrait::SymbolTable>(); |
| 423 | if (!moduleOp) |
| 424 | return rewriter.notifyMatchFailure(arg&: gpuPrintfOp, |
| 425 | msg: "Couldn't find a parent module" ); |
| 426 | |
| 427 | auto ocklBegin = |
| 428 | getOrDefineFunction(moduleOp, loc, b&: rewriter, name: "__ockl_printf_begin" , |
| 429 | type: LLVM::LLVMFunctionType::get(result: llvmI64, arguments: {llvmI64})); |
| 430 | LLVM::LLVMFuncOp ocklAppendArgs; |
| 431 | if (!adaptor.getArgs().empty()) { |
| 432 | ocklAppendArgs = getOrDefineFunction( |
| 433 | moduleOp, loc, b&: rewriter, name: "__ockl_printf_append_args" , |
| 434 | type: LLVM::LLVMFunctionType::get( |
| 435 | result: llvmI64, arguments: {llvmI64, /*numArgs*/ llvmI32, llvmI64, llvmI64, llvmI64, |
| 436 | llvmI64, llvmI64, llvmI64, llvmI64, /*isLast*/ llvmI32})); |
| 437 | } |
| 438 | auto ocklAppendStringN = getOrDefineFunction( |
| 439 | moduleOp, loc, b&: rewriter, name: "__ockl_printf_append_string_n" , |
| 440 | type: LLVM::LLVMFunctionType::get( |
| 441 | result: llvmI64, |
| 442 | arguments: {llvmI64, ptrType, /*length (bytes)*/ llvmI64, /*isLast*/ llvmI32})); |
| 443 | |
| 444 | /// Start the printf hostcall |
| 445 | Value zeroI64 = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: llvmI64, value: 0); |
| 446 | auto printfBeginCall = |
| 447 | LLVM::CallOp::create(builder&: rewriter, location: loc, func: ocklBegin, args: zeroI64); |
| 448 | Value printfDesc = printfBeginCall.getResult(); |
| 449 | |
| 450 | // Create the global op or find an existing one. |
| 451 | LLVM::GlobalOp global = getOrCreateStringConstant( |
| 452 | b&: rewriter, loc, moduleOp, llvmI8, namePrefix: "printfFormat_" , str: adaptor.getFormat()); |
| 453 | |
| 454 | // Get a pointer to the format string's first element and pass it to printf() |
| 455 | Value globalPtr = LLVM::AddressOfOp::create( |
| 456 | builder&: rewriter, location: loc, |
| 457 | res: LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: global.getAddrSpace()), |
| 458 | global_name: global.getSymNameAttr()); |
| 459 | Value stringStart = |
| 460 | LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: global.getGlobalType(), |
| 461 | basePtr: globalPtr, indices: ArrayRef<LLVM::GEPArg>{0, 0}); |
| 462 | Value stringLen = LLVM::ConstantOp::create( |
| 463 | builder&: rewriter, location: loc, type: llvmI64, value: cast<StringAttr>(Val: global.getValueAttr()).size()); |
| 464 | |
| 465 | Value oneI32 = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: llvmI32, value: 1); |
| 466 | Value zeroI32 = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: llvmI32, value: 0); |
| 467 | |
| 468 | auto appendFormatCall = LLVM::CallOp::create( |
| 469 | builder&: rewriter, location: loc, func: ocklAppendStringN, |
| 470 | args: ValueRange{printfDesc, stringStart, stringLen, |
| 471 | adaptor.getArgs().empty() ? oneI32 : zeroI32}); |
| 472 | printfDesc = appendFormatCall.getResult(); |
| 473 | |
| 474 | // __ockl_printf_append_args takes 7 values per append call |
| 475 | constexpr size_t argsPerAppend = 7; |
| 476 | size_t nArgs = adaptor.getArgs().size(); |
| 477 | for (size_t group = 0; group < nArgs; group += argsPerAppend) { |
| 478 | size_t bound = std::min(a: group + argsPerAppend, b: nArgs); |
| 479 | size_t numArgsThisCall = bound - group; |
| 480 | |
| 481 | SmallVector<mlir::Value, 2 + argsPerAppend + 1> arguments; |
| 482 | arguments.push_back(Elt: printfDesc); |
| 483 | arguments.push_back( |
| 484 | Elt: LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: llvmI32, value: numArgsThisCall)); |
| 485 | for (size_t i = group; i < bound; ++i) { |
| 486 | Value arg = adaptor.getArgs()[i]; |
| 487 | if (auto floatType = dyn_cast<FloatType>(Val: arg.getType())) { |
| 488 | if (!floatType.isF64()) |
| 489 | arg = LLVM::FPExtOp::create( |
| 490 | builder&: rewriter, location: loc, res: typeConverter->convertType(t: rewriter.getF64Type()), |
| 491 | arg); |
| 492 | arg = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: llvmI64, arg); |
| 493 | } |
| 494 | if (arg.getType().getIntOrFloatBitWidth() != 64) |
| 495 | arg = LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: llvmI64, arg); |
| 496 | |
| 497 | arguments.push_back(Elt: arg); |
| 498 | } |
| 499 | // Pad out to 7 arguments since the hostcall always needs 7 |
| 500 | for (size_t = numArgsThisCall; extra < argsPerAppend; ++extra) { |
| 501 | arguments.push_back(Elt: zeroI64); |
| 502 | } |
| 503 | |
| 504 | auto isLast = (bound == nArgs) ? oneI32 : zeroI32; |
| 505 | arguments.push_back(Elt: isLast); |
| 506 | auto call = LLVM::CallOp::create(builder&: rewriter, location: loc, func: ocklAppendArgs, args: arguments); |
| 507 | printfDesc = call.getResult(); |
| 508 | } |
| 509 | rewriter.eraseOp(op: gpuPrintfOp); |
| 510 | return success(); |
| 511 | } |
| 512 | |
| 513 | LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite( |
| 514 | gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor, |
| 515 | ConversionPatternRewriter &rewriter) const { |
| 516 | Location loc = gpuPrintfOp->getLoc(); |
| 517 | |
| 518 | mlir::Type llvmI8 = typeConverter->convertType(t: rewriter.getIntegerType(width: 8)); |
| 519 | mlir::Type ptrType = |
| 520 | LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace); |
| 521 | |
| 522 | Operation *moduleOp = gpuPrintfOp->getParentWithTrait<OpTrait::SymbolTable>(); |
| 523 | if (!moduleOp) |
| 524 | return rewriter.notifyMatchFailure(arg&: gpuPrintfOp, |
| 525 | msg: "Couldn't find a parent module" ); |
| 526 | |
| 527 | auto printfType = |
| 528 | LLVM::LLVMFunctionType::get(result: rewriter.getI32Type(), arguments: {ptrType}, |
| 529 | /*isVarArg=*/true); |
| 530 | LLVM::LLVMFuncOp printfDecl = |
| 531 | getOrDefineFunction(moduleOp, loc, b&: rewriter, name: funcName, type: printfType); |
| 532 | printfDecl.setCConv(callingConvention); |
| 533 | |
| 534 | // Create the global op or find an existing one. |
| 535 | LLVM::GlobalOp global = getOrCreateStringConstant( |
| 536 | b&: rewriter, loc, moduleOp, llvmI8, namePrefix: "printfFormat_" , str: adaptor.getFormat(), |
| 537 | /*alignment=*/0, addrSpace: addressSpace); |
| 538 | |
| 539 | // Get a pointer to the format string's first element |
| 540 | Value globalPtr = LLVM::AddressOfOp::create( |
| 541 | builder&: rewriter, location: loc, |
| 542 | res: LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: global.getAddrSpace()), |
| 543 | global_name: global.getSymNameAttr()); |
| 544 | Value stringStart = |
| 545 | LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: global.getGlobalType(), |
| 546 | basePtr: globalPtr, indices: ArrayRef<LLVM::GEPArg>{0, 0}); |
| 547 | |
| 548 | // Construct arguments and function call |
| 549 | auto argsRange = adaptor.getArgs(); |
| 550 | SmallVector<Value, 4> printfArgs; |
| 551 | printfArgs.reserve(N: argsRange.size() + 1); |
| 552 | printfArgs.push_back(Elt: stringStart); |
| 553 | printfArgs.append(in_start: argsRange.begin(), in_end: argsRange.end()); |
| 554 | |
| 555 | auto call = LLVM::CallOp::create(builder&: rewriter, location: loc, func: printfDecl, args: printfArgs); |
| 556 | call.setCConv(callingConvention); |
| 557 | rewriter.eraseOp(op: gpuPrintfOp); |
| 558 | return success(); |
| 559 | } |
| 560 | |
| 561 | LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite( |
| 562 | gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor, |
| 563 | ConversionPatternRewriter &rewriter) const { |
| 564 | Location loc = gpuPrintfOp->getLoc(); |
| 565 | |
| 566 | mlir::Type llvmI8 = typeConverter->convertType(t: rewriter.getIntegerType(width: 8)); |
| 567 | mlir::Type ptrType = LLVM::LLVMPointerType::get(context: rewriter.getContext()); |
| 568 | |
| 569 | Operation *moduleOp = gpuPrintfOp->getParentWithTrait<OpTrait::SymbolTable>(); |
| 570 | if (!moduleOp) |
| 571 | return rewriter.notifyMatchFailure(arg&: gpuPrintfOp, |
| 572 | msg: "Couldn't find a parent module" ); |
| 573 | |
| 574 | // Create a valid global location removing any metadata attached to the |
| 575 | // location as debug info metadata inside of a function cannot be used outside |
| 576 | // of that function. |
| 577 | Location globalLoc = loc->findInstanceOfOrUnknown<FileLineColLoc>(); |
| 578 | |
| 579 | auto vprintfType = |
| 580 | LLVM::LLVMFunctionType::get(result: rewriter.getI32Type(), arguments: {ptrType, ptrType}); |
| 581 | LLVM::LLVMFuncOp vprintfDecl = getOrDefineFunction( |
| 582 | moduleOp, loc: globalLoc, b&: rewriter, name: "vprintf" , type: vprintfType); |
| 583 | |
| 584 | // Create the global op or find an existing one. |
| 585 | LLVM::GlobalOp global = |
| 586 | getOrCreateStringConstant(b&: rewriter, loc: globalLoc, moduleOp, llvmI8, |
| 587 | namePrefix: "printfFormat_" , str: adaptor.getFormat()); |
| 588 | |
| 589 | // Get a pointer to the format string's first element |
| 590 | Value globalPtr = LLVM::AddressOfOp::create(builder&: rewriter, location: loc, global); |
| 591 | Value stringStart = |
| 592 | LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: global.getGlobalType(), |
| 593 | basePtr: globalPtr, indices: ArrayRef<LLVM::GEPArg>{0, 0}); |
| 594 | SmallVector<Type> types; |
| 595 | SmallVector<Value> args; |
| 596 | // Promote and pack the arguments into a stack allocation. |
| 597 | for (Value arg : adaptor.getArgs()) { |
| 598 | Type type = arg.getType(); |
| 599 | Value promotedArg = arg; |
| 600 | assert(type.isIntOrFloat()); |
| 601 | if (isa<FloatType>(Val: type)) { |
| 602 | type = rewriter.getF64Type(); |
| 603 | promotedArg = LLVM::FPExtOp::create(builder&: rewriter, location: loc, res: type, arg); |
| 604 | } |
| 605 | types.push_back(Elt: type); |
| 606 | args.push_back(Elt: promotedArg); |
| 607 | } |
| 608 | Type structType = |
| 609 | LLVM::LLVMStructType::getLiteral(context: gpuPrintfOp.getContext(), types); |
| 610 | Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: rewriter.getI64Type(), |
| 611 | value: rewriter.getI64IntegerAttr(value: 1)); |
| 612 | Value tempAlloc = |
| 613 | LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: structType, arraySize: one, |
| 614 | /*alignment=*/0); |
| 615 | for (auto [index, arg] : llvm::enumerate(First&: args)) { |
| 616 | Value ptr = LLVM::GEPOp::create( |
| 617 | builder&: rewriter, location: loc, resultType: ptrType, elementType: structType, basePtr: tempAlloc, |
| 618 | indices: ArrayRef<LLVM::GEPArg>{0, static_cast<int32_t>(index)}); |
| 619 | LLVM::StoreOp::create(builder&: rewriter, location: loc, value: arg, addr: ptr); |
| 620 | } |
| 621 | std::array<Value, 2> printfArgs = {stringStart, tempAlloc}; |
| 622 | |
| 623 | LLVM::CallOp::create(builder&: rewriter, location: loc, func: vprintfDecl, args: printfArgs); |
| 624 | rewriter.eraseOp(op: gpuPrintfOp); |
| 625 | return success(); |
| 626 | } |
| 627 | |
| 628 | /// Helper for impl::scalarizeVectorOp. Scalarizes vectors to elements. |
| 629 | /// Used either directly (for ops on 1D vectors) or as the callback passed to |
| 630 | /// detail::handleMultidimensionalVectors (for ops on higher-rank vectors). |
| 631 | static Value scalarizeVectorOpHelper(Operation *op, ValueRange operands, |
| 632 | Type llvm1DVectorTy, |
| 633 | ConversionPatternRewriter &rewriter, |
| 634 | const LLVMTypeConverter &converter) { |
| 635 | TypeRange operandTypes(operands); |
| 636 | VectorType vectorType = cast<VectorType>(Val&: llvm1DVectorTy); |
| 637 | Location loc = op->getLoc(); |
| 638 | Value result = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: vectorType); |
| 639 | Type indexType = converter.convertType(t: rewriter.getIndexType()); |
| 640 | Type elementType = vectorType.getElementType(); |
| 641 | |
| 642 | for (int64_t i = 0; i < vectorType.getNumElements(); ++i) { |
| 643 | Value index = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: indexType, value: i); |
| 644 | auto = [&](Value operand) -> Value { |
| 645 | if (!isa<VectorType>(Val: operand.getType())) |
| 646 | return operand; |
| 647 | return LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, vector: operand, position: index); |
| 648 | }; |
| 649 | auto scalarOperands = llvm::map_to_vector(C&: operands, F&: extractElement); |
| 650 | OperationState state(loc, op->getName(), scalarOperands, elementType, |
| 651 | op->getDiscardableAttrDictionary().getValue()); |
| 652 | state.propertiesAttr = op->getPropertiesAsAttribute(); |
| 653 | Operation *scalarOp = rewriter.create(state); |
| 654 | result = LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: result, |
| 655 | value: scalarOp->getResult(idx: 0), position: index); |
| 656 | } |
| 657 | return result; |
| 658 | } |
| 659 | |
| 660 | /// Unrolls op to array/vector elements. |
| 661 | LogicalResult impl::scalarizeVectorOp(Operation *op, ValueRange operands, |
| 662 | ConversionPatternRewriter &rewriter, |
| 663 | const LLVMTypeConverter &converter) { |
| 664 | TypeRange operandTypes(operands); |
| 665 | if (llvm::any_of(Range&: operandTypes, P: llvm::IsaPred<VectorType>)) { |
| 666 | VectorType vectorType = |
| 667 | cast<VectorType>(Val: converter.convertType(t: op->getResultTypes()[0])); |
| 668 | rewriter.replaceOp(op, newValues: scalarizeVectorOpHelper(op, operands, llvm1DVectorTy: vectorType, |
| 669 | rewriter, converter)); |
| 670 | return success(); |
| 671 | } |
| 672 | |
| 673 | if (llvm::any_of(Range&: operandTypes, P: llvm::IsaPred<LLVM::LLVMArrayType>)) { |
| 674 | return LLVM::detail::handleMultidimensionalVectors( |
| 675 | op, operands, typeConverter: converter, |
| 676 | createOperand: [&](Type llvm1DVectorTy, ValueRange operands) -> Value { |
| 677 | return scalarizeVectorOpHelper(op, operands, llvm1DVectorTy, rewriter, |
| 678 | converter); |
| 679 | }, |
| 680 | rewriter); |
| 681 | } |
| 682 | |
| 683 | return rewriter.notifyMatchFailure(arg&: op, msg: "no llvm.array or vector to unroll" ); |
| 684 | } |
| 685 | |
| 686 | static IntegerAttr wrapNumericMemorySpace(MLIRContext *ctx, unsigned space) { |
| 687 | return IntegerAttr::get(type: IntegerType::get(context: ctx, width: 64), value: space); |
| 688 | } |
| 689 | |
| 690 | /// Generates a symbol with 0-sized array type for dynamic shared memory usage, |
| 691 | /// or uses existing symbol. |
| 692 | static LLVM::GlobalOp getDynamicSharedMemorySymbol( |
| 693 | ConversionPatternRewriter &rewriter, gpu::GPUModuleOp moduleOp, |
| 694 | gpu::DynamicSharedMemoryOp op, const LLVMTypeConverter *typeConverter, |
| 695 | MemRefType memrefType, unsigned alignmentBit) { |
| 696 | uint64_t alignmentByte = alignmentBit / memrefType.getElementTypeBitWidth(); |
| 697 | |
| 698 | FailureOr<unsigned> addressSpace = |
| 699 | typeConverter->getMemRefAddressSpace(type: memrefType); |
| 700 | if (failed(Result: addressSpace)) { |
| 701 | op->emitError() << "conversion of memref memory space " |
| 702 | << memrefType.getMemorySpace() |
| 703 | << " to integer address space " |
| 704 | "failed. Consider adding memory space conversions." ; |
| 705 | } |
| 706 | |
| 707 | // Step 1. Collect symbol names of LLVM::GlobalOp Ops. Also if any of |
| 708 | // LLVM::GlobalOp is suitable for shared memory, return it. |
| 709 | llvm::StringSet<> existingGlobalNames; |
| 710 | for (auto globalOp : moduleOp.getBody()->getOps<LLVM::GlobalOp>()) { |
| 711 | existingGlobalNames.insert(key: globalOp.getSymName()); |
| 712 | if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(Val: globalOp.getType())) { |
| 713 | if (globalOp.getAddrSpace() == addressSpace.value() && |
| 714 | arrayType.getNumElements() == 0 && |
| 715 | globalOp.getAlignment().value_or(u: 0) == alignmentByte) { |
| 716 | return globalOp; |
| 717 | } |
| 718 | } |
| 719 | } |
| 720 | |
| 721 | // Step 2. Find a unique symbol name |
| 722 | unsigned uniquingCounter = 0; |
| 723 | SmallString<128> symName = SymbolTable::generateSymbolName<128>( |
| 724 | name: "__dynamic_shmem_" , |
| 725 | uniqueChecker: [&](StringRef candidate) { |
| 726 | return existingGlobalNames.contains(key: candidate); |
| 727 | }, |
| 728 | uniquingCounter); |
| 729 | |
| 730 | // Step 3. Generate a global op |
| 731 | OpBuilder::InsertionGuard guard(rewriter); |
| 732 | rewriter.setInsertionPointToStart(moduleOp.getBody()); |
| 733 | |
| 734 | auto zeroSizedArrayType = LLVM::LLVMArrayType::get( |
| 735 | elementType: typeConverter->convertType(t: memrefType.getElementType()), numElements: 0); |
| 736 | |
| 737 | return LLVM::GlobalOp::create(builder&: rewriter, location: op->getLoc(), type: zeroSizedArrayType, |
| 738 | /*isConstant=*/false, linkage: LLVM::Linkage::Internal, |
| 739 | name: symName, /*value=*/Attribute(), alignment: alignmentByte, |
| 740 | addrSpace: addressSpace.value()); |
| 741 | } |
| 742 | |
| 743 | LogicalResult GPUDynamicSharedMemoryOpLowering::matchAndRewrite( |
| 744 | gpu::DynamicSharedMemoryOp op, OpAdaptor adaptor, |
| 745 | ConversionPatternRewriter &rewriter) const { |
| 746 | Location loc = op.getLoc(); |
| 747 | MemRefType memrefType = op.getResultMemref().getType(); |
| 748 | Type elementType = typeConverter->convertType(t: memrefType.getElementType()); |
| 749 | |
| 750 | // Step 1: Generate a memref<0xi8> type |
| 751 | MemRefLayoutAttrInterface layout = {}; |
| 752 | auto memrefType0sz = |
| 753 | MemRefType::get(shape: {0}, elementType, layout, memorySpace: memrefType.getMemorySpace()); |
| 754 | |
| 755 | // Step 2: Generate a global symbol or existing for the dynamic shared |
| 756 | // memory with memref<0xi8> type |
| 757 | auto moduleOp = op->getParentOfType<gpu::GPUModuleOp>(); |
| 758 | LLVM::GlobalOp shmemOp = getDynamicSharedMemorySymbol( |
| 759 | rewriter, moduleOp, op, typeConverter: getTypeConverter(), memrefType: memrefType0sz, alignmentBit); |
| 760 | |
| 761 | // Step 3. Get address of the global symbol |
| 762 | OpBuilder::InsertionGuard guard(rewriter); |
| 763 | rewriter.setInsertionPoint(op); |
| 764 | auto basePtr = LLVM::AddressOfOp::create(builder&: rewriter, location: loc, global: shmemOp); |
| 765 | Type baseType = basePtr->getResultTypes().front(); |
| 766 | |
| 767 | // Step 4. Generate GEP using offsets |
| 768 | SmallVector<LLVM::GEPArg> gepArgs = {0}; |
| 769 | Value shmemPtr = LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: baseType, elementType, |
| 770 | basePtr, indices: gepArgs); |
| 771 | // Step 5. Create a memref descriptor |
| 772 | SmallVector<Value> shape, strides; |
| 773 | Value sizeBytes; |
| 774 | getMemRefDescriptorSizes(loc, memRefType: memrefType0sz, dynamicSizes: {}, rewriter, sizes&: shape, strides, |
| 775 | size&: sizeBytes); |
| 776 | auto memRefDescriptor = this->createMemRefDescriptor( |
| 777 | loc, memRefType: memrefType0sz, allocatedPtr: shmemPtr, alignedPtr: shmemPtr, sizes: shape, strides, rewriter); |
| 778 | |
| 779 | // Step 5. Replace the op with memref descriptor |
| 780 | rewriter.replaceOp(op, newValues: {memRefDescriptor}); |
| 781 | return success(); |
| 782 | } |
| 783 | |
| 784 | LogicalResult GPUReturnOpLowering::matchAndRewrite( |
| 785 | gpu::ReturnOp op, OpAdaptor adaptor, |
| 786 | ConversionPatternRewriter &rewriter) const { |
| 787 | Location loc = op.getLoc(); |
| 788 | unsigned numArguments = op.getNumOperands(); |
| 789 | SmallVector<Value, 4> updatedOperands; |
| 790 | |
| 791 | bool useBarePtrCallConv = getTypeConverter()->getOptions().useBarePtrCallConv; |
| 792 | if (useBarePtrCallConv) { |
| 793 | // For the bare-ptr calling convention, extract the aligned pointer to |
| 794 | // be returned from the memref descriptor. |
| 795 | for (auto it : llvm::zip(t: op->getOperands(), u: adaptor.getOperands())) { |
| 796 | Type oldTy = std::get<0>(t&: it).getType(); |
| 797 | Value newOperand = std::get<1>(t&: it); |
| 798 | if (isa<MemRefType>(Val: oldTy) && getTypeConverter()->canConvertToBarePtr( |
| 799 | type: cast<BaseMemRefType>(Val&: oldTy))) { |
| 800 | MemRefDescriptor memrefDesc(newOperand); |
| 801 | newOperand = memrefDesc.allocatedPtr(builder&: rewriter, loc); |
| 802 | } else if (isa<UnrankedMemRefType>(Val: oldTy)) { |
| 803 | // Unranked memref is not supported in the bare pointer calling |
| 804 | // convention. |
| 805 | return failure(); |
| 806 | } |
| 807 | updatedOperands.push_back(Elt: newOperand); |
| 808 | } |
| 809 | } else { |
| 810 | updatedOperands = llvm::to_vector<4>(Range: adaptor.getOperands()); |
| 811 | (void)copyUnrankedDescriptors(builder&: rewriter, loc, origTypes: op.getOperands().getTypes(), |
| 812 | operands&: updatedOperands, |
| 813 | /*toDynamic=*/true); |
| 814 | } |
| 815 | |
| 816 | // If ReturnOp has 0 or 1 operand, create it and return immediately. |
| 817 | if (numArguments <= 1) { |
| 818 | rewriter.replaceOpWithNewOp<LLVM::ReturnOp>( |
| 819 | op, args: TypeRange(), args&: updatedOperands, |
| 820 | args: op->getDiscardableAttrDictionary().getValue()); |
| 821 | return success(); |
| 822 | } |
| 823 | |
| 824 | // Otherwise, we need to pack the arguments into an LLVM struct type before |
| 825 | // returning. |
| 826 | auto packedType = getTypeConverter()->packFunctionResults( |
| 827 | types: op.getOperandTypes(), useBarePointerCallConv: useBarePtrCallConv); |
| 828 | if (!packedType) { |
| 829 | return rewriter.notifyMatchFailure(arg&: op, msg: "could not convert result types" ); |
| 830 | } |
| 831 | |
| 832 | Value packed = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: packedType); |
| 833 | for (auto [idx, operand] : llvm::enumerate(First&: updatedOperands)) { |
| 834 | packed = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: packed, value: operand, position: idx); |
| 835 | } |
| 836 | rewriter.replaceOpWithNewOp<LLVM::ReturnOp>( |
| 837 | op, args: TypeRange(), args&: packed, args: op->getDiscardableAttrDictionary().getValue()); |
| 838 | return success(); |
| 839 | } |
| 840 | |
| 841 | void mlir::populateGpuMemorySpaceAttributeConversions( |
| 842 | TypeConverter &typeConverter, const MemorySpaceMapping &mapping) { |
| 843 | typeConverter.addTypeAttributeConversion( |
| 844 | callback: [mapping](BaseMemRefType type, gpu::AddressSpaceAttr memorySpaceAttr) { |
| 845 | gpu::AddressSpace memorySpace = memorySpaceAttr.getValue(); |
| 846 | unsigned addressSpace = mapping(memorySpace); |
| 847 | return wrapNumericMemorySpace(ctx: memorySpaceAttr.getContext(), |
| 848 | space: addressSpace); |
| 849 | }); |
| 850 | } |
| 851 | |