1//===- GPUOpsLowering.cpp - GPU FuncOp / ReturnOp lowering ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "GPUOpsLowering.h"
10
11#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
12#include "mlir/Conversion/LLVMCommon/VectorPattern.h"
13#include "mlir/Dialect/LLVMIR/LLVMAggregateOps.h"
14#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
15#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
16#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMVectorOps.h"
21#include "mlir/IR/Attributes.h"
22#include "mlir/IR/Builders.h"
23#include "mlir/IR/BuiltinTypes.h"
24#include "mlir/IR/SymbolTable.h"
25#include "llvm/ADT/SmallVectorExtras.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/Support/DebugLog.h"
28#include "llvm/Support/FormatVariadic.h"
29
30#define DEBUG_TYPE "gpu-lowering"
31
32using namespace mlir;
33
34LLVM::LLVMFuncOp mlir::getOrDefineFunction(Operation *moduleOp, Location loc,
35 OpBuilder &b, StringRef name,
36 LLVM::LLVMFunctionType type) {
37 auto existing = dyn_cast_or_null<LLVM::LLVMFuncOp>(
38 Val: SymbolTable::lookupSymbolIn(op: moduleOp, symbol: name));
39 if (existing)
40 return existing;
41
42 OpBuilder::InsertionGuard guard(b);
43 b.setInsertionPointToStart(&moduleOp->getRegion(index: 0).front());
44 return LLVM::LLVMFuncOp::create(builder&: b, location: loc, name, type, linkage: LLVM::Linkage::External);
45}
46
47static SmallString<16> getUniqueSymbolName(Operation *moduleOp,
48 StringRef prefix) {
49 // Get a unique global name.
50 unsigned stringNumber = 0;
51 SmallString<16> stringConstName;
52 do {
53 stringConstName.clear();
54 (prefix + Twine(stringNumber++)).toStringRef(Out&: stringConstName);
55 } while (SymbolTable::lookupSymbolIn(op: moduleOp, symbol: stringConstName));
56 return stringConstName;
57}
58
59LLVM::GlobalOp mlir::getOrCreateStringConstant(OpBuilder &b, Location loc,
60 Operation *moduleOp, Type llvmI8,
61 StringRef namePrefix,
62 StringRef str,
63 uint64_t alignment,
64 unsigned addrSpace) {
65 llvm::SmallString<20> nullTermStr(str);
66 nullTermStr.push_back(Elt: '\0'); // Null terminate for C
67 auto globalType =
68 LLVM::LLVMArrayType::get(elementType: llvmI8, numElements: nullTermStr.size_in_bytes());
69 StringAttr attr = b.getStringAttr(bytes: nullTermStr);
70
71 // Try to find existing global.
72 for (auto globalOp : moduleOp->getRegion(index: 0).getOps<LLVM::GlobalOp>())
73 if (globalOp.getGlobalType() == globalType && globalOp.getConstant() &&
74 globalOp.getValueAttr() == attr &&
75 globalOp.getAlignment().value_or(u: 0) == alignment &&
76 globalOp.getAddrSpace() == addrSpace)
77 return globalOp;
78
79 // Not found: create new global.
80 OpBuilder::InsertionGuard guard(b);
81 b.setInsertionPointToStart(&moduleOp->getRegion(index: 0).front());
82 SmallString<16> name = getUniqueSymbolName(moduleOp, prefix: namePrefix);
83 return LLVM::GlobalOp::create(builder&: b, location: loc, type: globalType,
84 /*isConstant=*/true, linkage: LLVM::Linkage::Internal,
85 name, value: attr, alignment, addrSpace);
86}
87
88FailureOr<LoweredLLVMFuncAttrs> GPUFuncOpLowering::buildLoweredGPULLVMFuncAttrs(
89 gpu::GPUFuncOp gpuFuncOp, Type llvmFuncType, OpBuilder &rewriter) const {
90 FailureOr<LoweredLLVMFuncAttrs> loweredAttrs =
91 lowerDiscardableAttrsForLLVMFunc(funcOp: gpuFuncOp, llvmFuncType);
92 if (failed(Result: loweredAttrs))
93 return failure();
94
95 MLIRContext *ctx = rewriter.getContext();
96 LLVM::LLVMFuncOp::Properties &props = loweredAttrs->properties;
97 props.sym_name = rewriter.getStringAttr(bytes: gpuFuncOp.getName());
98 props.function_type = TypeAttr::get(type: llvmFuncType);
99 const bool isKernelFunc = gpuFuncOp.isKernel();
100 props.setCConv(LLVM::CConvAttr::get(context: ctx, CallingConv: isKernelFunc
101 ? kernelCallingConvention
102 : nonKernelCallingConvention));
103
104 NamedAttrList &discardable = loweredAttrs->discardableAttrs;
105 auto *gpuDialect = cast<gpu::GPUDialect>(Val: gpuFuncOp->getDialect());
106
107 auto appendIfNameAndValue = [&](StringAttr name, Attribute value) {
108 if (name && value)
109 discardable.append(name, attr: value);
110 };
111
112 DenseI32ArrayAttr knownBlockSize = gpuFuncOp.getKnownBlockSizeAttr();
113 DenseI32ArrayAttr knownGridSize = gpuFuncOp.getKnownGridSizeAttr();
114 DenseI32ArrayAttr knownClusterSize = gpuFuncOp.getKnownClusterSizeAttr();
115
116 appendIfNameAndValue(gpuDialect->getKnownBlockSizeAttrHelper().getName(),
117 knownBlockSize);
118 appendIfNameAndValue(gpuDialect->getKnownGridSizeAttrHelper().getName(),
119 knownGridSize);
120 appendIfNameAndValue(gpuDialect->getKnownClusterSizeAttrHelper().getName(),
121 knownClusterSize);
122
123 if (isKernelFunc) {
124 discardable.append(name: gpuDialect->getKernelFuncAttrName(),
125 attr: rewriter.getUnitAttr());
126 // Add a dialect specific kernel attribute in addition to GPU kernel
127 // attribute. The former is necessary for further translation while the
128 // latter is expected by gpu.launch_func.
129 appendIfNameAndValue(kernelAttributeName, rewriter.getUnitAttr());
130 appendIfNameAndValue(kernelBlockSizeAttributeName, knownBlockSize);
131 appendIfNameAndValue(kernelClusterSizeAttributeName, knownClusterSize);
132 }
133
134 return loweredAttrs;
135}
136
137LogicalResult
138GPUFuncOpLowering::matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, OpAdaptor adaptor,
139 ConversionPatternRewriter &rewriter) const {
140 Location loc = gpuFuncOp.getLoc();
141
142 SmallVector<LLVM::GlobalOp, 3> workgroupBuffers;
143 if (encodeWorkgroupAttributionsAsArguments) {
144 // Append an `llvm.ptr` argument to the function signature to encode
145 // workgroup attributions.
146
147 ArrayRef<BlockArgument> workgroupAttributions =
148 gpuFuncOp.getWorkgroupAttributionBBArgs();
149 size_t numAttributions = workgroupAttributions.size();
150
151 // Insert all arguments at the end.
152 unsigned index = gpuFuncOp.getNumArguments();
153 SmallVector<unsigned> argIndices(numAttributions, index);
154
155 // New arguments will simply be `llvm.ptr` with the correct address space
156 Type workgroupPtrType =
157 rewriter.getType<LLVM::LLVMPointerType>(args: workgroupAddrSpace);
158 Repeated<Type> argTypes(numAttributions, workgroupPtrType);
159
160 // Attributes: noalias, llvm.mlir.workgroup_attribution(<size>, <type>)
161 std::array attrs{
162 rewriter.getNamedAttr(name: LLVM::LLVMDialect::getNoAliasAttrName(),
163 val: rewriter.getUnitAttr()),
164 rewriter.getNamedAttr(
165 name: getDialect().getWorkgroupAttributionAttrHelper().getName(),
166 val: rewriter.getUnitAttr()),
167 };
168 SmallVector<DictionaryAttr> argAttrs;
169 for (BlockArgument attribution : workgroupAttributions) {
170 auto attributionType = cast<MemRefType>(Val: attribution.getType());
171 IntegerAttr numElements =
172 rewriter.getI64IntegerAttr(value: attributionType.getNumElements());
173 Type llvmElementType =
174 getTypeConverter()->convertType(t: attributionType.getElementType());
175 if (!llvmElementType)
176 return failure();
177 TypeAttr type = TypeAttr::get(type: llvmElementType);
178 attrs.back().setValue(
179 rewriter.getAttr<LLVM::WorkgroupAttributionAttr>(args&: numElements, args&: type));
180 argAttrs.push_back(Elt: rewriter.getDictionaryAttr(value: attrs));
181 }
182
183 // Location match function location
184 SmallVector<Location> argLocs(numAttributions, gpuFuncOp.getLoc());
185
186 // Perform signature modification
187 rewriter.modifyOpInPlace(
188 root: gpuFuncOp, callable: [gpuFuncOp, &argIndices, &argTypes, &argAttrs, &argLocs]() {
189 LogicalResult inserted =
190 static_cast<FunctionOpInterface>(gpuFuncOp).insertArguments(
191 argIndices, argTypes, argAttrs, argLocs);
192 (void)inserted;
193 assert(succeeded(inserted) &&
194 "expected GPU funcs to support inserting any argument");
195 });
196 } else {
197 workgroupBuffers.reserve(N: gpuFuncOp.getNumWorkgroupAttributions());
198 for (auto [idx, attribution] :
199 llvm::enumerate(First: gpuFuncOp.getWorkgroupAttributionBBArgs())) {
200 auto type = dyn_cast<MemRefType>(Val: attribution.getType());
201 assert(type && type.hasStaticShape() && "unexpected type in attribution");
202
203 uint64_t numElements = type.getNumElements();
204
205 auto elementType =
206 cast<Type>(Val: typeConverter->convertType(t: type.getElementType()));
207 auto arrayType = LLVM::LLVMArrayType::get(elementType, numElements);
208 std::string name =
209 std::string(llvm::formatv(Fmt: "__wg_{0}_{1}", Vals: gpuFuncOp.getName(), Vals&: idx));
210 uint64_t alignment = 0;
211 if (auto alignAttr = dyn_cast_or_null<IntegerAttr>(
212 Val: gpuFuncOp.getWorkgroupAttributionAttr(
213 index: idx, name: LLVM::LLVMDialect::getAlignAttrName())))
214 alignment = alignAttr.getInt();
215 auto globalOp = LLVM::GlobalOp::create(
216 builder&: rewriter, location: gpuFuncOp.getLoc(), type: arrayType, /*isConstant=*/false,
217 linkage: LLVM::Linkage::Internal, name, /*value=*/Attribute(), alignment,
218 addrSpace: workgroupAddrSpace);
219 workgroupBuffers.push_back(Elt: globalOp);
220 }
221 }
222
223 // Remap proper input types.
224 TypeConverter::SignatureConversion signatureConversion(
225 gpuFuncOp.front().getNumArguments());
226
227 Type funcType = getTypeConverter()->convertFunctionSignature(
228 funcTy: gpuFuncOp.getFunctionType(), /*isVariadic=*/false,
229 useBarePtrCallConv: getTypeConverter()->getOptions().useBarePtrCallConv, result&: signatureConversion);
230 if (!funcType) {
231 return rewriter.notifyMatchFailure(op: gpuFuncOp, reasonCallback: [&](Diagnostic &diag) {
232 diag << "failed to convert function signature type for: "
233 << gpuFuncOp.getFunctionType();
234 });
235 }
236
237 ArrayAttr argAttrs = gpuFuncOp.getArgAttrsAttr();
238
239 FailureOr<LoweredLLVMFuncAttrs> loweredAttrs =
240 buildLoweredGPULLVMFuncAttrs(gpuFuncOp, llvmFuncType: funcType, rewriter);
241 if (failed(Result: loweredAttrs))
242 return rewriter.notifyMatchFailure(arg&: gpuFuncOp,
243 msg: "failed to lower func attributes");
244
245 auto llvmFuncOp = LLVM::LLVMFuncOp::create(builder&: rewriter, location: gpuFuncOp.getLoc(),
246 properties: loweredAttrs->properties,
247 discardableAttributes: loweredAttrs->discardableAttrs);
248
249 {
250 // Insert operations that correspond to converted workgroup and private
251 // memory attributions to the body of the function. This must operate on
252 // the original function, before the body region is inlined in the new
253 // function to maintain the relation between block arguments and the
254 // parent operation that assigns their semantics.
255 OpBuilder::InsertionGuard guard(rewriter);
256
257 // Rewrite workgroup memory attributions to addresses of global buffers.
258 rewriter.setInsertionPointToStart(&gpuFuncOp.front());
259 unsigned numProperArguments = gpuFuncOp.getNumArguments();
260
261 if (encodeWorkgroupAttributionsAsArguments) {
262 // Build a MemRefDescriptor with each of the arguments added above.
263
264 unsigned numAttributions = gpuFuncOp.getNumWorkgroupAttributions();
265 assert(numProperArguments >= numAttributions &&
266 "Expecting attributions to be encoded as arguments already");
267
268 // Arguments encoding workgroup attributions will be in positions
269 // [numProperArguments, numProperArguments+numAttributions)
270 ArrayRef<BlockArgument> attributionArguments =
271 gpuFuncOp.getArguments().slice(N: numProperArguments - numAttributions,
272 M: numAttributions);
273 for (auto [idx, vals] : llvm::enumerate(
274 First: llvm::zip_equal(t: gpuFuncOp.getWorkgroupAttributionBBArgs(),
275 u&: attributionArguments))) {
276 auto [attribution, arg] = vals;
277 auto type = cast<MemRefType>(Val: attribution.getType());
278
279 // Arguments are of llvm.ptr type and attributions are of memref type:
280 // we need to wrap them in memref descriptors.
281 Value descr = MemRefDescriptor::fromStaticShape(
282 builder&: rewriter, loc, typeConverter: *getTypeConverter(), type, memory: arg);
283
284 // And remap the arguments
285 signatureConversion.remapInput(origInputNo: numProperArguments + idx, replacements: descr);
286 }
287 } else {
288 for (const auto [idx, global] : llvm::enumerate(First&: workgroupBuffers)) {
289 auto ptrType = LLVM::LLVMPointerType::get(context: rewriter.getContext(),
290 addressSpace: global.getAddrSpace());
291 Value address = LLVM::AddressOfOp::create(builder&: rewriter, location: loc, res: ptrType,
292 global_name: global.getSymNameAttr());
293 Value memory =
294 LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: global.getType(),
295 basePtr: address, indices: ArrayRef<LLVM::GEPArg>{0, 0});
296
297 // Build a memref descriptor pointing to the buffer to plug with the
298 // existing memref infrastructure. This may use more registers than
299 // otherwise necessary given that memref sizes are fixed, but we can try
300 // and canonicalize that away later.
301 Value attribution = gpuFuncOp.getWorkgroupAttributionBBArgs()[idx];
302 auto type = cast<MemRefType>(Val: attribution.getType());
303 Value descr = MemRefDescriptor::fromStaticShape(
304 builder&: rewriter, loc, typeConverter: *getTypeConverter(), type, memory);
305 signatureConversion.remapInput(origInputNo: numProperArguments + idx, replacements: descr);
306 }
307 }
308
309 // Rewrite private memory attributions to alloca'ed buffers.
310 unsigned numWorkgroupAttributions = gpuFuncOp.getNumWorkgroupAttributions();
311 auto int64Ty = IntegerType::get(context: rewriter.getContext(), width: 64);
312 for (const auto [idx, attribution] :
313 llvm::enumerate(First: gpuFuncOp.getPrivateAttributions())) {
314 auto type = cast<MemRefType>(Val: attribution.getType());
315 assert(type && type.hasStaticShape() && "unexpected type in attribution");
316
317 // Explicitly drop memory space when lowering private memory
318 // attributions since NVVM models it as `alloca`s in the default
319 // memory space and does not support `alloca`s with addrspace(5).
320 Type elementType = typeConverter->convertType(t: type.getElementType());
321 auto ptrType =
322 LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: allocaAddrSpace);
323 Value numElements = LLVM::ConstantOp::create(
324 builder&: rewriter, location: gpuFuncOp.getLoc(), type: int64Ty, value: type.getNumElements());
325 uint64_t alignment = 0;
326 if (auto alignAttr =
327 dyn_cast_or_null<IntegerAttr>(Val: gpuFuncOp.getPrivateAttributionAttr(
328 index: idx, name: LLVM::LLVMDialect::getAlignAttrName())))
329 alignment = alignAttr.getInt();
330 Value allocated =
331 LLVM::AllocaOp::create(builder&: rewriter, location: gpuFuncOp.getLoc(), resultType: ptrType,
332 elementType, arraySize: numElements, alignment);
333 Value descr = MemRefDescriptor::fromStaticShape(
334 builder&: rewriter, loc, typeConverter: *getTypeConverter(), type, memory: allocated);
335 signatureConversion.remapInput(
336 origInputNo: numProperArguments + numWorkgroupAttributions + idx, replacements: descr);
337 }
338 }
339
340 // Move the region to the new function, update the entry block signature.
341 rewriter.inlineRegionBefore(region&: gpuFuncOp.getBody(), parent&: llvmFuncOp.getBody(),
342 before: llvmFuncOp.end());
343 if (failed(Result: rewriter.convertRegionTypes(region: &llvmFuncOp.getBody(), converter: *typeConverter,
344 entryConversion: &signatureConversion)))
345 return failure();
346
347 // Get memref type from function arguments and set the noalias to
348 // pointer arguments.
349 for (const auto [idx, argTy] :
350 llvm::enumerate(First: gpuFuncOp.getArgumentTypes())) {
351 auto remapping = signatureConversion.getInputMapping(input: idx);
352 NamedAttrList argAttr =
353 argAttrs ? cast<DictionaryAttr>(Val: argAttrs[idx]) : NamedAttrList();
354 auto copyAttribute = [&](StringRef attrName) {
355 Attribute attr = argAttr.erase(name: attrName);
356 if (!attr)
357 return;
358 for (size_t i = 0, e = remapping->size; i < e; ++i)
359 llvmFuncOp.setArgAttr(index: remapping->inputNo + i, name: attrName, value: attr);
360 };
361 auto copyPointerAttribute = [&](StringRef attrName) {
362 Attribute attr = argAttr.erase(name: attrName);
363
364 if (!attr)
365 return;
366 if (remapping->size > 1 &&
367 attrName == LLVM::LLVMDialect::getNoAliasAttrName()) {
368 emitWarning(loc: llvmFuncOp.getLoc(),
369 message: "Cannot copy noalias with non-bare pointers.\n");
370 return;
371 }
372 for (size_t i = 0, e = remapping->size; i < e; ++i) {
373 if (isa<LLVM::LLVMPointerType>(
374 Val: llvmFuncOp.getArgument(idx: remapping->inputNo + i).getType())) {
375 llvmFuncOp.setArgAttr(index: remapping->inputNo + i, name: attrName, value: attr);
376 }
377 }
378 };
379
380 if (argAttr.empty())
381 continue;
382
383 copyAttribute(LLVM::LLVMDialect::getReturnedAttrName());
384 copyAttribute(LLVM::LLVMDialect::getNoUndefAttrName());
385 copyAttribute(LLVM::LLVMDialect::getInRegAttrName());
386 bool lowersToPointer = false;
387 for (size_t i = 0, e = remapping->size; i < e; ++i) {
388 lowersToPointer |= isa<LLVM::LLVMPointerType>(
389 Val: llvmFuncOp.getArgument(idx: remapping->inputNo + i).getType());
390 }
391
392 if (lowersToPointer) {
393 copyPointerAttribute(LLVM::LLVMDialect::getNoAliasAttrName());
394 copyPointerAttribute(LLVM::LLVMDialect::getNoCaptureAttrName());
395 copyPointerAttribute(LLVM::LLVMDialect::getNoFreeAttrName());
396 copyPointerAttribute(LLVM::LLVMDialect::getAlignAttrName());
397 copyPointerAttribute(LLVM::LLVMDialect::getReadonlyAttrName());
398 copyPointerAttribute(LLVM::LLVMDialect::getWriteOnlyAttrName());
399 copyPointerAttribute(LLVM::LLVMDialect::getReadnoneAttrName());
400 copyPointerAttribute(LLVM::LLVMDialect::getNonNullAttrName());
401 copyPointerAttribute(LLVM::LLVMDialect::getDereferenceableAttrName());
402 copyPointerAttribute(
403 LLVM::LLVMDialect::getDereferenceableOrNullAttrName());
404 copyPointerAttribute(
405 LLVM::LLVMDialect::WorkgroupAttributionAttrHelper::getNameStr());
406 }
407 }
408 rewriter.eraseOp(op: gpuFuncOp);
409 return success();
410}
411
412LogicalResult GPUPrintfOpToHIPLowering::matchAndRewrite(
413 gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor,
414 ConversionPatternRewriter &rewriter) const {
415 Location loc = gpuPrintfOp->getLoc();
416
417 mlir::Type llvmI8 = typeConverter->convertType(t: rewriter.getI8Type());
418 auto ptrType = LLVM::LLVMPointerType::get(context: rewriter.getContext());
419 mlir::Type llvmI32 = typeConverter->convertType(t: rewriter.getI32Type());
420 mlir::Type llvmI64 = typeConverter->convertType(t: rewriter.getI64Type());
421
422 Operation *moduleOp = gpuPrintfOp->getParentWithTrait<OpTrait::SymbolTable>();
423 if (!moduleOp)
424 return rewriter.notifyMatchFailure(arg&: gpuPrintfOp,
425 msg: "Couldn't find a parent module");
426
427 auto ocklBegin =
428 getOrDefineFunction(moduleOp, loc, b&: rewriter, name: "__ockl_printf_begin",
429 type: LLVM::LLVMFunctionType::get(result: llvmI64, arguments: {llvmI64}));
430 LLVM::LLVMFuncOp ocklAppendArgs;
431 if (!adaptor.getArgs().empty()) {
432 ocklAppendArgs = getOrDefineFunction(
433 moduleOp, loc, b&: rewriter, name: "__ockl_printf_append_args",
434 type: LLVM::LLVMFunctionType::get(
435 result: llvmI64, arguments: {llvmI64, /*numArgs*/ llvmI32, llvmI64, llvmI64, llvmI64,
436 llvmI64, llvmI64, llvmI64, llvmI64, /*isLast*/ llvmI32}));
437 }
438 auto ocklAppendStringN = getOrDefineFunction(
439 moduleOp, loc, b&: rewriter, name: "__ockl_printf_append_string_n",
440 type: LLVM::LLVMFunctionType::get(
441 result: llvmI64,
442 arguments: {llvmI64, ptrType, /*length (bytes)*/ llvmI64, /*isLast*/ llvmI32}));
443
444 /// Start the printf hostcall
445 Value zeroI64 = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: llvmI64, value: 0);
446 auto printfBeginCall =
447 LLVM::CallOp::create(builder&: rewriter, location: loc, func: ocklBegin, args: zeroI64);
448 Value printfDesc = printfBeginCall.getResult();
449
450 // Create the global op or find an existing one.
451 LLVM::GlobalOp global = getOrCreateStringConstant(
452 b&: rewriter, loc, moduleOp, llvmI8, namePrefix: "printfFormat_", str: adaptor.getFormat());
453
454 // Get a pointer to the format string's first element and pass it to printf()
455 Value globalPtr = LLVM::AddressOfOp::create(
456 builder&: rewriter, location: loc,
457 res: LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: global.getAddrSpace()),
458 global_name: global.getSymNameAttr());
459 Value stringStart =
460 LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: global.getGlobalType(),
461 basePtr: globalPtr, indices: ArrayRef<LLVM::GEPArg>{0, 0});
462 Value stringLen = LLVM::ConstantOp::create(
463 builder&: rewriter, location: loc, type: llvmI64, value: cast<StringAttr>(Val: global.getValueAttr()).size());
464
465 Value oneI32 = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: llvmI32, value: 1);
466 Value zeroI32 = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: llvmI32, value: 0);
467
468 auto appendFormatCall = LLVM::CallOp::create(
469 builder&: rewriter, location: loc, func: ocklAppendStringN,
470 args: ValueRange{printfDesc, stringStart, stringLen,
471 adaptor.getArgs().empty() ? oneI32 : zeroI32});
472 printfDesc = appendFormatCall.getResult();
473
474 // __ockl_printf_append_args takes 7 values per append call
475 constexpr size_t argsPerAppend = 7;
476 size_t nArgs = adaptor.getArgs().size();
477 for (size_t group = 0; group < nArgs; group += argsPerAppend) {
478 size_t bound = std::min(a: group + argsPerAppend, b: nArgs);
479 size_t numArgsThisCall = bound - group;
480
481 SmallVector<mlir::Value, 2 + argsPerAppend + 1> arguments;
482 arguments.push_back(Elt: printfDesc);
483 arguments.push_back(
484 Elt: LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: llvmI32, value: numArgsThisCall));
485 for (size_t i = group; i < bound; ++i) {
486 Value arg = adaptor.getArgs()[i];
487 if (auto floatType = dyn_cast<FloatType>(Val: arg.getType())) {
488 if (!floatType.isF64())
489 arg = LLVM::FPExtOp::create(
490 builder&: rewriter, location: loc, res: typeConverter->convertType(t: rewriter.getF64Type()),
491 arg);
492 arg = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: llvmI64, arg);
493 }
494 if (arg.getType().getIntOrFloatBitWidth() != 64)
495 arg = LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: llvmI64, arg);
496
497 arguments.push_back(Elt: arg);
498 }
499 // Pad out to 7 arguments since the hostcall always needs 7
500 for (size_t extra = numArgsThisCall; extra < argsPerAppend; ++extra) {
501 arguments.push_back(Elt: zeroI64);
502 }
503
504 auto isLast = (bound == nArgs) ? oneI32 : zeroI32;
505 arguments.push_back(Elt: isLast);
506 auto call = LLVM::CallOp::create(builder&: rewriter, location: loc, func: ocklAppendArgs, args: arguments);
507 printfDesc = call.getResult();
508 }
509 rewriter.eraseOp(op: gpuPrintfOp);
510 return success();
511}
512
513LogicalResult GPUPrintfOpToLLVMCallLowering::matchAndRewrite(
514 gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor,
515 ConversionPatternRewriter &rewriter) const {
516 Location loc = gpuPrintfOp->getLoc();
517
518 mlir::Type llvmI8 = typeConverter->convertType(t: rewriter.getIntegerType(width: 8));
519 mlir::Type ptrType =
520 LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace);
521
522 Operation *moduleOp = gpuPrintfOp->getParentWithTrait<OpTrait::SymbolTable>();
523 if (!moduleOp)
524 return rewriter.notifyMatchFailure(arg&: gpuPrintfOp,
525 msg: "Couldn't find a parent module");
526
527 auto printfType =
528 LLVM::LLVMFunctionType::get(result: rewriter.getI32Type(), arguments: {ptrType},
529 /*isVarArg=*/true);
530 LLVM::LLVMFuncOp printfDecl =
531 getOrDefineFunction(moduleOp, loc, b&: rewriter, name: funcName, type: printfType);
532 printfDecl.setCConv(callingConvention);
533
534 // Create the global op or find an existing one.
535 LLVM::GlobalOp global = getOrCreateStringConstant(
536 b&: rewriter, loc, moduleOp, llvmI8, namePrefix: "printfFormat_", str: adaptor.getFormat(),
537 /*alignment=*/0, addrSpace: addressSpace);
538
539 // Get a pointer to the format string's first element
540 Value globalPtr = LLVM::AddressOfOp::create(
541 builder&: rewriter, location: loc,
542 res: LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: global.getAddrSpace()),
543 global_name: global.getSymNameAttr());
544 Value stringStart =
545 LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: global.getGlobalType(),
546 basePtr: globalPtr, indices: ArrayRef<LLVM::GEPArg>{0, 0});
547
548 // Construct arguments and function call
549 auto argsRange = adaptor.getArgs();
550 SmallVector<Value, 4> printfArgs;
551 printfArgs.reserve(N: argsRange.size() + 1);
552 printfArgs.push_back(Elt: stringStart);
553 printfArgs.append(in_start: argsRange.begin(), in_end: argsRange.end());
554
555 auto call = LLVM::CallOp::create(builder&: rewriter, location: loc, func: printfDecl, args: printfArgs);
556 call.setCConv(callingConvention);
557 rewriter.eraseOp(op: gpuPrintfOp);
558 return success();
559}
560
561LogicalResult GPUPrintfOpToVPrintfLowering::matchAndRewrite(
562 gpu::PrintfOp gpuPrintfOp, gpu::PrintfOpAdaptor adaptor,
563 ConversionPatternRewriter &rewriter) const {
564 Location loc = gpuPrintfOp->getLoc();
565
566 mlir::Type llvmI8 = typeConverter->convertType(t: rewriter.getIntegerType(width: 8));
567 mlir::Type ptrType = LLVM::LLVMPointerType::get(context: rewriter.getContext());
568
569 Operation *moduleOp = gpuPrintfOp->getParentWithTrait<OpTrait::SymbolTable>();
570 if (!moduleOp)
571 return rewriter.notifyMatchFailure(arg&: gpuPrintfOp,
572 msg: "Couldn't find a parent module");
573
574 // Create a valid global location removing any metadata attached to the
575 // location as debug info metadata inside of a function cannot be used outside
576 // of that function.
577 Location globalLoc = loc->findInstanceOfOrUnknown<FileLineColLoc>();
578
579 auto vprintfType =
580 LLVM::LLVMFunctionType::get(result: rewriter.getI32Type(), arguments: {ptrType, ptrType});
581 LLVM::LLVMFuncOp vprintfDecl = getOrDefineFunction(
582 moduleOp, loc: globalLoc, b&: rewriter, name: "vprintf", type: vprintfType);
583
584 // Create the global op or find an existing one.
585 LLVM::GlobalOp global =
586 getOrCreateStringConstant(b&: rewriter, loc: globalLoc, moduleOp, llvmI8,
587 namePrefix: "printfFormat_", str: adaptor.getFormat());
588
589 // Get a pointer to the format string's first element
590 Value globalPtr = LLVM::AddressOfOp::create(builder&: rewriter, location: loc, global);
591 Value stringStart =
592 LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: global.getGlobalType(),
593 basePtr: globalPtr, indices: ArrayRef<LLVM::GEPArg>{0, 0});
594 SmallVector<Type> types;
595 SmallVector<Value> args;
596 // Promote and pack the arguments into a stack allocation.
597 for (Value arg : adaptor.getArgs()) {
598 Type type = arg.getType();
599 Value promotedArg = arg;
600 assert(type.isIntOrFloat());
601 if (isa<FloatType>(Val: type)) {
602 type = rewriter.getF64Type();
603 promotedArg = LLVM::FPExtOp::create(builder&: rewriter, location: loc, res: type, arg);
604 }
605 types.push_back(Elt: type);
606 args.push_back(Elt: promotedArg);
607 }
608 Type structType =
609 LLVM::LLVMStructType::getLiteral(context: gpuPrintfOp.getContext(), types);
610 Value one = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: rewriter.getI64Type(),
611 value: rewriter.getI64IntegerAttr(value: 1));
612 Value tempAlloc =
613 LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: ptrType, elementType: structType, arraySize: one,
614 /*alignment=*/0);
615 for (auto [index, arg] : llvm::enumerate(First&: args)) {
616 Value ptr = LLVM::GEPOp::create(
617 builder&: rewriter, location: loc, resultType: ptrType, elementType: structType, basePtr: tempAlloc,
618 indices: ArrayRef<LLVM::GEPArg>{0, static_cast<int32_t>(index)});
619 LLVM::StoreOp::create(builder&: rewriter, location: loc, value: arg, addr: ptr);
620 }
621 std::array<Value, 2> printfArgs = {stringStart, tempAlloc};
622
623 LLVM::CallOp::create(builder&: rewriter, location: loc, func: vprintfDecl, args: printfArgs);
624 rewriter.eraseOp(op: gpuPrintfOp);
625 return success();
626}
627
628/// Helper for impl::scalarizeVectorOp. Scalarizes vectors to elements.
629/// Used either directly (for ops on 1D vectors) or as the callback passed to
630/// detail::handleMultidimensionalVectors (for ops on higher-rank vectors).
631static Value scalarizeVectorOpHelper(Operation *op, ValueRange operands,
632 Type llvm1DVectorTy,
633 ConversionPatternRewriter &rewriter,
634 const LLVMTypeConverter &converter) {
635 TypeRange operandTypes(operands);
636 VectorType vectorType = cast<VectorType>(Val&: llvm1DVectorTy);
637 Location loc = op->getLoc();
638 Value result = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: vectorType);
639 Type indexType = converter.convertType(t: rewriter.getIndexType());
640 Type elementType = vectorType.getElementType();
641
642 for (int64_t i = 0; i < vectorType.getNumElements(); ++i) {
643 Value index = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: indexType, value: i);
644 auto extractElement = [&](Value operand) -> Value {
645 if (!isa<VectorType>(Val: operand.getType()))
646 return operand;
647 return LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, vector: operand, position: index);
648 };
649 auto scalarOperands = llvm::map_to_vector(C&: operands, F&: extractElement);
650 OperationState state(loc, op->getName(), scalarOperands, elementType,
651 op->getDiscardableAttrDictionary().getValue());
652 state.propertiesAttr = op->getPropertiesAsAttribute();
653 Operation *scalarOp = rewriter.create(state);
654 result = LLVM::InsertElementOp::create(builder&: rewriter, location: loc, vector: result,
655 value: scalarOp->getResult(idx: 0), position: index);
656 }
657 return result;
658}
659
660/// Unrolls op to array/vector elements.
661LogicalResult impl::scalarizeVectorOp(Operation *op, ValueRange operands,
662 ConversionPatternRewriter &rewriter,
663 const LLVMTypeConverter &converter) {
664 TypeRange operandTypes(operands);
665 if (llvm::any_of(Range&: operandTypes, P: llvm::IsaPred<VectorType>)) {
666 VectorType vectorType =
667 cast<VectorType>(Val: converter.convertType(t: op->getResultTypes()[0]));
668 rewriter.replaceOp(op, newValues: scalarizeVectorOpHelper(op, operands, llvm1DVectorTy: vectorType,
669 rewriter, converter));
670 return success();
671 }
672
673 if (llvm::any_of(Range&: operandTypes, P: llvm::IsaPred<LLVM::LLVMArrayType>)) {
674 return LLVM::detail::handleMultidimensionalVectors(
675 op, operands, typeConverter: converter,
676 createOperand: [&](Type llvm1DVectorTy, ValueRange operands) -> Value {
677 return scalarizeVectorOpHelper(op, operands, llvm1DVectorTy, rewriter,
678 converter);
679 },
680 rewriter);
681 }
682
683 return rewriter.notifyMatchFailure(arg&: op, msg: "no llvm.array or vector to unroll");
684}
685
686static IntegerAttr wrapNumericMemorySpace(MLIRContext *ctx, unsigned space) {
687 return IntegerAttr::get(type: IntegerType::get(context: ctx, width: 64), value: space);
688}
689
690/// Generates a symbol with 0-sized array type for dynamic shared memory usage,
691/// or uses existing symbol.
692static LLVM::GlobalOp getDynamicSharedMemorySymbol(
693 ConversionPatternRewriter &rewriter, gpu::GPUModuleOp moduleOp,
694 gpu::DynamicSharedMemoryOp op, const LLVMTypeConverter *typeConverter,
695 MemRefType memrefType, unsigned alignmentBit) {
696 uint64_t alignmentByte = alignmentBit / memrefType.getElementTypeBitWidth();
697
698 FailureOr<unsigned> addressSpace =
699 typeConverter->getMemRefAddressSpace(type: memrefType);
700 if (failed(Result: addressSpace)) {
701 op->emitError() << "conversion of memref memory space "
702 << memrefType.getMemorySpace()
703 << " to integer address space "
704 "failed. Consider adding memory space conversions.";
705 }
706
707 // Step 1. Collect symbol names of LLVM::GlobalOp Ops. Also if any of
708 // LLVM::GlobalOp is suitable for shared memory, return it.
709 llvm::StringSet<> existingGlobalNames;
710 for (auto globalOp : moduleOp.getBody()->getOps<LLVM::GlobalOp>()) {
711 existingGlobalNames.insert(key: globalOp.getSymName());
712 if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(Val: globalOp.getType())) {
713 if (globalOp.getAddrSpace() == addressSpace.value() &&
714 arrayType.getNumElements() == 0 &&
715 globalOp.getAlignment().value_or(u: 0) == alignmentByte) {
716 return globalOp;
717 }
718 }
719 }
720
721 // Step 2. Find a unique symbol name
722 unsigned uniquingCounter = 0;
723 SmallString<128> symName = SymbolTable::generateSymbolName<128>(
724 name: "__dynamic_shmem_",
725 uniqueChecker: [&](StringRef candidate) {
726 return existingGlobalNames.contains(key: candidate);
727 },
728 uniquingCounter);
729
730 // Step 3. Generate a global op
731 OpBuilder::InsertionGuard guard(rewriter);
732 rewriter.setInsertionPointToStart(moduleOp.getBody());
733
734 auto zeroSizedArrayType = LLVM::LLVMArrayType::get(
735 elementType: typeConverter->convertType(t: memrefType.getElementType()), numElements: 0);
736
737 return LLVM::GlobalOp::create(builder&: rewriter, location: op->getLoc(), type: zeroSizedArrayType,
738 /*isConstant=*/false, linkage: LLVM::Linkage::Internal,
739 name: symName, /*value=*/Attribute(), alignment: alignmentByte,
740 addrSpace: addressSpace.value());
741}
742
743LogicalResult GPUDynamicSharedMemoryOpLowering::matchAndRewrite(
744 gpu::DynamicSharedMemoryOp op, OpAdaptor adaptor,
745 ConversionPatternRewriter &rewriter) const {
746 Location loc = op.getLoc();
747 MemRefType memrefType = op.getResultMemref().getType();
748 Type elementType = typeConverter->convertType(t: memrefType.getElementType());
749
750 // Step 1: Generate a memref<0xi8> type
751 MemRefLayoutAttrInterface layout = {};
752 auto memrefType0sz =
753 MemRefType::get(shape: {0}, elementType, layout, memorySpace: memrefType.getMemorySpace());
754
755 // Step 2: Generate a global symbol or existing for the dynamic shared
756 // memory with memref<0xi8> type
757 auto moduleOp = op->getParentOfType<gpu::GPUModuleOp>();
758 LLVM::GlobalOp shmemOp = getDynamicSharedMemorySymbol(
759 rewriter, moduleOp, op, typeConverter: getTypeConverter(), memrefType: memrefType0sz, alignmentBit);
760
761 // Step 3. Get address of the global symbol
762 OpBuilder::InsertionGuard guard(rewriter);
763 rewriter.setInsertionPoint(op);
764 auto basePtr = LLVM::AddressOfOp::create(builder&: rewriter, location: loc, global: shmemOp);
765 Type baseType = basePtr->getResultTypes().front();
766
767 // Step 4. Generate GEP using offsets
768 SmallVector<LLVM::GEPArg> gepArgs = {0};
769 Value shmemPtr = LLVM::GEPOp::create(builder&: rewriter, location: loc, resultType: baseType, elementType,
770 basePtr, indices: gepArgs);
771 // Step 5. Create a memref descriptor
772 SmallVector<Value> shape, strides;
773 Value sizeBytes;
774 getMemRefDescriptorSizes(loc, memRefType: memrefType0sz, dynamicSizes: {}, rewriter, sizes&: shape, strides,
775 size&: sizeBytes);
776 auto memRefDescriptor = this->createMemRefDescriptor(
777 loc, memRefType: memrefType0sz, allocatedPtr: shmemPtr, alignedPtr: shmemPtr, sizes: shape, strides, rewriter);
778
779 // Step 5. Replace the op with memref descriptor
780 rewriter.replaceOp(op, newValues: {memRefDescriptor});
781 return success();
782}
783
784LogicalResult GPUReturnOpLowering::matchAndRewrite(
785 gpu::ReturnOp op, OpAdaptor adaptor,
786 ConversionPatternRewriter &rewriter) const {
787 Location loc = op.getLoc();
788 unsigned numArguments = op.getNumOperands();
789 SmallVector<Value, 4> updatedOperands;
790
791 bool useBarePtrCallConv = getTypeConverter()->getOptions().useBarePtrCallConv;
792 if (useBarePtrCallConv) {
793 // For the bare-ptr calling convention, extract the aligned pointer to
794 // be returned from the memref descriptor.
795 for (auto it : llvm::zip(t: op->getOperands(), u: adaptor.getOperands())) {
796 Type oldTy = std::get<0>(t&: it).getType();
797 Value newOperand = std::get<1>(t&: it);
798 if (isa<MemRefType>(Val: oldTy) && getTypeConverter()->canConvertToBarePtr(
799 type: cast<BaseMemRefType>(Val&: oldTy))) {
800 MemRefDescriptor memrefDesc(newOperand);
801 newOperand = memrefDesc.allocatedPtr(builder&: rewriter, loc);
802 } else if (isa<UnrankedMemRefType>(Val: oldTy)) {
803 // Unranked memref is not supported in the bare pointer calling
804 // convention.
805 return failure();
806 }
807 updatedOperands.push_back(Elt: newOperand);
808 }
809 } else {
810 updatedOperands = llvm::to_vector<4>(Range: adaptor.getOperands());
811 (void)copyUnrankedDescriptors(builder&: rewriter, loc, origTypes: op.getOperands().getTypes(),
812 operands&: updatedOperands,
813 /*toDynamic=*/true);
814 }
815
816 // If ReturnOp has 0 or 1 operand, create it and return immediately.
817 if (numArguments <= 1) {
818 rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(
819 op, args: TypeRange(), args&: updatedOperands,
820 args: op->getDiscardableAttrDictionary().getValue());
821 return success();
822 }
823
824 // Otherwise, we need to pack the arguments into an LLVM struct type before
825 // returning.
826 auto packedType = getTypeConverter()->packFunctionResults(
827 types: op.getOperandTypes(), useBarePointerCallConv: useBarePtrCallConv);
828 if (!packedType) {
829 return rewriter.notifyMatchFailure(arg&: op, msg: "could not convert result types");
830 }
831
832 Value packed = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: packedType);
833 for (auto [idx, operand] : llvm::enumerate(First&: updatedOperands)) {
834 packed = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: packed, value: operand, position: idx);
835 }
836 rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(
837 op, args: TypeRange(), args&: packed, args: op->getDiscardableAttrDictionary().getValue());
838 return success();
839}
840
841void mlir::populateGpuMemorySpaceAttributeConversions(
842 TypeConverter &typeConverter, const MemorySpaceMapping &mapping) {
843 typeConverter.addTypeAttributeConversion(
844 callback: [mapping](BaseMemRefType type, gpu::AddressSpaceAttr memorySpaceAttr) {
845 gpu::AddressSpace memorySpace = memorySpaceAttr.getValue();
846 unsigned addressSpace = mapping(memorySpace);
847 return wrapNumericMemorySpace(ctx: memorySpaceAttr.getContext(),
848 space: addressSpace);
849 });
850}
851