1//===- LowerToLLVMOffloadPrintfAMDGPU.cpp - AMDGPU printf lowering --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file lowers cir.offload.printf to the hostcall and buffered printf
10// runtime interfaces of AMDGPU.
11//
12//===----------------------------------------------------------------------===//
13
14#include "LowerToLLVM.h"
15#include "LowerToLLVMOffloadPrintf.h"
16
17#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
19#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
20#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
21#include "mlir/Dialect/LLVMIR/LLVMMemoryIntrinsicOps.h"
22#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
23#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
24#include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h"
25#include "mlir/IR/BuiltinOps.h"
26#include "mlir/IR/Matchers.h"
27#include "clang/CIR/Dialect/IR/CIRDialect.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/SparseBitVector.h"
30#include "llvm/Support/MathExtras.h"
31#include "llvm/TargetParser/Triple.h"
32#include "llvm/Transforms/Utils/AMDGPUEmitPrintf.h"
33
34namespace cir {
35namespace direct {
36
37namespace {
38
39// One entry per operand, format first. A string entry means the
40// operand is a constant string; a unit entry means it is not. The
41// attribute name lives in CIRDialect.
42
43// Fold a constant through widening integral casts, each at its own width.
44// TODO: use the regular fold helpers once they fold widening cast chains
45// without also folding narrowing casts. CastOp is not ConstantLike and
46// its fold folds narrowing casts, so the chain stays hand written today.
47std::optional<llvm::APInt>
48foldConstantInt(mlir::Value value,
49 llvm::SmallPtrSetImpl<mlir::Operation *> &seen) {
50 mlir::Operation *def = value.getDefiningOp();
51 if (!def || !seen.insert(Ptr: def).second)
52 return std::nullopt;
53 cir::IntAttr intAttr;
54 if (mlir::matchPattern(value, pattern: mlir::m_Constant(bind_value: &intAttr)))
55 return intAttr.getValue();
56 auto cast = mlir::dyn_cast<cir::CastOp>(Val: def);
57 if (!cast || cast.getKind() != cir::CastKind::integral)
58 return std::nullopt;
59 auto srcTy = mlir::dyn_cast<cir::IntType>(Val: cast.getSrc().getType());
60 auto dstTy = mlir::dyn_cast<cir::IntType>(Val: cast.getType());
61 if (!srcTy || !dstTy || dstTy.getWidth() < srcTy.getWidth() ||
62 dstTy.getWidth() > 64)
63 return std::nullopt;
64 // One widening step from a direct constant is the op's own fold;
65 // chains recurse, since fold sees only the immediate source.
66 llvm::SmallVector<mlir::OpFoldResult, 1> folded;
67 if (succeeded(Result: cast->fold(results&: folded)) && folded.size() == 1)
68 if (auto attr = mlir::dyn_cast<mlir::Attribute>(Val&: folded[0]))
69 if (auto foldedInt = mlir::dyn_cast<cir::IntAttr>(Val&: attr))
70 return foldedInt.getValue();
71 std::optional<llvm::APInt> foldedSrc = foldConstantInt(value: cast.getSrc(), seen);
72 if (!foldedSrc)
73 return std::nullopt;
74 return srcTy.isUnsigned() ? foldedSrc->zextOrTrunc(width: dstTy.getWidth())
75 : foldedSrc->sextOrTrunc(width: dstTy.getWidth());
76}
77
78std::optional<int64_t> getConstantInt(mlir::Value value) {
79 bool isUnsigned = false;
80 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val: value.getType()))
81 isUnsigned = !intTy.isSigned();
82 llvm::SmallPtrSet<mlir::Operation *, 4> seen;
83 std::optional<llvm::APInt> result = foldConstantInt(value, seen);
84 if (!result || result->getBitWidth() > 64)
85 return std::nullopt;
86 // An unsigned value past INT64_MAX cannot be a byte offset here:
87 // returning it would wrap negative in the signed offset below.
88 if (isUnsigned ? result->getActiveBits() > 63
89 : result->getSignificantBits() > 64)
90 return std::nullopt;
91 return isUnsigned ? result->getZExtValue() : result->getSExtValue();
92}
93
94bool isByteType(mlir::Type ty) {
95 if (auto intTy = mlir::dyn_cast<cir::IntType>(Val&: ty))
96 return intTy.getWidth() == 8;
97 return ty.isInteger(width: 8);
98}
99
100std::optional<int64_t> scaledOffset(int64_t index, mlir::Type elemTy,
101 const mlir::DataLayout &dataLayout) {
102 llvm::TypeSize size = dataLayout.getTypeAllocSize(t: elemTy);
103 if (size.isScalable() || size.getFixedValue() >= (1ULL << 63))
104 return std::nullopt;
105 int64_t offset;
106 if (llvm::MulOverflow(X: index, Y: static_cast<int64_t>(size.getFixedValue()),
107 Result&: offset))
108 return std::nullopt;
109 return offset;
110}
111
112bool getConstantStringInfo(mlir::Value ptr, mlir::ModuleOp mod,
113 const mlir::DataLayout &dataLayout,
114 mlir::SymbolTableCollection &symbolTables,
115 std::string &str) {
116 int64_t offset = 0;
117 llvm::SmallPtrSet<mlir::Operation *, 8> seen;
118 while (mlir::Operation *def = ptr.getDefiningOp()) {
119 if (!seen.insert(Ptr: def).second)
120 return false;
121 if (auto cast = mlir::dyn_cast<cir::CastOp>(Val: def)) {
122 if (cast.getKind() != cir::CastKind::array_to_ptrdecay &&
123 cast.getKind() != cir::CastKind::bitcast &&
124 cast.getKind() != cir::CastKind::address_space)
125 break;
126 ptr = cast.getSrc();
127 } else if (auto stride = mlir::dyn_cast<cir::PtrStrideOp>(Val: def)) {
128 mlir::Type elemTy = stride.getElementType();
129 // Lowering treats void and function elements as bytes, and
130 // neither has a data layout entry.
131 if (mlir::isa<cir::VoidType, cir::FuncType>(Val: elemTy))
132 elemTy = cir::IntType::get(context: elemTy.getContext(), width: 8, is_signed: false);
133 std::optional<int64_t> index = getConstantInt(value: stride.getStride());
134 std::optional<int64_t> step =
135 index ? scaledOffset(index: *index, elemTy, dataLayout) : std::nullopt;
136 if (!step || llvm::AddOverflow(X: offset, Y: *step, Result&: offset))
137 return false;
138 ptr = stride.getBase();
139 } else if (auto elt = mlir::dyn_cast<cir::GetElementOp>(Val: def)) {
140 bool isArrayBase =
141 mlir::isa<cir::ArrayType>(Val: elt.getBaseType().getPointee());
142 std::optional<int64_t> index = getConstantInt(value: elt.getIndex());
143 std::optional<int64_t> step =
144 isArrayBase && index
145 ? scaledOffset(index: *index, elemTy: elt.getElementType(), dataLayout)
146 : std::nullopt;
147 if (!step || llvm::AddOverflow(X: offset, Y: *step, Result&: offset))
148 return false;
149 ptr = elt.getBase();
150 } else {
151 break;
152 }
153 }
154
155 auto getGlobal = ptr.getDefiningOp<cir::GetGlobalOp>();
156 if (!getGlobal)
157 return false;
158 mlir::Operation *symbol =
159 symbolTables.lookupSymbolIn(symbolTableOp: mod, name: getGlobal.getNameAttr());
160 auto global = mlir::dyn_cast_if_present<cir::GlobalOp>(Val: symbol);
161 if (!global)
162 return false;
163
164 std::string data;
165 std::optional<mlir::Attribute> init = global.getInitialValue();
166 auto arrayTy = mlir::dyn_cast<cir::ArrayType>(Val: global.getSymType());
167 if (!global.getConstant() || !init || !arrayTy ||
168 !isByteType(ty: arrayTy.getElementType()))
169 return false;
170 if (auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(Val&: *init)) {
171 if (auto strAttr = mlir::dyn_cast<mlir::StringAttr>(Val: constArr.getElts())) {
172 data = strAttr.getValue().str();
173 } else if (auto arrayAttr =
174 mlir::dyn_cast<mlir::ArrayAttr>(Val: constArr.getElts())) {
175 // The element list may be shorter (trailing zeros elided) or
176 // longer than the array type; reserving the larger of the two
177 // keeps the later resize cheap either way.
178 data.reserve(res_arg: std::max<size_t>(a: arrayAttr.size(), b: arrayTy.getSize()));
179 for (mlir::Attribute elt : arrayAttr) {
180 llvm::APInt byte;
181 if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(Val&: elt)) {
182 if (!isByteType(ty: intAttr.getType()))
183 return false;
184 byte = intAttr.getValue();
185 } else if (auto intAttr = mlir::dyn_cast<mlir::IntegerAttr>(Val&: elt)) {
186 if (!isByteType(ty: intAttr.getType()))
187 return false;
188 byte = intAttr.getValue();
189 } else {
190 return false;
191 }
192 data.push_back(c: static_cast<char>(byte.getZExtValue()));
193 }
194 } else {
195 return false;
196 }
197 } else if (!mlir::isa<cir::ZeroAttr>(Val: *init)) {
198 return false;
199 }
200 // Add back the trailing NULs the string attribute leaves out.
201 data.resize(n: arrayTy.getSize(), c: '\0');
202
203 if (offset < 0 || static_cast<uint64_t>(offset) > data.size())
204 return false;
205 llvm::StringRef rest = llvm::StringRef(data).drop_front(N: offset);
206 str = rest.substr(Start: 0, N: rest.find(C: '\0')).str();
207 return true;
208}
209
210// helper struct to package the string related data
211struct StringData {
212 std::string str;
213 mlir::Value realSize;
214 mlir::Value alignedSize;
215 bool isConst = true;
216};
217
218class AMDGPUPrintfLowering {
219public:
220 AMDGPUPrintfLowering(cir::OffloadPrintfOp op,
221 mlir::ConversionPatternRewriter &rewriter,
222 const mlir::DataLayout &dataLayout,
223 mlir::SymbolTableCollection &symbolTables,
224 unsigned genericAddrSpace)
225 : op(op), rewriter(rewriter), dataLayout(dataLayout),
226 symbolTables(symbolTables), mod(op->getParentOfType<mlir::ModuleOp>()),
227 genericAddrSpace(genericAddrSpace) {
228 if (auto attr = op->getAttrOfType<mlir::ArrayAttr>(
229 name: cir::CIRDialect::getPrintfConstStringsAttrName()))
230 if (attr.size() == op.getArgs().size() + 1)
231 constStrings = attr;
232 }
233
234 mlir::LogicalResult lower(mlir::ValueRange args, bool isBuffered);
235
236private:
237 std::optional<std::string> getConstString(size_t operandIndex) const {
238 if (!constStrings || operandIndex >= constStrings.size())
239 return std::nullopt;
240 auto strAttr = mlir::dyn_cast<mlir::StringAttr>(Val: constStrings[operandIndex]);
241 if (!strAttr)
242 return std::nullopt;
243 return strAttr.getValue().str();
244 }
245 mlir::Value getI32(uint32_t value) {
246 return mlir::LLVM::ConstantOp::create(
247 builder&: rewriter, location: op.getLoc(), type: rewriter.getI32Type(), value: llvm::APInt(32, value));
248 }
249 mlir::Value getI64(uint64_t value) {
250 return mlir::LLVM::ConstantOp::create(
251 builder&: rewriter, location: op.getLoc(), type: rewriter.getI64Type(), value: llvm::APInt(64, value));
252 }
253 mlir::Value createCall(llvm::StringRef name, mlir::Type resTy,
254 mlir::ValueRange args);
255 mlir::Value gepInBounds(mlir::Value ptr, mlir::LLVM::GEPArg offset);
256 void addPrintfFormatMetadata(llvm::StringRef entry, bool onlyIfEmpty);
257
258 mlir::Value fitArgInto64Bits(mlir::Value arg);
259 mlir::Value callPrintfBegin(mlir::Value version);
260 mlir::Value callAppendArgs(mlir::Value desc, int numArgs, mlir::Value arg0,
261 mlir::Value arg1, mlir::Value arg2,
262 mlir::Value arg3, mlir::Value arg4,
263 mlir::Value arg5, mlir::Value arg6, bool isLast);
264 mlir::Value appendArg(mlir::Value desc, mlir::Value arg, bool isLast);
265 mlir::Value getStrlenWithNull(mlir::Value str);
266 mlir::Value callAppendStringN(mlir::Value desc, mlir::Value str,
267 mlir::Value length, bool isLast);
268 mlir::Value appendString(mlir::Value desc, mlir::Value arg, bool isLast);
269 mlir::Value processArg(mlir::Value desc, mlir::Value arg, bool specIsCString,
270 bool isLast);
271 bool isCStringArg(mlir::ValueRange args, size_t i) const;
272 mlir::Value callBufferedPrintfStart(
273 mlir::ValueRange args, mlir::Value fmt, bool isConstFmtStr,
274 llvm::SmallVectorImpl<StringData> &stringContents, mlir::Value &argSize);
275 void
276 processConstantStringArg(const StringData &sd,
277 llvm::SmallVectorImpl<mlir::Value> &whatToStore);
278 mlir::Value processNonStringArg(mlir::Value arg);
279 void callBufferedPrintfArgPush(mlir::ValueRange args, mlir::Value ptrToStore,
280 llvm::ArrayRef<StringData> stringContents,
281 bool isConstFmtStr);
282 mlir::Value emitAMDGPUPrintfCall(mlir::ValueRange args, bool isBuffered);
283
284 cir::OffloadPrintfOp op;
285 mlir::ConversionPatternRewriter &rewriter;
286 const mlir::DataLayout &dataLayout;
287 mlir::SymbolTableCollection &symbolTables;
288 mlir::ModuleOp mod;
289 unsigned genericAddrSpace;
290 mlir::ArrayAttr constStrings;
291
292 /// The rest of the op's block, which the expansion branches to.
293 mlir::Block *cont = nullptr;
294 std::string fmtStr;
295 llvm::SparseBitVector<8> specIsCString;
296};
297
298} // namespace
299
300bool isAMDGPUPrintfModule(mlir::ModuleOp module) {
301 llvm::StringRef tripleStr;
302 if (auto tripleAttr = module->getAttrOfType<mlir::StringAttr>(
303 name: cir::CIRDialect::getTripleAttrName()))
304 tripleStr = tripleAttr.getValue();
305 llvm::Triple triple(tripleStr);
306 return triple.isAMDGCN() ||
307 (triple.isSPIRV() && triple.getVendor() == llvm::Triple::AMD);
308}
309
310void annotateAMDGPUPrintfStrings(mlir::ModuleOp mod,
311 mlir::SymbolTableCollection &symbolTables) {
312 mlir::MLIRContext *ctx = mod.getContext();
313 mod.walk(callback: [&](cir::OffloadPrintfOp op) {
314 mlir::ModuleOp opModule = op->getParentOfType<mlir::ModuleOp>();
315 if (!isAMDGPUPrintfModule(module: opModule))
316 return;
317 mlir::DataLayout opLayout(opModule);
318 auto resolve = [&](mlir::Value value) -> mlir::Attribute {
319 std::string str;
320 if (getConstantStringInfo(ptr: value, mod: opModule, dataLayout: opLayout, symbolTables, str))
321 return mlir::StringAttr::get(context: ctx, bytes: str);
322 return mlir::UnitAttr::get(context: ctx);
323 };
324 llvm::SmallVector<mlir::Attribute> entries;
325 entries.reserve(N: op.getArgs().size() + 1);
326 entries.push_back(Elt: resolve(op.getFormat()));
327 for (mlir::Value arg : op.getArgs())
328 entries.push_back(Elt: resolve(arg));
329 op->setAttr(name: cir::CIRDialect::getPrintfConstStringsAttrName(),
330 value: mlir::ArrayAttr::get(context: ctx, value: entries));
331 });
332}
333
334mlir::Value AMDGPUPrintfLowering::createCall(llvm::StringRef name,
335 mlir::Type resTy,
336 mlir::ValueRange args) {
337 auto fnTy = mlir::LLVM::LLVMFunctionType::get(
338 result: resTy, arguments: llvm::to_vector(Range: args.getTypes()), /*isVarArg=*/false);
339 createLLVMFuncOpIfNotExist(rewriter, symbolTables, srcOp: op, fnName: name, fnTy);
340 return mlir::LLVM::CallOp::create(
341 builder&: rewriter, location: op.getLoc(), calleeType: fnTy,
342 callee: mlir::FlatSymbolRefAttr::get(ctx: rewriter.getContext(), value: name), args)
343 .getResult();
344}
345
346mlir::Value AMDGPUPrintfLowering::gepInBounds(mlir::Value ptr,
347 mlir::LLVM::GEPArg offset) {
348 return mlir::LLVM::GEPOp::create(builder&: rewriter, location: op.getLoc(), resultType: ptr.getType(),
349 elementType: rewriter.getI8Type(), basePtr: ptr,
350 indices: llvm::ArrayRef<mlir::LLVM::GEPArg>{offset},
351 noWrapFlags: mlir::LLVM::GEPNoWrapFlags::inbounds);
352}
353
354void AMDGPUPrintfLowering::addPrintfFormatMetadata(llvm::StringRef entry,
355 bool onlyIfEmpty) {
356 constexpr llvm::StringLiteral name = "llvm.printf.fmts";
357 mlir::Attribute node = mlir::LLVM::MDNodeAttr::get(
358 context: rewriter.getContext(),
359 operands: {mlir::LLVM::MDStringAttr::get(
360 context: rewriter.getContext(),
361 value: mlir::StringAttr::get(context: rewriter.getContext(), bytes: entry))});
362
363 mlir::LLVM::NamedMetadataOp metaD;
364 for (auto md : mod.getOps<mlir::LLVM::NamedMetadataOp>()) {
365 if (md.getMetadataName() == name) {
366 metaD = md;
367 break;
368 }
369 }
370
371 if (!metaD) {
372 mlir::OpBuilder::InsertionGuard guard(rewriter);
373 rewriter.setInsertionPointToEnd(mod.getBody());
374 mlir::LLVM::NamedMetadataOp::create(builder&: rewriter, location: mod.getLoc(), metadata_name: name,
375 nodes: rewriter.getArrayAttr(value: node));
376 return;
377 }
378
379 if (onlyIfEmpty && !metaD.getNodes().empty())
380 return;
381 llvm::SmallVector<mlir::Attribute> nodes(metaD.getNodes().getValue());
382 nodes.push_back(Elt: node);
383 rewriter.modifyOpInPlace(
384 root: metaD, callable: [&] { metaD.setNodesAttr(rewriter.getArrayAttr(value: nodes)); });
385}
386
387mlir::Value AMDGPUPrintfLowering::fitArgInto64Bits(mlir::Value arg) {
388 mlir::Type int64Ty = rewriter.getI64Type();
389 mlir::Type ty = arg.getType();
390
391 if (auto intTy = mlir::dyn_cast<mlir::IntegerType>(Val&: ty)) {
392 switch (intTy.getWidth()) {
393 case 32:
394 return mlir::LLVM::ZExtOp::create(builder&: rewriter, location: op.getLoc(), res: int64Ty, arg);
395 case 64:
396 return arg;
397 }
398 }
399
400 if (ty.isF64()) {
401 return mlir::LLVM::BitcastOp::create(builder&: rewriter, location: op.getLoc(), res: int64Ty, arg);
402 }
403
404 if (mlir::isa<mlir::LLVM::LLVMPointerType>(Val: ty)) {
405 return mlir::LLVM::PtrToIntOp::create(builder&: rewriter, location: op.getLoc(), res: int64Ty, arg);
406 }
407
408 llvm_unreachable("unexpected type");
409}
410
411mlir::Value AMDGPUPrintfLowering::callPrintfBegin(mlir::Value version) {
412 return createCall(name: "__ockl_printf_begin", resTy: rewriter.getI64Type(), args: version);
413}
414
415mlir::Value AMDGPUPrintfLowering::callAppendArgs(
416 mlir::Value desc, int numArgs, mlir::Value arg0, mlir::Value arg1,
417 mlir::Value arg2, mlir::Value arg3, mlir::Value arg4, mlir::Value arg5,
418 mlir::Value arg6, bool isLast) {
419 mlir::Value isLastValue = getI32(value: isLast);
420 mlir::Value numArgsValue = getI32(value: numArgs);
421 return createCall(name: "__ockl_printf_append_args", resTy: rewriter.getI64Type(),
422 args: {desc, numArgsValue, arg0, arg1, arg2, arg3, arg4, arg5,
423 arg6, isLastValue});
424}
425
426mlir::Value AMDGPUPrintfLowering::appendArg(mlir::Value desc, mlir::Value arg,
427 bool isLast) {
428 mlir::Value arg0 = fitArgInto64Bits(arg);
429 mlir::Value zero = getI64(value: 0);
430 return callAppendArgs(desc, numArgs: 1, arg0, arg1: zero, arg2: zero, arg3: zero, arg4: zero, arg5: zero, arg6: zero,
431 isLast);
432}
433
434// The device library does not provide strlen, so we build our own loop
435// here. While we are at it, we also include the terminating null in the length.
436mlir::Value AMDGPUPrintfLowering::getStrlenWithNull(mlir::Value str) {
437 mlir::Block *prev = rewriter.getInsertionBlock();
438 mlir::Region *region = cont->getParent();
439
440 mlir::Value charZero = mlir::LLVM::ConstantOp::create(
441 builder&: rewriter, location: op.getLoc(), type: rewriter.getI8Type(), value: llvm::APInt(8, 0));
442 mlir::Value one = getI64(value: 1);
443 mlir::Value zero = getI64(value: 0);
444 mlir::Type int64Ty = rewriter.getI64Type();
445
446 // The length is either zero for a null pointer, or the computed value for an
447 // actual string. We need a join block for a phi that represents the final
448 // value.
449 //
450 // Strictly speaking, the zero does not matter since
451 // __ockl_printf_append_string_n ignores the length if the pointer is null.
452 mlir::Block *join = rewriter.createBlock(parent: region, insertPt: cont->getIterator(),
453 argTypes: {int64Ty}, locs: {op.getLoc()});
454 mlir::Block *whileBlock = rewriter.createBlock(
455 parent: region, insertPt: join->getIterator(), argTypes: {str.getType()}, locs: {op.getLoc()});
456 mlir::Block *whileDone = rewriter.createBlock(parent: region, insertPt: join->getIterator());
457
458 // Emit an early return for when the pointer is null.
459 rewriter.setInsertionPointToEnd(prev);
460 mlir::Value cmpNull = mlir::LLVM::ICmpOp::create(
461 builder&: rewriter, location: op.getLoc(), predicate: mlir::LLVM::ICmpPredicate::eq, lhs: str,
462 rhs: mlir::LLVM::ZeroOp::create(builder&: rewriter, location: op.getLoc(), res: str.getType()));
463 mlir::LLVM::CondBrOp::create(builder&: rewriter, location: op.getLoc(), condition: cmpNull, trueDest: join, trueOperands: zero,
464 falseDest: whileBlock, falseOperands: str);
465
466 // Entry to the while loop.
467 rewriter.setInsertionPointToEnd(whileBlock);
468
469 mlir::Value ptrPhi = whileBlock->getArgument(i: 0);
470 mlir::Value ptrNext = mlir::LLVM::GEPOp::create(
471 builder&: rewriter, location: op.getLoc(), resultType: str.getType(), elementType: rewriter.getI8Type(), basePtr: ptrPhi,
472 indices: llvm::ArrayRef<mlir::LLVM::GEPArg>{one});
473
474 // Condition for the while loop.
475 mlir::Value data = mlir::LLVM::LoadOp::create(builder&: rewriter, location: op.getLoc(),
476 type: rewriter.getI8Type(), addr: ptrPhi);
477 mlir::Value cmp = mlir::LLVM::ICmpOp::create(
478 builder&: rewriter, location: op.getLoc(), predicate: mlir::LLVM::ICmpPredicate::eq, lhs: data, rhs: charZero);
479 mlir::LLVM::CondBrOp::create(builder&: rewriter, location: op.getLoc(), condition: cmp, trueDest: whileDone,
480 trueOperands: mlir::ValueRange(), falseDest: whileBlock, falseOperands: ptrNext);
481
482 // Add one to the computed length.
483 rewriter.setInsertionPointToEnd(whileDone);
484 auto addrTy = mlir::IntegerType::get(
485 context: rewriter.getContext(), width: *dataLayout.getTypeIndexBitwidth(t: str.getType()));
486 mlir::Value endAddr =
487 mlir::LLVM::PtrToAddrOp::create(builder&: rewriter, location: op.getLoc(), res: addrTy, arg: ptrPhi);
488 mlir::Value beginAddr =
489 mlir::LLVM::PtrToAddrOp::create(builder&: rewriter, location: op.getLoc(), res: addrTy, arg: str);
490 mlir::Value len =
491 mlir::LLVM::SubOp::create(builder&: rewriter, location: op.getLoc(), lhs: endAddr, rhs: beginAddr);
492 if (addrTy != int64Ty)
493 len = mlir::LLVM::ZExtOp::create(builder&: rewriter, location: op.getLoc(), res: int64Ty, arg: len);
494 len = mlir::LLVM::AddOp::create(builder&: rewriter, location: op.getLoc(), lhs: len, rhs: one);
495
496 // Final join.
497 mlir::LLVM::BrOp::create(builder&: rewriter, location: op.getLoc(), operands: len, dest: join);
498 rewriter.setInsertionPointToEnd(join);
499 return join->getArgument(i: 0);
500}
501
502mlir::Value AMDGPUPrintfLowering::callAppendStringN(mlir::Value desc,
503 mlir::Value str,
504 mlir::Value length,
505 bool isLast) {
506 mlir::Value isLastInt32 = getI32(value: isLast);
507 // __ockl_printf_append_string_n takes a generic pointer, which can address a
508 // string in any address space.
509 auto ptrTy =
510 mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: genericAddrSpace);
511 if (str.getType() != ptrTy)
512 str =
513 mlir::LLVM::AddrSpaceCastOp::create(builder&: rewriter, location: op.getLoc(), res: ptrTy, arg: str);
514 return createCall(name: "__ockl_printf_append_string_n", resTy: rewriter.getI64Type(),
515 args: {desc, str, length, isLastInt32});
516}
517
518mlir::Value AMDGPUPrintfLowering::appendString(mlir::Value desc,
519 mlir::Value arg, bool isLast) {
520 mlir::Value length = getStrlenWithNull(str: arg);
521 return callAppendStringN(desc, str: arg, length, isLast);
522}
523
524mlir::Value AMDGPUPrintfLowering::processArg(mlir::Value desc, mlir::Value arg,
525 bool specIsCString, bool isLast) {
526 if (specIsCString && mlir::isa<mlir::LLVM::LLVMPointerType>(Val: arg.getType())) {
527 return appendString(desc, arg, isLast);
528 }
529 // If the format specifies a string but the argument is not, the frontend will
530 // have printed a warning. We just rely on undefined behaviour and send the
531 // argument anyway.
532 return appendArg(desc, arg, isLast);
533}
534
535// As in processArg, a non-pointer argument for a string is sent as a scalar.
536bool AMDGPUPrintfLowering::isCStringArg(mlir::ValueRange args, size_t i) const {
537 return specIsCString.test(Idx: i) &&
538 mlir::isa<mlir::LLVM::LLVMPointerType>(Val: args[i].getType());
539}
540
541// Calculates frame size required for current printf expansion and allocates
542// space on printf buffer. Printf frame includes following contents
543// [ ControlDWord , format string/Hash , Arguments (each aligned to 8 byte) ]
544mlir::Value AMDGPUPrintfLowering::callBufferedPrintfStart(
545 mlir::ValueRange args, mlir::Value fmt, bool isConstFmtStr,
546 llvm::SmallVectorImpl<StringData> &stringContents, mlir::Value &argSize) {
547 mlir::Value nonConstStrLen;
548 mlir::Value lenWithNull;
549 mlir::Value lenWithNullAligned;
550 mlir::Value tempAdd;
551
552 // First 4 bytes to be reserved for control dword
553 size_t bufSize = 4;
554 if (isConstFmtStr)
555 // First 8 bytes of MD5 hash
556 bufSize += 8;
557 else {
558 lenWithNull = getStrlenWithNull(str: fmt);
559
560 // Align the computed length to next 8 byte boundary
561 tempAdd = mlir::LLVM::AddOp::create(builder&: rewriter, location: op.getLoc(), lhs: lenWithNull,
562 rhs: getI64(value: 7U));
563 nonConstStrLen =
564 mlir::LLVM::AndOp::create(builder&: rewriter, location: op.getLoc(), lhs: tempAdd, rhs: getI64(value: ~7U));
565
566 stringContents.push_back(Elt: {.str: "", .realSize: lenWithNull, .alignedSize: nonConstStrLen, .isConst: false});
567 }
568
569 for (size_t i = 1; i < args.size(); i++) {
570 if (isCStringArg(args, i)) {
571 if (std::optional<std::string> argStr = getConstString(operandIndex: i)) {
572 auto alignedLen = llvm::alignTo(Value: argStr->size() + 1, Align: 8);
573 stringContents.push_back(Elt: {.str: *argStr, /*realSize=*/{},
574 /*alignedSize=*/{}, /*isConst=*/true});
575 bufSize += alignedLen;
576 } else {
577 lenWithNull = getStrlenWithNull(str: args[i]);
578
579 // Align the computed length to next 8 byte boundary
580 tempAdd = mlir::LLVM::AddOp::create(builder&: rewriter, location: op.getLoc(), lhs: lenWithNull,
581 rhs: getI64(value: 7U));
582 lenWithNullAligned = mlir::LLVM::AndOp::create(builder&: rewriter, location: op.getLoc(),
583 lhs: tempAdd, rhs: getI64(value: ~7U));
584
585 if (nonConstStrLen) {
586 mlir::Value val = mlir::LLVM::AddOp::create(
587 builder&: rewriter, location: op.getLoc(), lhs: lenWithNullAligned, rhs: nonConstStrLen);
588 nonConstStrLen = val;
589 } else
590 nonConstStrLen = lenWithNullAligned;
591
592 stringContents.push_back(Elt: {.str: "", .realSize: lenWithNull, .alignedSize: lenWithNullAligned, .isConst: false});
593 }
594 } else {
595 uint64_t allocSize =
596 dataLayout.getTypeAllocSize(t: args[i].getType()).getFixedValue();
597 // We end up expanding non string arguments to 8 bytes
598 // (args smaller than 8 bytes)
599 bufSize += llvm::alignTo(Value: allocSize, Align: 8);
600 }
601 }
602
603 // calculate final size value to be passed to printf_alloc
604 mlir::Value sizeToReserve = getI64(value: bufSize);
605 if (nonConstStrLen)
606 sizeToReserve = mlir::LLVM::AddOp::create(builder&: rewriter, location: op.getLoc(),
607 lhs: nonConstStrLen, rhs: sizeToReserve);
608
609 argSize = mlir::LLVM::TruncOp::create(builder&: rewriter, location: op.getLoc(),
610 res: rewriter.getI32Type(), arg: sizeToReserve);
611
612 // call the printf_alloc function
613 unsigned globalAS = 0;
614 if (auto as = mlir::dyn_cast_if_present<mlir::IntegerAttr>(
615 Val: dataLayout.getGlobalMemorySpace()))
616 globalAS = as.getValue().getZExtValue();
617 auto ptrTy =
618 mlir::LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: globalAS);
619 auto allocName =
620 mlir::StringAttr::get(context: rewriter.getContext(), bytes: "__printf_alloc");
621 bool declared = symbolTables.lookupSymbolIn(symbolTableOp: mod, symbol: allocName);
622 mlir::Value ptr = createCall(name: allocName, resTy: ptrTy, args: argSize);
623 if (!declared)
624 mlir::cast<mlir::LLVM::LLVMFuncOp>(
625 Val: symbolTables.lookupSymbolIn(symbolTableOp: mod, symbol: allocName))
626 .setNoUnwind(true);
627 return ptr;
628}
629
630// Prepare constant string argument to push onto the buffer
631void AMDGPUPrintfLowering::processConstantStringArg(
632 const StringData &sd, llvm::SmallVectorImpl<mlir::Value> &whatToStore) {
633 llvm::SmallVector<uint32_t, 16> words;
634 llvm::packAMDGPUPrintfConstantString(Str: sd.str, Words&: words);
635 for (uint32_t word : words)
636 whatToStore.push_back(Elt: getI32(value: word));
637}
638
639mlir::Value AMDGPUPrintfLowering::processNonStringArg(mlir::Value arg) {
640 mlir::Type ty = arg.getType();
641
642 if (auto intTy = mlir::dyn_cast<mlir::IntegerType>(Val&: ty)) {
643 if (intTy.getWidth() < 64) {
644 return mlir::LLVM::ZExtOp::create(builder&: rewriter, location: op.getLoc(),
645 res: rewriter.getI64Type(), arg);
646 }
647 }
648
649 if (mlir::isa<mlir::FloatType>(Val: ty)) {
650 if (dataLayout.getTypeAllocSize(t: ty).getFixedValue() < 8) {
651 return mlir::LLVM::FPExtOp::create(builder&: rewriter, location: op.getLoc(),
652 res: rewriter.getF64Type(), arg);
653 }
654 }
655
656 return arg;
657}
658
659void AMDGPUPrintfLowering::callBufferedPrintfArgPush(
660 mlir::ValueRange args, mlir::Value ptrToStore,
661 llvm::ArrayRef<StringData> stringContents, bool isConstFmtStr) {
662 const StringData *strIt = stringContents.begin();
663 size_t i = isConstFmtStr ? 1 : 0;
664 for (; i < args.size(); i++) {
665 if (i != 0 && !isCStringArg(args, i)) {
666 // The runtime reads each non-string argument from a slot of at least 8
667 // bytes, also when the stored value is smaller.
668 mlir::Value toStore = processNonStringArg(arg: args[i]);
669 mlir::LLVM::StoreOp::create(builder&: rewriter, location: op.getLoc(), value: toStore, addr: ptrToStore);
670 uint64_t allocSize =
671 dataLayout.getTypeAllocSize(t: toStore.getType()).getFixedValue();
672 ptrToStore = gepInBounds(ptr: ptrToStore, offset: llvm::alignTo(Value: allocSize, Align: 8));
673 continue;
674 }
675
676 if (!strIt->isConst) {
677 // This copies the contents of the string, however the next offset
678 // is at aligned length, the extra space that might be created due
679 // to alignment padding is not populated with any specific value
680 // here. This would be safe as long as runtime is sync with
681 // the offsets.
682 auto memcpy = mlir::LLVM::MemcpyOp::create(
683 builder&: rewriter, location: op.getLoc(), dst: ptrToStore, src: args[i], len: strIt->realSize,
684 /*isVolatile=*/false);
685 mlir::NamedAttribute align1(mlir::LLVM::LLVMDialect::getAlignAttrName(),
686 rewriter.getI64IntegerAttr(value: 1));
687 memcpy.setArgAttrsAttr(
688 rewriter.getArrayAttr(value: {rewriter.getDictionaryAttr(value: align1),
689 rewriter.getDictionaryAttr(value: align1),
690 rewriter.getDictionaryAttr(value: {})}));
691
692 ptrToStore = gepInBounds(ptr: ptrToStore, offset: strIt->alignedSize);
693
694 // done with current argument, move to next
695 strIt++;
696 continue;
697 }
698
699 llvm::SmallVector<mlir::Value, 32> whatToStore;
700 processConstantStringArg(sd: *strIt, whatToStore);
701 strIt++;
702
703 for (mlir::Value toStore : whatToStore) {
704 mlir::LLVM::StoreOp::create(builder&: rewriter, location: op.getLoc(), value: toStore, addr: ptrToStore);
705 ptrToStore = gepInBounds(
706 ptr: ptrToStore,
707 offset: dataLayout.getTypeAllocSize(t: toStore.getType()).getFixedValue());
708 }
709 }
710}
711
712mlir::Value AMDGPUPrintfLowering::emitAMDGPUPrintfCall(mlir::ValueRange args,
713 bool isBuffered) {
714 size_t numOps = args.size();
715 assert(numOps >= 1);
716
717 mlir::Value fmt = args[0];
718
719 if (std::optional<std::string> resolved = getConstString(operandIndex: 0)) {
720 fmtStr = *resolved;
721 llvm::locateAMDGPUPrintfCStrings(BV&: specIsCString, Fmt: fmtStr);
722 }
723
724 if (isBuffered) {
725 llvm::SmallVector<StringData, 8> stringContents;
726 bool isConstFmtStr = !fmtStr.empty();
727
728 mlir::Value argSize;
729 mlir::Value ptr = callBufferedPrintfStart(args, fmt, isConstFmtStr,
730 stringContents, argSize);
731
732 // The buffered version still follows OpenCL printf standards for
733 // printf return value, i.e 0 on success, -1 on failure.
734 mlir::Value cmp = mlir::LLVM::ICmpOp::create(
735 builder&: rewriter, location: op.getLoc(), predicate: mlir::LLVM::ICmpPredicate::ne, lhs: ptr,
736 rhs: mlir::LLVM::ZeroOp::create(builder&: rewriter, location: op.getLoc(), res: ptr.getType()));
737
738 // The rest of the op's block is the end block.
739 mlir::Block *end = cont;
740 mlir::Block *argPush =
741 rewriter.createBlock(parent: cont->getParent(), insertPt: cont->getIterator());
742
743 rewriter.setInsertionPointToEnd(cmp.getParentBlock());
744 mlir::LLVM::CondBrOp::create(builder&: rewriter, location: op.getLoc(), condition: cmp, trueDest: argPush, falseDest: end);
745 rewriter.setInsertionPointToEnd(argPush);
746
747 // Create controlDWord and store as the first entry, format as follows
748 // Bit 0 (LSB) -> stream (1 if stderr, 0 if stdout, printf always outputs to
749 // stdout) Bit 1 -> constant format string (1 if constant) Bits 2-31 -> size
750 // of printf data frame
751 mlir::Value constantTwo = getI32(value: 2);
752 mlir::Value controlDWord =
753 mlir::LLVM::ShlOp::create(builder&: rewriter, location: op.getLoc(), lhs: argSize, rhs: constantTwo);
754 if (isConstFmtStr)
755 controlDWord = mlir::LLVM::OrOp::create(builder&: rewriter, location: op.getLoc(),
756 lhs: controlDWord, rhs: constantTwo);
757
758 mlir::LLVM::StoreOp::create(builder&: rewriter, location: op.getLoc(), value: controlDWord, addr: ptr);
759
760 ptr = gepInBounds(ptr, offset: 4);
761
762 // Create MD5 hash for constant format string, push low 64 bits of the
763 // same onto buffer and metadata.
764 if (isConstFmtStr) {
765 addPrintfFormatMetadata(entry: llvm::getAMDGPUPrintfFormatMetadata(Fmt: fmtStr),
766 /*onlyIfEmpty=*/false);
767
768 mlir::LLVM::StoreOp::create(
769 builder&: rewriter, location: op.getLoc(),
770 value: getI64(value: llvm::getAMDGPUPrintfFormatHash(Fmt: fmtStr)), addr: ptr);
771 ptr = gepInBounds(ptr, offset: 8);
772 } else {
773 // Include a dummy metadata instance in case of only non constant
774 // format string usage, This might be an absurd usecase but needs to
775 // be done for completeness
776 addPrintfFormatMetadata(entry: llvm::AMDGPUPrintfNonConstFormatMetadata,
777 /*onlyIfEmpty=*/true);
778 }
779
780 // Push The printf arguments onto buffer
781 callBufferedPrintfArgPush(args, ptrToStore: ptr, stringContents, isConstFmtStr);
782
783 // End block, returns -1 on failure
784 mlir::LLVM::BrOp::create(builder&: rewriter, location: op.getLoc(), dest: end);
785 rewriter.setInsertionPointToStart(end);
786 mlir::Value notCmp = mlir::LLVM::XOrOp::create(
787 builder&: rewriter, location: op.getLoc(), lhs: cmp,
788 rhs: mlir::LLVM::ConstantOp::create(
789 builder&: rewriter, location: op.getLoc(), type: rewriter.getI1Type(), value: llvm::APInt(1, 1)));
790 return mlir::LLVM::SExtOp::create(builder&: rewriter, location: op.getLoc(),
791 res: rewriter.getI32Type(), arg: notCmp);
792 }
793
794 mlir::Value desc = callPrintfBegin(version: getI64(value: 0));
795 desc = appendString(desc, arg: fmt, isLast: numOps == 1);
796
797 // FIXME: This invokes hostcall once for each argument. We can pack up to
798 // seven scalar printf arguments in a single hostcall. See the signature of
799 // callAppendArgs().
800 for (unsigned int i = 1; i != numOps; ++i) {
801 bool isLast = i == numOps - 1;
802 bool isCString = specIsCString.test(Idx: i);
803 desc = processArg(desc, arg: args[i], specIsCString: isCString, isLast);
804 }
805
806 return mlir::LLVM::TruncOp::create(builder&: rewriter, location: op.getLoc(),
807 res: rewriter.getI32Type(), arg: desc);
808}
809
810mlir::LogicalResult AMDGPUPrintfLowering::lower(mlir::ValueRange args,
811 bool isBuffered) {
812 // Unlike fitArgInto64Bits's unreachable, report a type hostcall cannot send.
813 if (!isBuffered) {
814 for (auto [cirArg, arg] : llvm::zip(t: op.getArgs(), u: args.drop_front())) {
815 mlir::Type ty = arg.getType();
816 if (ty.isInteger(width: 32) || ty.isInteger(width: 64) || ty.isF64() ||
817 mlir::isa<mlir::LLVM::LLVMPointerType>(Val: ty))
818 continue;
819 return op.emitError() << "unsupported argument type " << cirArg.getType()
820 << " for AMDGPU printf";
821 }
822 }
823
824 mlir::Block *block = op->getBlock();
825 cont = rewriter.splitBlock(block, before: mlir::Block::iterator(op));
826 rewriter.setInsertionPointToEnd(block);
827
828 mlir::Value result = emitAMDGPUPrintfCall(args, isBuffered);
829 if (rewriter.getInsertionBlock() != cont)
830 mlir::LLVM::BrOp::create(builder&: rewriter, location: op.getLoc(), dest: cont);
831
832 rewriter.replaceOp(op, newValues: result);
833 return mlir::success();
834}
835
836mlir::LogicalResult lowerAMDGPUPrintf(cir::OffloadPrintfOp op,
837 mlir::ValueRange operands,
838 mlir::ConversionPatternRewriter &rewriter,
839 const mlir::DataLayout &dataLayout,
840 mlir::SymbolTableCollection &symbolTables,
841 unsigned genericAddrSpace,
842 bool isBuffered) {
843 return AMDGPUPrintfLowering(op, rewriter, dataLayout, symbolTables,
844 genericAddrSpace)
845 .lower(args: operands, isBuffered);
846}
847
848} // namespace direct
849} // namespace cir
850