1//=== AMDGPUPrintfRuntimeBinding.cpp - OpenCL printf implementation -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// \file
9//
10// The pass bind printfs to a kernel arg pointer that will be bound to a buffer
11// later by the runtime.
12//
13// This pass traverses the functions in the module and converts
14// each call to printf to a sequence of operations that
15// store the following into the printf buffer:
16// - format string (passed as a module's metadata unique ID)
17// - bitwise copies of printf arguments
18// The backend passes will need to store metadata in the kernel
19//===----------------------------------------------------------------------===//
20
21#include "AMDGPU.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/IR/DiagnosticInfo.h"
25#include "llvm/IR/Dominators.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/Instructions.h"
28#include "llvm/IR/Module.h"
29#include "llvm/InitializePasses.h"
30#include "llvm/Support/DataExtractor.h"
31#include "llvm/TargetParser/Triple.h"
32#include "llvm/Transforms/Utils/BasicBlockUtils.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "printfToRuntime"
37enum { DWORD_ALIGN = 4 };
38
39namespace {
40class AMDGPUPrintfRuntimeBinding final : public ModulePass {
41
42public:
43 static char ID;
44
45 explicit AMDGPUPrintfRuntimeBinding();
46
47private:
48 bool runOnModule(Module &M) override;
49};
50
51class AMDGPUPrintfRuntimeBindingImpl {
52public:
53 AMDGPUPrintfRuntimeBindingImpl() = default;
54 bool run(Module &M);
55
56private:
57 void getConversionSpecifiers(SmallVectorImpl<char> &OpConvSpecifiers,
58 StringRef fmt, size_t num_ops) const;
59
60 bool lowerPrintfForGpu(Module &M);
61
62 const DataLayout *TD;
63 SmallVector<CallInst *, 32> Printfs;
64};
65} // namespace
66
67char AMDGPUPrintfRuntimeBinding::ID = 0;
68
69INITIALIZE_PASS_BEGIN(AMDGPUPrintfRuntimeBinding,
70 "amdgpu-printf-runtime-binding", "AMDGPU Printf lowering",
71 false, false)
72INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
73INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
74INITIALIZE_PASS_END(AMDGPUPrintfRuntimeBinding, "amdgpu-printf-runtime-binding",
75 "AMDGPU Printf lowering", false, false)
76
77char &llvm::AMDGPUPrintfRuntimeBindingID = AMDGPUPrintfRuntimeBinding::ID;
78
79namespace llvm {
80ModulePass *createAMDGPUPrintfRuntimeBinding() {
81 return new AMDGPUPrintfRuntimeBinding();
82}
83} // namespace llvm
84
85AMDGPUPrintfRuntimeBinding::AMDGPUPrintfRuntimeBinding() : ModulePass(ID) {
86 initializeAMDGPUPrintfRuntimeBindingPass(Registry&: *PassRegistry::getPassRegistry());
87}
88
89void AMDGPUPrintfRuntimeBindingImpl::getConversionSpecifiers(
90 SmallVectorImpl<char> &OpConvSpecifiers, StringRef Fmt,
91 size_t NumOps) const {
92 // not all format characters are collected.
93 // At this time the format characters of interest
94 // are %p and %s, which use to know if we
95 // are either storing a literal string or a
96 // pointer to the printf buffer.
97 static const char ConvSpecifiers[] = "cdieEfgGaosuxXp";
98 size_t CurFmtSpecifierIdx = 0;
99 size_t PrevFmtSpecifierIdx = 0;
100
101 while ((CurFmtSpecifierIdx = Fmt.find_first_of(
102 Chars: ConvSpecifiers, From: CurFmtSpecifierIdx)) != StringRef::npos) {
103 bool ArgDump = false;
104 StringRef CurFmt = Fmt.substr(Start: PrevFmtSpecifierIdx,
105 N: CurFmtSpecifierIdx - PrevFmtSpecifierIdx);
106 size_t pTag = CurFmt.find_last_of(C: '%');
107 if (pTag != StringRef::npos) {
108 ArgDump = true;
109 while (pTag && CurFmt[--pTag] == '%') {
110 ArgDump = !ArgDump;
111 }
112 }
113
114 if (ArgDump)
115 OpConvSpecifiers.push_back(Elt: Fmt[CurFmtSpecifierIdx]);
116
117 PrevFmtSpecifierIdx = ++CurFmtSpecifierIdx;
118 }
119}
120
121static bool shouldPrintAsStr(char Specifier, Type *OpType) {
122 return Specifier == 's' && isa<PointerType>(Val: OpType);
123}
124
125constexpr StringLiteral NonLiteralStr("???");
126static_assert(NonLiteralStr.size() == 3);
127
128static StringRef getAsConstantStr(Value *V) {
129 StringRef S;
130 if (!getConstantStringInfo(V, Str&: S))
131 S = NonLiteralStr;
132
133 return S;
134}
135
136static void diagnoseInvalidFormatString(const CallBase *CI) {
137 DiagnosticInfoUnsupported UnsupportedFormatStr(
138 *CI->getParent()->getParent(),
139 "printf format string must be a trivially resolved constant string "
140 "global variable",
141 CI->getDebugLoc());
142 CI->getContext().diagnose(DI: UnsupportedFormatStr);
143}
144
145bool AMDGPUPrintfRuntimeBindingImpl::lowerPrintfForGpu(Module &M) {
146 LLVMContext &Ctx = M.getContext();
147 IRBuilder<> Builder(Ctx);
148 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
149
150 // Instead of creating global variables, the printf format strings are
151 // extracted and passed as metadata. This avoids polluting llvm's symbol
152 // tables in this module. Metadata is going to be extracted by the backend
153 // passes and inserted into the OpenCL binary as appropriate.
154 NamedMDNode *metaD = M.getOrInsertNamedMetadata(Name: "llvm.printf.fmts");
155 unsigned UniqID = metaD->getNumOperands();
156
157 for (auto *CI : Printfs) {
158 unsigned NumOps = CI->arg_size();
159
160 SmallString<16> OpConvSpecifiers;
161 Value *Op = CI->getArgOperand(i: 0);
162
163 StringRef FormatStr;
164 if (!getConstantStringInfo(V: Op, Str&: FormatStr)) {
165 Value *Stripped = Op->stripPointerCasts();
166 if (!isa<UndefValue>(Val: Stripped) && !isa<ConstantPointerNull>(Val: Stripped))
167 diagnoseInvalidFormatString(CI);
168 continue;
169 }
170
171 // We need this call to ascertain that we are printing a string or a
172 // pointer. It takes out the specifiers and fills up the first arg.
173 getConversionSpecifiers(OpConvSpecifiers, Fmt: FormatStr, NumOps: NumOps - 1);
174
175 // Add metadata for the string
176 std::string AStreamHolder;
177 raw_string_ostream Sizes(AStreamHolder);
178 int Sum = DWORD_ALIGN;
179 Sizes << CI->arg_size() - 1;
180 Sizes << ':';
181 for (unsigned ArgCount = 1;
182 ArgCount < CI->arg_size() && ArgCount <= OpConvSpecifiers.size();
183 ArgCount++) {
184 Value *Arg = CI->getArgOperand(i: ArgCount);
185 Type *ArgType = Arg->getType();
186 unsigned ArgSize = TD->getTypeAllocSize(Ty: ArgType);
187 //
188 // ArgSize by design should be a multiple of DWORD_ALIGN,
189 // expand the arguments that do not follow this rule.
190 //
191 if (ArgSize % DWORD_ALIGN != 0) {
192 Type *ResType = Type::getInt32Ty(C&: Ctx);
193 if (auto *VecType = dyn_cast<VectorType>(Val: ArgType))
194 ResType = VectorType::get(ElementType: ResType, EC: VecType->getElementCount());
195 Builder.SetInsertPoint(CI);
196 Builder.SetCurrentDebugLocation(CI->getDebugLoc());
197
198 if (ArgType->isFloatingPointTy()) {
199 Arg = Builder.CreateBitCast(
200 V: Arg,
201 DestTy: IntegerType::getIntNTy(C&: Ctx, N: ArgType->getPrimitiveSizeInBits()));
202 }
203
204 if (OpConvSpecifiers[ArgCount - 1] == 'x' ||
205 OpConvSpecifiers[ArgCount - 1] == 'X' ||
206 OpConvSpecifiers[ArgCount - 1] == 'u' ||
207 OpConvSpecifiers[ArgCount - 1] == 'o')
208 Arg = Builder.CreateZExt(V: Arg, DestTy: ResType);
209 else
210 Arg = Builder.CreateSExt(V: Arg, DestTy: ResType);
211 ArgType = Arg->getType();
212 ArgSize = TD->getTypeAllocSize(Ty: ArgType);
213 CI->setOperand(i_nocapture: ArgCount, Val_nocapture: Arg);
214 }
215 if (OpConvSpecifiers[ArgCount - 1] == 'f') {
216 ConstantFP *FpCons = dyn_cast<ConstantFP>(Val: Arg);
217 if (FpCons)
218 ArgSize = 4;
219 else {
220 FPExtInst *FpExt = dyn_cast<FPExtInst>(Val: Arg);
221 if (FpExt && FpExt->getType()->isDoubleTy() &&
222 FpExt->getOperand(i_nocapture: 0)->getType()->isFloatTy())
223 ArgSize = 4;
224 }
225 }
226 if (shouldPrintAsStr(Specifier: OpConvSpecifiers[ArgCount - 1], OpType: ArgType))
227 ArgSize = alignTo(Value: getAsConstantStr(V: Arg).size() + 1, Align: 4);
228
229 LLVM_DEBUG(dbgs() << "Printf ArgSize (in buffer) = " << ArgSize
230 << " for type: " << *ArgType << '\n');
231 Sizes << ArgSize << ':';
232 Sum += ArgSize;
233 }
234 LLVM_DEBUG(dbgs() << "Printf format string in source = " << FormatStr
235 << '\n');
236 for (char C : FormatStr) {
237 // Rest of the C escape sequences (e.g. \') are handled correctly
238 // by the MDParser
239 switch (C) {
240 case '\a':
241 Sizes << "\\a";
242 break;
243 case '\b':
244 Sizes << "\\b";
245 break;
246 case '\f':
247 Sizes << "\\f";
248 break;
249 case '\n':
250 Sizes << "\\n";
251 break;
252 case '\r':
253 Sizes << "\\r";
254 break;
255 case '\v':
256 Sizes << "\\v";
257 break;
258 case ':':
259 // ':' cannot be scanned by Flex, as it is defined as a delimiter
260 // Replace it with it's octal representation \72
261 Sizes << "\\72";
262 break;
263 default:
264 Sizes << C;
265 break;
266 }
267 }
268
269 // Insert the printf_alloc call
270 Builder.SetInsertPoint(CI);
271 Builder.SetCurrentDebugLocation(CI->getDebugLoc());
272
273 AttributeList Attr = AttributeList::get(C&: Ctx, Index: AttributeList::FunctionIndex,
274 Kinds: Attribute::NoUnwind);
275
276 Type *SizetTy = Type::getInt32Ty(C&: Ctx);
277
278 Type *Tys_alloc[1] = {SizetTy};
279 Type *I8Ty = Type::getInt8Ty(C&: Ctx);
280 Type *I8Ptr = PointerType::get(ElementType: I8Ty, AddressSpace: 1);
281 FunctionType *FTy_alloc = FunctionType::get(Result: I8Ptr, Params: Tys_alloc, isVarArg: false);
282 FunctionCallee PrintfAllocFn =
283 M.getOrInsertFunction(Name: StringRef("__printf_alloc"), T: FTy_alloc, AttributeList: Attr);
284
285 LLVM_DEBUG(dbgs() << "Printf metadata = " << Sizes.str() << '\n');
286 std::string fmtstr = itostr(X: ++UniqID) + ":" + Sizes.str();
287 MDString *fmtStrArray = MDString::get(Context&: Ctx, Str: fmtstr);
288
289 MDNode *myMD = MDNode::get(Context&: Ctx, MDs: fmtStrArray);
290 metaD->addOperand(M: myMD);
291 Value *sumC = ConstantInt::get(Ty: SizetTy, V: Sum, IsSigned: false);
292 SmallVector<Value *, 1> alloc_args;
293 alloc_args.push_back(Elt: sumC);
294 CallInst *pcall = CallInst::Create(Func: PrintfAllocFn, Args: alloc_args,
295 NameStr: "printf_alloc_fn", InsertBefore: CI->getIterator());
296
297 //
298 // Insert code to split basicblock with a
299 // piece of hammock code.
300 // basicblock splits after buffer overflow check
301 //
302 ConstantPointerNull *zeroIntPtr =
303 ConstantPointerNull::get(T: PointerType::get(ElementType: I8Ty, AddressSpace: 1));
304 auto *cmp = cast<ICmpInst>(Val: Builder.CreateICmpNE(LHS: pcall, RHS: zeroIntPtr, Name: ""));
305 if (!CI->use_empty()) {
306 Value *result =
307 Builder.CreateSExt(V: Builder.CreateNot(V: cmp), DestTy: I32Ty, Name: "printf_res");
308 CI->replaceAllUsesWith(V: result);
309 }
310 SplitBlock(Old: CI->getParent(), SplitPt: cmp);
311 Instruction *Brnch =
312 SplitBlockAndInsertIfThen(Cond: cmp, SplitBefore: cmp->getNextNode(), Unreachable: false);
313 BasicBlock::iterator BrnchPoint = Brnch->getIterator();
314
315 Builder.SetInsertPoint(Brnch);
316
317 // store unique printf id in the buffer
318 //
319 GetElementPtrInst *BufferIdx = GetElementPtrInst::Create(
320 PointeeType: I8Ty, Ptr: pcall, IdxList: ConstantInt::get(Context&: Ctx, V: APInt(32, 0)), NameStr: "PrintBuffID",
321 InsertBefore: BrnchPoint);
322
323 Type *idPointer = PointerType::get(ElementType: I32Ty, AddressSpace: AMDGPUAS::GLOBAL_ADDRESS);
324 Value *id_gep_cast =
325 new BitCastInst(BufferIdx, idPointer, "PrintBuffIdCast", BrnchPoint);
326
327 new StoreInst(ConstantInt::get(Ty: I32Ty, V: UniqID), id_gep_cast, BrnchPoint);
328
329 // 1st 4 bytes hold the printf_id
330 // the following GEP is the buffer pointer
331 BufferIdx = GetElementPtrInst::Create(PointeeType: I8Ty, Ptr: pcall,
332 IdxList: ConstantInt::get(Context&: Ctx, V: APInt(32, 4)),
333 NameStr: "PrintBuffGep", InsertBefore: BrnchPoint);
334
335 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
336 for (unsigned ArgCount = 1;
337 ArgCount < CI->arg_size() && ArgCount <= OpConvSpecifiers.size();
338 ArgCount++) {
339 Value *Arg = CI->getArgOperand(i: ArgCount);
340 Type *ArgType = Arg->getType();
341 SmallVector<Value *, 32> WhatToStore;
342 if (ArgType->isFPOrFPVectorTy() && !isa<VectorType>(Val: ArgType)) {
343 if (OpConvSpecifiers[ArgCount - 1] == 'f') {
344 if (auto *FpCons = dyn_cast<ConstantFP>(Val: Arg)) {
345 APFloat Val(FpCons->getValueAPF());
346 bool Lost = false;
347 Val.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
348 losesInfo: &Lost);
349 Arg = ConstantFP::get(Context&: Ctx, V: Val);
350 } else if (auto *FpExt = dyn_cast<FPExtInst>(Val: Arg)) {
351 if (FpExt->getType()->isDoubleTy() &&
352 FpExt->getOperand(i_nocapture: 0)->getType()->isFloatTy()) {
353 Arg = FpExt->getOperand(i_nocapture: 0);
354 }
355 }
356 }
357 WhatToStore.push_back(Elt: Arg);
358 } else if (isa<PointerType>(Val: ArgType)) {
359 if (shouldPrintAsStr(Specifier: OpConvSpecifiers[ArgCount - 1], OpType: ArgType)) {
360 StringRef S = getAsConstantStr(V: Arg);
361 if (!S.empty()) {
362 const uint64_t ReadSize = 4;
363
364 DataExtractor Extractor(S, /*IsLittleEndian=*/true, 8);
365 DataExtractor::Cursor Offset(0);
366 while (Offset && Offset.tell() < S.size()) {
367 uint64_t ReadNow = std::min(a: ReadSize, b: S.size() - Offset.tell());
368 uint64_t ReadBytes = 0;
369 switch (ReadNow) {
370 default: llvm_unreachable("min(4, X) > 4?");
371 case 1:
372 ReadBytes = Extractor.getU8(C&: Offset);
373 break;
374 case 2:
375 ReadBytes = Extractor.getU16(C&: Offset);
376 break;
377 case 3:
378 ReadBytes = Extractor.getU24(C&: Offset);
379 break;
380 case 4:
381 ReadBytes = Extractor.getU32(C&: Offset);
382 break;
383 }
384
385 cantFail(Err: Offset.takeError(),
386 Msg: "failed to read bytes from constant array");
387
388 APInt IntVal(8 * ReadSize, ReadBytes);
389
390 // TODO: Should not bothering aligning up.
391 if (ReadNow < ReadSize)
392 IntVal = IntVal.zext(width: 8 * ReadSize);
393
394 Type *IntTy = Type::getIntNTy(C&: Ctx, N: IntVal.getBitWidth());
395 WhatToStore.push_back(Elt: ConstantInt::get(Ty: IntTy, V: IntVal));
396 }
397 } else {
398 // Empty string, give a hint to RT it is no NULL
399 Value *ANumV = ConstantInt::get(Ty: Int32Ty, V: 0xFFFFFF00, IsSigned: false);
400 WhatToStore.push_back(Elt: ANumV);
401 }
402 } else {
403 WhatToStore.push_back(Elt: Arg);
404 }
405 } else {
406 WhatToStore.push_back(Elt: Arg);
407 }
408 for (unsigned I = 0, E = WhatToStore.size(); I != E; ++I) {
409 Value *TheBtCast = WhatToStore[I];
410 unsigned ArgSize = TD->getTypeAllocSize(Ty: TheBtCast->getType());
411 StoreInst *StBuff = new StoreInst(TheBtCast, BufferIdx, BrnchPoint);
412 LLVM_DEBUG(dbgs() << "inserting store to printf buffer:\n"
413 << *StBuff << '\n');
414 (void)StBuff;
415 if (I + 1 == E && ArgCount + 1 == CI->arg_size())
416 break;
417 BufferIdx = GetElementPtrInst::Create(
418 PointeeType: I8Ty, Ptr: BufferIdx, IdxList: {ConstantInt::get(Ty: I32Ty, V: ArgSize)},
419 NameStr: "PrintBuffNextPtr", InsertBefore: BrnchPoint);
420 LLVM_DEBUG(dbgs() << "inserting gep to the printf buffer:\n"
421 << *BufferIdx << '\n');
422 }
423 }
424 }
425
426 // erase the printf calls
427 for (auto *CI : Printfs)
428 CI->eraseFromParent();
429
430 Printfs.clear();
431 return true;
432}
433
434bool AMDGPUPrintfRuntimeBindingImpl::run(Module &M) {
435 Triple TT(M.getTargetTriple());
436 if (TT.getArch() == Triple::r600)
437 return false;
438
439 auto PrintfFunction = M.getFunction(Name: "printf");
440 if (!PrintfFunction || !PrintfFunction->isDeclaration())
441 return false;
442
443 for (auto &U : PrintfFunction->uses()) {
444 if (auto *CI = dyn_cast<CallInst>(Val: U.getUser())) {
445 if (CI->isCallee(U: &U) && !CI->isNoBuiltin())
446 Printfs.push_back(Elt: CI);
447 }
448 }
449
450 if (Printfs.empty())
451 return false;
452
453 TD = &M.getDataLayout();
454
455 return lowerPrintfForGpu(M);
456}
457
458bool AMDGPUPrintfRuntimeBinding::runOnModule(Module &M) {
459 return AMDGPUPrintfRuntimeBindingImpl().run(M);
460}
461
462PreservedAnalyses
463AMDGPUPrintfRuntimeBindingPass::run(Module &M, ModuleAnalysisManager &AM) {
464 bool Changed = AMDGPUPrintfRuntimeBindingImpl().run(M);
465 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
466}
467