1//=== AMDGPUPrintfRuntimeBinding.cpp - OpenCL printf implementation -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// \file
9//
10// The pass bind printfs to a kernel arg pointer that will be bound to a buffer
11// later by the runtime.
12//
13// This pass traverses the functions in the module and converts
14// each call to printf to a sequence of operations that
15// store the following into the printf buffer:
16// - format string (passed as a module's metadata unique ID)
17// - bitwise copies of printf arguments
18// The backend passes will need to store metadata in the kernel
19//===----------------------------------------------------------------------===//
20
21#include "AMDGPU.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/IR/DiagnosticInfo.h"
25#include "llvm/IR/Dominators.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/Instructions.h"
28#include "llvm/IR/Module.h"
29#include "llvm/InitializePasses.h"
30#include "llvm/Support/DataExtractor.h"
31#include "llvm/TargetParser/Triple.h"
32#include "llvm/Transforms/Utils/BasicBlockUtils.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "printfToRuntime"
37enum { DWORD_ALIGN = 4 };
38
39namespace {
40class AMDGPUPrintfRuntimeBinding final : public ModulePass {
41
42public:
43 static char ID;
44
45 explicit AMDGPUPrintfRuntimeBinding() : ModulePass(ID) {}
46
47private:
48 bool runOnModule(Module &M) override;
49};
50
51class AMDGPUPrintfRuntimeBindingImpl {
52public:
53 AMDGPUPrintfRuntimeBindingImpl() = default;
54 bool run(Module &M);
55
56private:
57 void getConversionSpecifiers(SmallVectorImpl<char> &OpConvSpecifiers,
58 StringRef fmt, size_t num_ops) const;
59
60 bool lowerPrintfForGpu(Module &M);
61
62 const DataLayout *TD;
63 SmallVector<CallInst *, 32> Printfs;
64};
65} // namespace
66
67char AMDGPUPrintfRuntimeBinding::ID = 0;
68
69INITIALIZE_PASS_BEGIN(AMDGPUPrintfRuntimeBinding,
70 "amdgpu-printf-runtime-binding", "AMDGPU Printf lowering",
71 false, false)
72INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
73INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
74INITIALIZE_PASS_END(AMDGPUPrintfRuntimeBinding, "amdgpu-printf-runtime-binding",
75 "AMDGPU Printf lowering", false, false)
76
77char &llvm::AMDGPUPrintfRuntimeBindingID = AMDGPUPrintfRuntimeBinding::ID;
78
79ModulePass *llvm::createAMDGPUPrintfRuntimeBinding() {
80 return new AMDGPUPrintfRuntimeBinding();
81}
82
83void AMDGPUPrintfRuntimeBindingImpl::getConversionSpecifiers(
84 SmallVectorImpl<char> &OpConvSpecifiers, StringRef Fmt,
85 size_t NumOps) const {
86 // not all format characters are collected.
87 // At this time the format characters of interest
88 // are %p and %s, which use to know if we
89 // are either storing a literal string or a
90 // pointer to the printf buffer.
91 static const char ConvSpecifiers[] = "cdieEfFgGaAosuxXp";
92 size_t CurFmtSpecifierIdx = 0;
93 size_t PrevFmtSpecifierIdx = 0;
94
95 while ((CurFmtSpecifierIdx = Fmt.find_first_of(
96 Chars: ConvSpecifiers, From: CurFmtSpecifierIdx)) != StringRef::npos) {
97 bool ArgDump = false;
98 StringRef CurFmt = Fmt.substr(Start: PrevFmtSpecifierIdx,
99 N: CurFmtSpecifierIdx - PrevFmtSpecifierIdx);
100 size_t pTag = CurFmt.find_last_of(C: '%');
101 if (pTag != StringRef::npos) {
102 ArgDump = true;
103 while (pTag && CurFmt[--pTag] == '%') {
104 ArgDump = !ArgDump;
105 }
106 }
107
108 if (ArgDump)
109 OpConvSpecifiers.push_back(Elt: Fmt[CurFmtSpecifierIdx]);
110
111 PrevFmtSpecifierIdx = ++CurFmtSpecifierIdx;
112 }
113}
114
115static bool shouldPrintAsStr(char Specifier, Type *OpType) {
116 return Specifier == 's' && isa<PointerType>(Val: OpType);
117}
118
119constexpr StringLiteral NonLiteralStr("???");
120static_assert(NonLiteralStr.size() == 3);
121
122static StringRef getAsConstantStr(Value *V) {
123 StringRef S;
124 if (!getConstantStringInfo(V, Str&: S))
125 S = NonLiteralStr;
126
127 return S;
128}
129
130static void diagnoseInvalidFormatString(const CallBase *CI) {
131 CI->getContext().diagnose(DI: DiagnosticInfoUnsupported(
132 *CI->getFunction(),
133 "printf format string must be a trivially resolved constant string "
134 "global variable",
135 CI->getDebugLoc()));
136}
137
138bool AMDGPUPrintfRuntimeBindingImpl::lowerPrintfForGpu(Module &M) {
139 LLVMContext &Ctx = M.getContext();
140 IRBuilder<> Builder(Ctx);
141 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
142
143 // Instead of creating global variables, the printf format strings are
144 // extracted and passed as metadata. This avoids polluting llvm's symbol
145 // tables in this module. Metadata is going to be extracted by the backend
146 // passes and inserted into the OpenCL binary as appropriate.
147 NamedMDNode *metaD = M.getOrInsertNamedMetadata(Name: "llvm.printf.fmts");
148 unsigned UniqID = metaD->getNumOperands();
149
150 for (auto *CI : Printfs) {
151 unsigned NumOps = CI->arg_size();
152
153 SmallString<16> OpConvSpecifiers;
154 Value *Op = CI->getArgOperand(i: 0);
155
156 StringRef FormatStr;
157 if (!getConstantStringInfo(V: Op, Str&: FormatStr)) {
158 Value *Stripped = Op->stripPointerCasts();
159 if (!isa<UndefValue>(Val: Stripped) && !isa<ConstantPointerNull>(Val: Stripped))
160 diagnoseInvalidFormatString(CI);
161 continue;
162 }
163
164 // We need this call to ascertain that we are printing a string or a
165 // pointer. It takes out the specifiers and fills up the first arg.
166 getConversionSpecifiers(OpConvSpecifiers, Fmt: FormatStr, NumOps: NumOps - 1);
167
168 // Add metadata for the string
169 std::string AStreamHolder;
170 raw_string_ostream Sizes(AStreamHolder);
171 int Sum = DWORD_ALIGN;
172 Sizes << CI->arg_size() - 1;
173 Sizes << ':';
174 for (unsigned ArgCount = 1;
175 ArgCount < CI->arg_size() && ArgCount <= OpConvSpecifiers.size();
176 ArgCount++) {
177 Value *Arg = CI->getArgOperand(i: ArgCount);
178 Type *ArgType = Arg->getType();
179 unsigned ArgSize = TD->getTypeAllocSize(Ty: ArgType);
180 //
181 // ArgSize by design should be a multiple of DWORD_ALIGN,
182 // expand the arguments that do not follow this rule.
183 //
184 if (ArgSize % DWORD_ALIGN != 0) {
185 Type *ResType = ArgType->getWithNewType(EltTy: Type::getInt32Ty(C&: Ctx));
186 Builder.SetInsertPoint(CI);
187 Builder.SetCurrentDebugLocation(CI->getDebugLoc());
188
189 if (ArgType->isFPOrFPVectorTy()) {
190 Arg = Builder.CreateFPExt(
191 V: Arg, DestTy: ArgType->getWithNewType(EltTy: Type::getFloatTy(C&: Ctx)));
192 } else if (OpConvSpecifiers[ArgCount - 1] == 'x' ||
193 OpConvSpecifiers[ArgCount - 1] == 'X' ||
194 OpConvSpecifiers[ArgCount - 1] == 'u' ||
195 OpConvSpecifiers[ArgCount - 1] == 'o')
196 Arg = Builder.CreateZExt(V: Arg, DestTy: ResType);
197 else
198 Arg = Builder.CreateSExt(V: Arg, DestTy: ResType);
199 ArgType = Arg->getType();
200 ArgSize = TD->getTypeAllocSize(Ty: ArgType);
201 CI->setOperand(i_nocapture: ArgCount, Val_nocapture: Arg);
202 }
203 if (OpConvSpecifiers[ArgCount - 1] == 'f') {
204 ConstantFP *FpCons = dyn_cast<ConstantFP>(Val: Arg);
205 if (FpCons)
206 ArgSize = 4;
207 else {
208 FPExtInst *FpExt = dyn_cast<FPExtInst>(Val: Arg);
209 if (FpExt && FpExt->getType()->isDoubleTy() &&
210 FpExt->getOperand(i_nocapture: 0)->getType()->isFloatTy())
211 ArgSize = 4;
212 }
213 }
214 if (shouldPrintAsStr(Specifier: OpConvSpecifiers[ArgCount - 1], OpType: ArgType))
215 ArgSize = alignTo(Value: getAsConstantStr(V: Arg).size() + 1, Align: 4);
216
217 LLVM_DEBUG(dbgs() << "Printf ArgSize (in buffer) = " << ArgSize
218 << " for type: " << *ArgType << '\n');
219 Sizes << ArgSize << ':';
220 Sum += ArgSize;
221 }
222 LLVM_DEBUG(dbgs() << "Printf format string in source = " << FormatStr
223 << '\n');
224 for (char C : FormatStr) {
225 // Rest of the C escape sequences (e.g. \') are handled correctly
226 // by the MDParser
227 switch (C) {
228 case '\a':
229 Sizes << "\\a";
230 break;
231 case '\b':
232 Sizes << "\\b";
233 break;
234 case '\f':
235 Sizes << "\\f";
236 break;
237 case '\n':
238 Sizes << "\\n";
239 break;
240 case '\r':
241 Sizes << "\\r";
242 break;
243 case '\v':
244 Sizes << "\\v";
245 break;
246 case ':':
247 // ':' cannot be scanned by Flex, as it is defined as a delimiter
248 // Replace it with it's octal representation \72
249 Sizes << "\\72";
250 break;
251 default:
252 Sizes << C;
253 break;
254 }
255 }
256
257 // Insert the printf_alloc call
258 Builder.SetInsertPoint(CI);
259 Builder.SetCurrentDebugLocation(CI->getDebugLoc());
260
261 AttributeList Attr = AttributeList::get(C&: Ctx, Index: AttributeList::FunctionIndex,
262 Kinds: Attribute::NoUnwind);
263
264 Type *SizetTy = Type::getInt32Ty(C&: Ctx);
265
266 Type *Tys_alloc[1] = {SizetTy};
267 Type *I8Ty = Type::getInt8Ty(C&: Ctx);
268 Type *I8Ptr = PointerType::get(C&: Ctx, AddressSpace: 1);
269 FunctionType *FTy_alloc = FunctionType::get(Result: I8Ptr, Params: Tys_alloc, isVarArg: false);
270 FunctionCallee PrintfAllocFn =
271 M.getOrInsertFunction(Name: StringRef("__printf_alloc"), T: FTy_alloc, AttributeList: Attr);
272
273 LLVM_DEBUG(dbgs() << "Printf metadata = " << Sizes.str() << '\n');
274 std::string fmtstr = itostr(X: ++UniqID) + ":" + Sizes.str();
275 MDString *fmtStrArray = MDString::get(Context&: Ctx, Str: fmtstr);
276
277 MDNode *myMD = MDNode::get(Context&: Ctx, MDs: fmtStrArray);
278 metaD->addOperand(M: myMD);
279 Value *sumC = ConstantInt::get(Ty: SizetTy, V: Sum, IsSigned: false);
280 SmallVector<Value *, 1> alloc_args;
281 alloc_args.push_back(Elt: sumC);
282 CallInst *pcall = CallInst::Create(Func: PrintfAllocFn, Args: alloc_args,
283 NameStr: "printf_alloc_fn", InsertBefore: CI->getIterator());
284
285 //
286 // Insert code to split basicblock with a
287 // piece of hammock code.
288 // basicblock splits after buffer overflow check
289 //
290 ConstantPointerNull *zeroIntPtr =
291 ConstantPointerNull::get(T: PointerType::get(C&: Ctx, AddressSpace: 1));
292 auto *cmp = cast<ICmpInst>(Val: Builder.CreateICmpNE(LHS: pcall, RHS: zeroIntPtr, Name: ""));
293 if (!CI->use_empty()) {
294 Value *result =
295 Builder.CreateSExt(V: Builder.CreateNot(V: cmp), DestTy: I32Ty, Name: "printf_res");
296 CI->replaceAllUsesWith(V: result);
297 }
298 SplitBlock(Old: CI->getParent(), SplitPt: cmp);
299 Instruction *Brnch =
300 SplitBlockAndInsertIfThen(Cond: cmp, SplitBefore: cmp->getNextNode(), Unreachable: false);
301 BasicBlock::iterator BrnchPoint = Brnch->getIterator();
302
303 Builder.SetInsertPoint(Brnch);
304
305 // store unique printf id in the buffer
306 //
307 GetElementPtrInst *BufferIdx = GetElementPtrInst::Create(
308 PointeeType: I8Ty, Ptr: pcall, IdxList: ConstantInt::get(Context&: Ctx, V: APInt(32, 0)), NameStr: "PrintBuffID",
309 InsertBefore: BrnchPoint);
310
311 Type *idPointer = PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::GLOBAL_ADDRESS);
312 Value *id_gep_cast =
313 new BitCastInst(BufferIdx, idPointer, "PrintBuffIdCast", BrnchPoint);
314
315 new StoreInst(ConstantInt::get(Ty: I32Ty, V: UniqID), id_gep_cast, BrnchPoint);
316
317 // 1st 4 bytes hold the printf_id
318 // the following GEP is the buffer pointer
319 BufferIdx = GetElementPtrInst::Create(PointeeType: I8Ty, Ptr: pcall,
320 IdxList: ConstantInt::get(Context&: Ctx, V: APInt(32, 4)),
321 NameStr: "PrintBuffGep", InsertBefore: BrnchPoint);
322
323 Type *Int32Ty = Type::getInt32Ty(C&: Ctx);
324 for (unsigned ArgCount = 1;
325 ArgCount < CI->arg_size() && ArgCount <= OpConvSpecifiers.size();
326 ArgCount++) {
327 Value *Arg = CI->getArgOperand(i: ArgCount);
328 Type *ArgType = Arg->getType();
329 SmallVector<Value *, 32> WhatToStore;
330 if (ArgType->isFPOrFPVectorTy() && !isa<VectorType>(Val: ArgType)) {
331 if (OpConvSpecifiers[ArgCount - 1] == 'f') {
332 if (auto *FpCons = dyn_cast<ConstantFP>(Val: Arg)) {
333 APFloat Val(FpCons->getValueAPF());
334 bool Lost = false;
335 Val.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
336 losesInfo: &Lost);
337 Arg = ConstantFP::get(Context&: Ctx, V: Val);
338 } else if (auto *FpExt = dyn_cast<FPExtInst>(Val: Arg)) {
339 if (FpExt->getType()->isDoubleTy() &&
340 FpExt->getOperand(i_nocapture: 0)->getType()->isFloatTy()) {
341 Arg = FpExt->getOperand(i_nocapture: 0);
342 }
343 }
344 }
345 WhatToStore.push_back(Elt: Arg);
346 } else if (isa<PointerType>(Val: ArgType)) {
347 if (shouldPrintAsStr(Specifier: OpConvSpecifiers[ArgCount - 1], OpType: ArgType)) {
348 StringRef S = getAsConstantStr(V: Arg);
349 if (!S.empty()) {
350 const uint64_t ReadSize = 4;
351
352 DataExtractor Extractor(S, /*IsLittleEndian=*/true);
353 DataExtractor::Cursor Offset(0);
354 while (Offset && Offset.tell() < S.size()) {
355 uint64_t ReadNow = std::min(a: ReadSize, b: S.size() - Offset.tell());
356 uint64_t ReadBytes = 0;
357 switch (ReadNow) {
358 default: llvm_unreachable("min(4, X) > 4?");
359 case 1:
360 ReadBytes = Extractor.getU8(C&: Offset);
361 break;
362 case 2:
363 ReadBytes = Extractor.getU16(C&: Offset);
364 break;
365 case 3:
366 ReadBytes = Extractor.getU24(C&: Offset);
367 break;
368 case 4:
369 ReadBytes = Extractor.getU32(C&: Offset);
370 break;
371 }
372
373 cantFail(Err: Offset.takeError(),
374 Msg: "failed to read bytes from constant array");
375
376 APInt IntVal(8 * ReadSize, ReadBytes);
377
378 // TODO: Should not bothering aligning up.
379 if (ReadNow < ReadSize)
380 IntVal = IntVal.zext(width: 8 * ReadSize);
381
382 Type *IntTy = Type::getIntNTy(C&: Ctx, N: IntVal.getBitWidth());
383 WhatToStore.push_back(Elt: ConstantInt::get(Ty: IntTy, V: IntVal));
384 }
385 } else {
386 // Empty string, give a hint to RT it is no NULL
387 Value *ANumV = ConstantInt::get(Ty: Int32Ty, V: 0xFFFFFF00, IsSigned: false);
388 WhatToStore.push_back(Elt: ANumV);
389 }
390 } else {
391 WhatToStore.push_back(Elt: Arg);
392 }
393 } else {
394 WhatToStore.push_back(Elt: Arg);
395 }
396 for (unsigned I = 0, E = WhatToStore.size(); I != E; ++I) {
397 Value *TheBtCast = WhatToStore[I];
398 unsigned ArgSize = TD->getTypeAllocSize(Ty: TheBtCast->getType());
399 StoreInst *StBuff = new StoreInst(TheBtCast, BufferIdx, BrnchPoint);
400 LLVM_DEBUG(dbgs() << "inserting store to printf buffer:\n"
401 << *StBuff << '\n');
402 (void)StBuff;
403 if (I + 1 == E && ArgCount + 1 == CI->arg_size())
404 break;
405 BufferIdx = GetElementPtrInst::Create(
406 PointeeType: I8Ty, Ptr: BufferIdx, IdxList: {ConstantInt::get(Ty: I32Ty, V: ArgSize)},
407 NameStr: "PrintBuffNextPtr", InsertBefore: BrnchPoint);
408 LLVM_DEBUG(dbgs() << "inserting gep to the printf buffer:\n"
409 << *BufferIdx << '\n');
410 }
411 }
412 }
413
414 // Erase the printf calls and replace all uses with 0, signaling success.
415 // Since OpenCL only specifies undefined behaviors and not success criteria,
416 // returning 0 sinalling success always is valid.
417 for (auto *CI : Printfs) {
418 CI->replaceAllUsesWith(V: ConstantInt::get(Ty: CI->getType(), V: 0));
419 CI->eraseFromParent();
420 }
421
422 Printfs.clear();
423 return true;
424}
425
426bool AMDGPUPrintfRuntimeBindingImpl::run(Module &M) {
427 auto *PrintfFunction = M.getFunction(Name: "printf");
428 if (!PrintfFunction || !PrintfFunction->isDeclaration() ||
429 M.getModuleFlag(Key: "openmp"))
430 return false;
431
432 // Verify the signature of the printf function and skip if it isn't correct.
433 const FunctionType *PrintfFunctionTy = PrintfFunction->getFunctionType();
434 if (PrintfFunctionTy->getNumParams() != 1 || !PrintfFunctionTy->isVarArg() ||
435 !PrintfFunctionTy->getReturnType()->isIntegerTy(BitWidth: 32))
436 return false;
437 Type *PrintfFormatArgTy = PrintfFunctionTy->getParamType(i: 0);
438 if (!PrintfFormatArgTy->isPointerTy() ||
439 !AMDGPU::isFlatGlobalAddrSpace(
440 AS: PrintfFormatArgTy->getPointerAddressSpace()))
441 return false;
442
443 for (auto &U : PrintfFunction->uses()) {
444 if (auto *CI = dyn_cast<CallInst>(Val: U.getUser())) {
445 if (CI->isCallee(U: &U) && !CI->isNoBuiltin())
446 Printfs.push_back(Elt: CI);
447 }
448 }
449
450 if (Printfs.empty())
451 return false;
452
453 TD = &M.getDataLayout();
454
455 return lowerPrintfForGpu(M);
456}
457
458bool AMDGPUPrintfRuntimeBinding::runOnModule(Module &M) {
459 return AMDGPUPrintfRuntimeBindingImpl().run(M);
460}
461
462PreservedAnalyses
463AMDGPUPrintfRuntimeBindingPass::run(Module &M, ModuleAnalysisManager &AM) {
464 bool Changed = AMDGPUPrintfRuntimeBindingImpl().run(M);
465 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
466}
467