1//===-- ExpandVariadicsPass.cpp --------------------------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is an optimization pass for variadic functions. If called from codegen,
10// it can serve as the implementation of variadic functions for a given target.
11//
12// The strategy is to turn the ... part of a variadic function into a va_list
13// and fix up the call sites. The majority of the pass is target independent.
14// The exceptions are the va_list type itself and the rules for where to store
15// variables in memory such that va_arg can iterate over them given a va_list.
16//
17// The majority of the plumbing is splitting the variadic function into a
18// single basic block that packs the variadic arguments into a va_list and
19// a second function that does the work of the original. That packing is
20// exactly what is done by va_start. Further, the transform from ... to va_list
21// replaced va_start with an operation to copy a va_list from the new argument,
22// which is exactly a va_copy. This is useful for reducing target-dependence.
23//
24// A va_list instance is a forward iterator, where the primary operation va_arg
25// is dereference-then-increment. This interface forces significant convergent
26// evolution between target specific implementations. The variation in runtime
27// data layout is limited to that representable by the iterator, parameterised
28// by the type passed to the va_arg instruction.
29//
30// Therefore the majority of the target specific subtlety is packing arguments
31// into a stack allocated buffer such that a va_list can be initialised with it
32// and the va_arg expansion for the target will find the arguments at runtime.
33//
34// The aggregate effect is to unblock other transforms, most critically the
35// general purpose inliner. Known calls to variadic functions become zero cost.
36//
37// Consistency with clang is primarily tested by emitting va_arg using clang
38// then expanding the variadic functions using this pass, followed by trying
39// to constant fold the functions to no-ops.
40//
41// Target specific behaviour is tested in IR - mainly checking that values are
42// put into positions in call frames that make sense for that particular target.
43//
44// There is one "clever" invariant in use. va_start intrinsics that are not
45// within a varidic functions are an error in the IR verifier. When this
46// transform moves blocks from a variadic function into a fixed arity one, it
47// moves va_start intrinsics along with everything else. That means that the
48// va_start intrinsics that need to be rewritten to use the trailing argument
49// are exactly those that are in non-variadic functions so no further state
50// is needed to distinguish those that need to be rewritten.
51//
52//===----------------------------------------------------------------------===//
53
54#include "llvm/Transforms/IPO/ExpandVariadics.h"
55#include "IPOOptions.h"
56#include "llvm/ADT/Sequence.h"
57#include "llvm/ADT/SmallVector.h"
58#include "llvm/Demangle/Demangle.h"
59#include "llvm/IR/IRBuilder.h"
60#include "llvm/IR/IntrinsicInst.h"
61#include "llvm/IR/Module.h"
62#include "llvm/IR/PassManager.h"
63#include "llvm/InitializePasses.h"
64#include "llvm/Pass.h"
65#include "llvm/Support/NVPTXAddrSpace.h"
66#include "llvm/TargetParser/Triple.h"
67#include "llvm/Transforms/Utils/ModuleUtils.h"
68
69#define DEBUG_TYPE "expand-variadics"
70
71using namespace llvm;
72
73namespace {
74
75// Instances of this class encapsulate the target-dependant behaviour as a
76// function of triple. Implementing a new ABI is adding a case to the switch
77// in create(llvm::Triple) at the end of this file.
78// This class may end up instantiated in TargetMachine instances, keeping it
79// here for now until enough targets are implemented for the API to evolve.
80class VariadicABIInfo {
81protected:
82 VariadicABIInfo() = default;
83
84public:
85 static std::unique_ptr<VariadicABIInfo> create(const Triple &T);
86
87 // Allow overriding whether the pass runs on a per-target basis
88 virtual bool enableForTarget() = 0;
89
90 // Whether a valist instance is passed by value or by address
91 // I.e. does it need to be alloca'ed and stored into, or can
92 // it be passed directly in a SSA register
93 virtual bool vaListPassedInSSARegister() = 0;
94
95 // The type of a va_list iterator object
96 virtual Type *vaListType(LLVMContext &Ctx) = 0;
97
98 // The type of a va_list as a function argument as lowered by C
99 virtual Type *vaListParameterType(Module &M) = 0;
100
101 // Initialize an allocated va_list object to point to an already
102 // initialized contiguous memory region.
103 // Return the value to pass as the va_list argument
104 virtual Value *initializeVaList(Module &M, LLVMContext &Ctx,
105 IRBuilder<> &Builder, AllocaInst *VaList,
106 Value *Buffer) = 0;
107
108 struct VAArgSlotInfo {
109 Align DataAlign; // With respect to the call frame
110 bool Indirect; // Passed via a pointer
111 };
112 virtual VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) = 0;
113
114 // Targets implemented so far all have the same trivial lowering for these
115 bool vaEndIsNop() { return true; }
116 bool vaCopyIsMemcpy() { return true; }
117
118 // Per-target overrides of special symbols.
119 virtual bool ignoreFunction(const Function *F) { return false; }
120
121 // Any additional address spaces used in va intrinsics that should be
122 // expanded.
123 virtual SmallVector<unsigned> getTargetSpecificVaIntrinAddrSpaces() const {
124 return {};
125 }
126
127 virtual ~VariadicABIInfo() = default;
128};
129
130class ExpandVariadics : public ModulePass {
131
132 // The pass construction sets the default to optimize when called from middle
133 // end and lowering when called from the backend. The command line variable
134 // overrides that. This is useful for testing and debugging. It also allows
135 // building an applications with variadic functions wholly removed if one
136 // has sufficient control over the dependencies, e.g. a statically linked
137 // clang that has no variadic function calls remaining in the binary.
138
139public:
140 static char ID;
141 ExpandVariadicsMode Mode;
142 std::unique_ptr<VariadicABIInfo> ABI;
143
144 ExpandVariadics(ExpandVariadicsMode Mode) : ModulePass(ID), Mode(Mode) {}
145
146 StringRef getPassName() const override { return "Expand variadic functions"; }
147
148 bool rewriteABI() { return Mode == ExpandVariadicsMode::Lowering; }
149
150 template <typename T> bool isValidCallingConv(T *F) {
151 return F->getCallingConv() == CallingConv::C ||
152 F->getCallingConv() == CallingConv::SPIR_FUNC;
153 }
154
155 bool runOnModule(Module &M) override;
156
157 bool runOnFunction(Module &M, IRBuilder<> &Builder, Function *F);
158
159 Function *replaceAllUsesWithNewDeclaration(Module &M,
160 Function *OriginalFunction);
161
162 Function *deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder,
163 Function *OriginalFunction);
164
165 Function *defineVariadicWrapper(Module &M, IRBuilder<> &Builder,
166 Function *VariadicWrapper,
167 Function *FixedArityReplacement);
168
169 bool expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB, FunctionType *,
170 Function *NF);
171
172 // The intrinsic functions va_copy and va_end are removed unconditionally.
173 // They correspond to a memcpy and a no-op on all implemented targets.
174 // The va_start intrinsic is removed from basic blocks that were not created
175 // by this pass, some may remain if needed to maintain the external ABI.
176
177 template <Intrinsic::ID ID, typename InstructionType>
178 bool expandIntrinsicUsers(Module &M, IRBuilder<> &Builder,
179 PointerType *IntrinsicArgType) {
180 bool Changed = false;
181 const DataLayout &DL = M.getDataLayout();
182 if (Function *Intrinsic =
183 Intrinsic::getDeclarationIfExists(M: &M, id: ID, OverloadTys: {IntrinsicArgType})) {
184 for (User *U : make_early_inc_range(Range: Intrinsic->users()))
185 if (auto *I = dyn_cast<InstructionType>(U))
186 Changed |= expandVAIntrinsicCall(Builder, DL, I);
187
188 if (Intrinsic->use_empty())
189 Intrinsic->eraseFromParent();
190 }
191 return Changed;
192 }
193
194 bool expandVAIntrinsicUsersWithAddrspace(Module &M, IRBuilder<> &Builder,
195 unsigned Addrspace) {
196 auto &Ctx = M.getContext();
197 PointerType *IntrinsicArgType = PointerType::get(C&: Ctx, AddressSpace: Addrspace);
198 bool Changed = false;
199
200 // expand vastart before vacopy as vastart may introduce a vacopy
201 Changed |= expandIntrinsicUsers<Intrinsic::vastart, VAStartInst>(
202 M, Builder, IntrinsicArgType);
203 Changed |= expandIntrinsicUsers<Intrinsic::vaend, VAEndInst>(
204 M, Builder, IntrinsicArgType);
205 Changed |= expandIntrinsicUsers<Intrinsic::vacopy, VACopyInst>(
206 M, Builder, IntrinsicArgType);
207 return Changed;
208 }
209
210 bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL,
211 VAStartInst *Inst);
212
213 bool expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &,
214 VAEndInst *Inst);
215
216 bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL,
217 VACopyInst *Inst);
218
219 bool expandVAArgInst(IRBuilder<> &Builder, const DataLayout &DL,
220 VAArgInst *Inst);
221
222 FunctionType *inlinableVariadicFunctionType(Module &M, FunctionType *FTy,
223 Type *ReturnType) {
224 // The type of "FTy" with the ... removed and a va_list appended
225 SmallVector<Type *> ArgTypes(FTy->params());
226 ArgTypes.push_back(Elt: ABI->vaListParameterType(M));
227 return FunctionType::get(Result: ReturnType, Params: ArgTypes, /*IsVarArgs=*/isVarArg: false);
228 }
229
230 FunctionType *inlinableVariadicFunctionType(Module &M, FunctionType *FTy) {
231 return inlinableVariadicFunctionType(M, FTy, ReturnType: FTy->getReturnType());
232 }
233
234 bool expansionApplicableToFunction(Module &M, Function *F) {
235 if (F->isIntrinsic() || !F->isVarArg() ||
236 F->hasFnAttribute(Kind: Attribute::Naked))
237 return false;
238
239 if (ABI->ignoreFunction(F))
240 return false;
241
242 if (!isValidCallingConv(F))
243 return false;
244
245 if (rewriteABI())
246 return true;
247
248 if (!F->hasExactDefinition())
249 return false;
250
251 return true;
252 }
253
254 bool expansionApplicableToFunctionCall(CallBase *CB) {
255 if (CallInst *CI = dyn_cast<CallInst>(Val: CB)) {
256 if (CI->isMustTailCall()) {
257 // Cannot expand musttail calls
258 return false;
259 }
260
261 if (!isValidCallingConv(F: CI))
262 return false;
263
264 return true;
265 }
266
267 if (isa<InvokeInst>(Val: CB)) {
268 // Invoke not implemented in initial implementation of pass
269 return false;
270 }
271
272 // Other unimplemented derivative of CallBase
273 return false;
274 }
275
276 class ExpandedCallFrame {
277 // Helper for constructing an alloca instance containing the arguments bound
278 // to the variadic ... parameter, rearranged to allow indexing through a
279 // va_list iterator
280 enum { N = 4 };
281 SmallVector<Type *, N> FieldTypes;
282 enum Tag { Store, Memcpy, Padding };
283 SmallVector<std::tuple<Value *, uint64_t, Tag>, N> Source;
284
285 template <Tag tag> void append(Type *FieldType, Value *V, uint64_t Bytes) {
286 FieldTypes.push_back(Elt: FieldType);
287 Source.push_back(Elt: {V, Bytes, tag});
288 }
289
290 public:
291 void store(LLVMContext &Ctx, Type *T, Value *V) { append<Store>(FieldType: T, V, Bytes: 0); }
292
293 void memcpy(LLVMContext &Ctx, Type *T, Value *V, uint64_t Bytes) {
294 append<Memcpy>(FieldType: T, V, Bytes);
295 }
296
297 void padding(LLVMContext &Ctx, uint64_t By) {
298 append<Padding>(FieldType: ArrayType::get(ElementType: Type::getInt8Ty(C&: Ctx), NumElements: By), V: nullptr, Bytes: 0);
299 }
300
301 size_t size() const { return FieldTypes.size(); }
302 bool empty() const { return FieldTypes.empty(); }
303
304 StructType *asStruct(LLVMContext &Ctx, StringRef Name) {
305 const bool IsPacked = true;
306 return StructType::create(Context&: Ctx, Elements: FieldTypes,
307 Name: (Twine(Name) + ".vararg").str(), isPacked: IsPacked);
308 }
309
310 void initializeStructAlloca(const DataLayout &DL, IRBuilder<> &Builder,
311 AllocaInst *Alloced, StructType *VarargsTy) {
312
313 for (size_t I = 0; I < size(); I++) {
314
315 auto [V, bytes, tag] = Source[I];
316
317 if (tag == Padding) {
318 assert(V == nullptr);
319 continue;
320 }
321
322 auto Dst = Builder.CreateStructGEP(Ty: VarargsTy, Ptr: Alloced, Idx: I);
323
324 assert(V != nullptr);
325
326 if (tag == Store)
327 Builder.CreateStore(Val: V, Ptr: Dst);
328
329 if (tag == Memcpy)
330 Builder.CreateMemCpy(Dst, DstAlign: {}, Src: V, SrcAlign: {}, Size: bytes);
331 }
332 }
333 };
334};
335
336bool ExpandVariadics::runOnModule(Module &M) {
337 ExpandVariadicsMode Override = getIPOOptions(M).expand_variadics_override;
338 bool HasOverride = Override != ExpandVariadicsMode::Unspecified;
339 if (HasOverride)
340 Mode = Override;
341
342 bool Changed = false;
343 if (Mode == ExpandVariadicsMode::Disable)
344 return Changed;
345
346 Triple TT(M.getTargetTriple());
347 ABI = VariadicABIInfo::create(T: TT);
348 if (!ABI)
349 return Changed;
350
351 if (!HasOverride && !ABI->enableForTarget())
352 return Changed;
353
354 const DataLayout &DL = M.getDataLayout();
355 IRBuilder<> Builder(M);
356 Builder.disableGEPCanonicalization();
357
358 // Lowering needs to run on all functions exactly once.
359 // Optimize could run on functions containing va_start exactly once.
360 for (Function &F : make_early_inc_range(Range&: M))
361 Changed |= runOnFunction(M, Builder, F: &F);
362
363 // After runOnFunction, all known calls to known variadic functions have been
364 // replaced. va_start intrinsics are presently (and invalidly!) only present
365 // in functions that used to be variadic and have now been replaced to take a
366 // va_list instead. If lowering as opposed to optimising, calls to unknown
367 // variadic functions have also been replaced.
368
369 {
370 unsigned Addrspace = 0;
371 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);
372
373 Addrspace = DL.getAllocaAddrSpace();
374 if (Addrspace != 0)
375 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);
376
377 // Process any addrspaces targets declare to be important.
378 const SmallVector<unsigned> &TargetASVec =
379 ABI->getTargetSpecificVaIntrinAddrSpaces();
380 for (unsigned TargetAS : TargetASVec) {
381 if (TargetAS == 0 || TargetAS == DL.getAllocaAddrSpace())
382 continue;
383 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace: TargetAS);
384 }
385 }
386
387 if (Mode != ExpandVariadicsMode::Lowering)
388 return Changed;
389
390 for (Function &F : make_early_inc_range(Range&: M)) {
391 if (F.isDeclaration())
392 continue;
393
394 // Now need to track down indirect calls and va_arg instructions. Can't find
395 // those by walking uses of variadic functions, need to crawl the
396 // instruction stream. Fortunately this is only necessary for the ABI
397 // rewrite case.
398 for (BasicBlock &BB : F) {
399 for (Instruction &I : make_early_inc_range(Range&: BB)) {
400 if (auto *VA = dyn_cast<VAArgInst>(Val: &I)) {
401 Changed |= expandVAArgInst(Builder, DL, Inst: VA);
402 } else if (CallBase *CB = dyn_cast<CallBase>(Val: &I)) {
403 if (CB->isIndirectCall()) {
404 FunctionType *FTy = CB->getFunctionType();
405 if (FTy->isVarArg())
406 Changed |= expandCall(M, Builder, CB, FTy, /*NF=*/nullptr);
407 }
408 }
409 }
410 }
411 }
412
413 return Changed;
414}
415
416bool ExpandVariadics::runOnFunction(Module &M, IRBuilder<> &Builder,
417 Function *OriginalFunction) {
418 bool Changed = false;
419
420 if (!expansionApplicableToFunction(M, F: OriginalFunction))
421 return Changed;
422
423 [[maybe_unused]] const bool OriginalFunctionIsDeclaration =
424 OriginalFunction->isDeclaration();
425 assert(rewriteABI() || !OriginalFunctionIsDeclaration);
426
427 // Declare a new function and redirect every use to that new function
428 Function *VariadicWrapper =
429 replaceAllUsesWithNewDeclaration(M, OriginalFunction);
430 assert(VariadicWrapper->isDeclaration());
431 assert(OriginalFunction->use_empty());
432
433 // Create a new function taking va_list containing the implementation of the
434 // original
435 Function *FixedArityReplacement =
436 deriveFixedArityReplacement(M, Builder, OriginalFunction);
437 assert(OriginalFunction->isDeclaration());
438 assert(FixedArityReplacement->isDeclaration() ==
439 OriginalFunctionIsDeclaration);
440 assert(VariadicWrapper->isDeclaration());
441
442 // Create a single block forwarding wrapper that turns a ... into a va_list
443 [[maybe_unused]] Function *VariadicWrapperDefine =
444 defineVariadicWrapper(M, Builder, VariadicWrapper, FixedArityReplacement);
445 assert(VariadicWrapperDefine == VariadicWrapper);
446 assert(!VariadicWrapper->isDeclaration());
447
448 // Add the prof metadata from the original function to the wrapper. Because
449 // FixedArityReplacement is the owner of original function's prof metadata
450 // after the splice, we need to transfer it to VariadicWrapper.
451 VariadicWrapper->setMetadata(
452 KindID: LLVMContext::MD_prof,
453 Node: FixedArityReplacement->getMetadata(KindID: LLVMContext::MD_prof));
454
455 // We now have:
456 // 1. the original function, now as a declaration with no uses
457 // 2. a variadic function that unconditionally calls a fixed arity replacement
458 // 3. a fixed arity function equivalent to the original function
459
460 // Replace known calls to the variadic with calls to the va_list equivalent
461 for (User *U : make_early_inc_range(Range: VariadicWrapper->users())) {
462 if (CallBase *CB = dyn_cast<CallBase>(Val: U)) {
463 Value *CalledOperand = CB->getCalledOperand();
464 if (VariadicWrapper == CalledOperand)
465 Changed |=
466 expandCall(M, Builder, CB, VariadicWrapper->getFunctionType(),
467 NF: FixedArityReplacement);
468 }
469 }
470
471 // The original function will be erased.
472 // One of the two new functions will become a replacement for the original.
473 // When preserving the ABI, the other is an internal implementation detail.
474 // When rewriting the ABI, RAUW then the variadic one.
475 Function *const ExternallyAccessible =
476 rewriteABI() ? FixedArityReplacement : VariadicWrapper;
477 Function *const InternalOnly =
478 rewriteABI() ? VariadicWrapper : FixedArityReplacement;
479
480 // The external function is the replacement for the original
481 ExternallyAccessible->setLinkage(OriginalFunction->getLinkage());
482 ExternallyAccessible->setVisibility(OriginalFunction->getVisibility());
483 ExternallyAccessible->setComdat(OriginalFunction->getComdat());
484 ExternallyAccessible->takeName(V: OriginalFunction);
485
486 // Annotate the internal one as internal
487 InternalOnly->setVisibility(GlobalValue::DefaultVisibility);
488 InternalOnly->setLinkage(GlobalValue::InternalLinkage);
489
490 // The original is unused and obsolete
491 OriginalFunction->eraseFromParent();
492
493 InternalOnly->removeDeadConstantUsers();
494
495 if (rewriteABI()) {
496 // All known calls to the function have been removed by expandCall
497 // Resolve everything else by replaceAllUsesWith
498 VariadicWrapper->replaceAllUsesWith(V: FixedArityReplacement);
499 VariadicWrapper->eraseFromParent();
500 }
501
502 return Changed;
503}
504
505Function *
506ExpandVariadics::replaceAllUsesWithNewDeclaration(Module &M,
507 Function *OriginalFunction) {
508 auto &Ctx = M.getContext();
509 Function &F = *OriginalFunction;
510 FunctionType *FTy = F.getFunctionType();
511 Function *NF = Function::Create(Ty: FTy, Linkage: F.getLinkage(), AddrSpace: F.getAddressSpace());
512
513 NF->setName(F.getName() + ".varargs");
514
515 F.getParent()->getFunctionList().insert(where: F.getIterator(), New: NF);
516
517 AttrBuilder ParamAttrs(Ctx);
518 AttributeList Attrs = NF->getAttributes();
519 Attrs = Attrs.addParamAttributes(C&: Ctx, ArgNo: FTy->getNumParams(), B: ParamAttrs);
520 NF->setAttributes(Attrs);
521
522 OriginalFunction->replaceAllUsesWith(V: NF);
523 return NF;
524}
525
526Function *
527ExpandVariadics::deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder,
528 Function *OriginalFunction) {
529 Function &F = *OriginalFunction;
530 // The purpose here is split the variadic function F into two functions
531 // One is a variadic function that bundles the passed argument into a va_list
532 // and passes it to the second function. The second function does whatever
533 // the original F does, except that it takes a va_list instead of the ...
534
535 assert(expansionApplicableToFunction(M, &F));
536
537 auto &Ctx = M.getContext();
538
539 // Returned value isDeclaration() is equal to F.isDeclaration()
540 // but that property is not invariant throughout this function
541 const bool FunctionIsDefinition = !F.isDeclaration();
542
543 FunctionType *FTy = F.getFunctionType();
544 SmallVector<Type *> ArgTypes(FTy->params());
545 ArgTypes.push_back(Elt: ABI->vaListParameterType(M));
546
547 FunctionType *NFTy = inlinableVariadicFunctionType(M, FTy);
548 Function *NF = Function::Create(Ty: NFTy, Linkage: F.getLinkage(), AddrSpace: F.getAddressSpace());
549
550 // Note - same attribute handling as DeadArgumentElimination
551 NF->copyAttributesFrom(Src: &F);
552 NF->setComdat(F.getComdat());
553 F.getParent()->getFunctionList().insert(where: F.getIterator(), New: NF);
554 NF->setName(F.getName() + ".valist");
555
556 AttrBuilder ParamAttrs(Ctx);
557
558 AttributeList Attrs = NF->getAttributes();
559 Attrs = Attrs.addParamAttributes(C&: Ctx, ArgNo: NFTy->getNumParams() - 1, B: ParamAttrs);
560 NF->setAttributes(Attrs);
561
562 // Splice the implementation into the new function with minimal changes
563 if (FunctionIsDefinition) {
564 NF->splice(ToIt: NF->begin(), FromF: &F);
565
566 auto NewArg = NF->arg_begin();
567 for (Argument &Arg : F.args()) {
568 Arg.replaceAllUsesWith(V: NewArg);
569 NewArg->setName(Arg.getName()); // takeName without killing the old one
570 ++NewArg;
571 }
572 NewArg->setName("varargs");
573 }
574
575 SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;
576 F.getAllMetadata(MDs);
577 for (auto [KindID, Node] : MDs)
578 NF->addMetadata(KindID, MD&: *Node);
579 F.clearMetadata();
580
581 return NF;
582}
583
584Function *
585ExpandVariadics::defineVariadicWrapper(Module &M, IRBuilder<> &Builder,
586 Function *VariadicWrapper,
587 Function *FixedArityReplacement) {
588 auto &Ctx = Builder.getContext();
589 const DataLayout &DL = M.getDataLayout();
590 assert(VariadicWrapper->isDeclaration());
591 Function &F = *VariadicWrapper;
592
593 assert(F.isDeclaration());
594 Type *VaListTy = ABI->vaListType(Ctx);
595
596 auto *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: &F);
597 Builder.SetInsertPoint(BB);
598
599 AllocaInst *VaListInstance =
600 Builder.CreateAlloca(Ty: VaListTy, ArraySize: nullptr, Name: "va_start");
601
602 Builder.CreateLifetimeStart(Ptr: VaListInstance);
603
604 Builder.CreateIntrinsic(ID: Intrinsic::vastart, OverloadTypes: {DL.getAllocaPtrType(Ctx)},
605 Args: {VaListInstance});
606
607 SmallVector<Value *> Args(llvm::make_pointer_range(Range: F.args()));
608
609 Value *VaListValue = VaListInstance;
610 if (ABI->vaListPassedInSSARegister())
611 VaListValue = Builder.CreateLoad(Ty: VaListTy, Ptr: VaListInstance);
612
613 Type *ParameterType = ABI->vaListParameterType(M);
614 Args.push_back(Elt: Builder.CreateAddrSpaceCast(V: VaListValue, DestTy: ParameterType));
615
616 CallInst *Result = Builder.CreateCall(Callee: FixedArityReplacement, Args);
617
618 Builder.CreateIntrinsic(ID: Intrinsic::vaend, OverloadTypes: {DL.getAllocaPtrType(Ctx)},
619 Args: {VaListInstance});
620 Builder.CreateLifetimeEnd(Ptr: VaListInstance);
621
622 if (Result->getType()->isVoidTy())
623 Builder.CreateRetVoid();
624 else
625 Builder.CreateRet(V: Result);
626
627 return VariadicWrapper;
628}
629
630bool ExpandVariadics::expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB,
631 FunctionType *VarargFunctionType,
632 Function *NF) {
633 bool Changed = false;
634 const DataLayout &DL = M.getDataLayout();
635
636 if (ABI->ignoreFunction(F: CB->getCalledFunction()))
637 return Changed;
638
639 if (!expansionApplicableToFunctionCall(CB)) {
640 if (rewriteABI())
641 report_fatal_error(reason: "Cannot lower callbase instruction");
642 return Changed;
643 }
644
645 // This is tricky. The call instruction's function type might not match
646 // the type of the caller. When optimising, can leave it unchanged.
647 // Webassembly detects that inconsistency and repairs it.
648 if (CB->getFunctionType() != VarargFunctionType)
649 if (!rewriteABI())
650 return Changed;
651
652 auto &Ctx = CB->getContext();
653
654 Align MaxFieldAlign(1);
655
656 // The strategy is to allocate a call frame containing the variadic
657 // arguments laid out such that a target specific va_list can be initialized
658 // with it, such that target specific va_arg instructions will correctly
659 // iterate over it. This means getting the alignment right and sometimes
660 // embedding a pointer to the value instead of embedding the value itself.
661
662 Function *CBF = CB->getParent()->getParent();
663
664 ExpandedCallFrame Frame;
665
666 uint64_t CurrentOffset = 0;
667
668 for (unsigned I : seq(Begin: VarargFunctionType->getNumParams(), End: CB->arg_size())) {
669 Value *ArgVal = CB->getArgOperand(i: I);
670 const bool IsByVal = CB->paramHasAttr(ArgNo: I, Kind: Attribute::ByVal);
671 const bool IsByRef = CB->paramHasAttr(ArgNo: I, Kind: Attribute::ByRef);
672
673 // The type of the value being passed, decoded from byval/byref metadata if
674 // required
675 Type *const UnderlyingType = IsByVal ? CB->getParamByValType(ArgNo: I)
676 : IsByRef ? CB->getParamByRefType(ArgNo: I)
677 : ArgVal->getType();
678 const uint64_t UnderlyingSize =
679 DL.getTypeAllocSize(Ty: UnderlyingType).getFixedValue();
680
681 // The type to be written into the call frame
682 Type *FrameFieldType = UnderlyingType;
683
684 // The value to copy from when initialising the frame alloca
685 Value *SourceValue = ArgVal;
686
687 VariadicABIInfo::VAArgSlotInfo SlotInfo = ABI->slotInfo(DL, Parameter: UnderlyingType);
688
689 if (SlotInfo.Indirect) {
690 // The va_arg lowering loads through a pointer. Set up an alloca to aim
691 // that pointer at.
692 Builder.SetInsertPointPastAllocas(CBF);
693 Builder.SetCurrentDebugLocation(CB->getDebugLoc());
694 Value *CallerCopy =
695 Builder.CreateAlloca(Ty: UnderlyingType, ArraySize: nullptr, Name: "IndirectAlloca");
696
697 Builder.SetInsertPoint(CB);
698 if (IsByVal)
699 Builder.CreateMemCpy(Dst: CallerCopy, DstAlign: {}, Src: ArgVal, SrcAlign: {}, Size: UnderlyingSize);
700 else
701 Builder.CreateStore(Val: ArgVal, Ptr: CallerCopy);
702
703 // Indirection now handled, pass the alloca ptr by value
704 FrameFieldType = DL.getAllocaPtrType(Ctx);
705 SourceValue = CallerCopy;
706 }
707
708 // Alignment of the value within the frame
709 // This probably needs to be controllable as a function of type
710 Align DataAlign = SlotInfo.DataAlign;
711
712 MaxFieldAlign = std::max(a: MaxFieldAlign, b: DataAlign);
713
714 uint64_t DataAlignV = DataAlign.value();
715 if (uint64_t Rem = CurrentOffset % DataAlignV) {
716 // Inject explicit padding to deal with alignment requirements
717 uint64_t Padding = DataAlignV - Rem;
718 Frame.padding(Ctx, By: Padding);
719 CurrentOffset += Padding;
720 }
721
722 if (SlotInfo.Indirect) {
723 Frame.store(Ctx, T: FrameFieldType, V: SourceValue);
724 } else {
725 if (IsByVal)
726 Frame.memcpy(Ctx, T: FrameFieldType, V: SourceValue, Bytes: UnderlyingSize);
727 else
728 Frame.store(Ctx, T: FrameFieldType, V: SourceValue);
729 }
730
731 CurrentOffset += DL.getTypeAllocSize(Ty: FrameFieldType).getFixedValue();
732 }
733
734 if (Frame.empty()) {
735 // Not passing any arguments, hopefully va_arg won't try to read any
736 // Creating a single byte frame containing nothing to point the va_list
737 // instance as that is less special-casey in the compiler and probably
738 // easier to interpret in a debugger.
739 Frame.padding(Ctx, By: 1);
740 }
741
742 StructType *VarargsTy = Frame.asStruct(Ctx, Name: CBF->getName());
743
744 // The struct instance needs to be at least MaxFieldAlign for the alignment of
745 // the fields to be correct at runtime. Use the native stack alignment instead
746 // if that's greater as that tends to give better codegen.
747 // This is an awkward way to guess whether there is a known stack alignment
748 // without hitting an assert in DL.getStackAlignment, 1024 is an arbitrary
749 // number likely to be greater than the natural stack alignment.
750 Align AllocaAlign = MaxFieldAlign;
751 if (MaybeAlign StackAlign = DL.getStackAlignment();
752 StackAlign && *StackAlign > AllocaAlign)
753 AllocaAlign = *StackAlign;
754
755 // Put the alloca to hold the variadic args in the entry basic block.
756 Builder.SetInsertPointPastAllocas(CBF);
757
758 // SetCurrentDebugLocation when the builder SetInsertPoint method does not
759 Builder.SetCurrentDebugLocation(CB->getDebugLoc());
760
761 // The awkward construction here is to set the alignment on the instance
762 AllocaInst *Alloced = Builder.Insert(
763 I: new AllocaInst(VarargsTy, DL.getAllocaAddrSpace(), nullptr, AllocaAlign),
764 Name: "vararg_buffer");
765 Changed = true;
766 assert(Alloced->getAllocatedType() == VarargsTy);
767
768 // Initialize the fields in the struct
769 Builder.SetInsertPoint(CB);
770 Builder.CreateLifetimeStart(Ptr: Alloced);
771 Frame.initializeStructAlloca(DL, Builder, Alloced, VarargsTy);
772
773 const unsigned NumArgs = VarargFunctionType->getNumParams();
774 SmallVector<Value *> Args(CB->arg_begin(), CB->arg_begin() + NumArgs);
775
776 // Initialize a va_list pointing to that struct and pass it as the last
777 // argument
778 AllocaInst *VaList = nullptr;
779 {
780 if (!ABI->vaListPassedInSSARegister()) {
781 Type *VaListTy = ABI->vaListType(Ctx);
782 Builder.SetInsertPointPastAllocas(CBF);
783 Builder.SetCurrentDebugLocation(CB->getDebugLoc());
784 VaList = Builder.CreateAlloca(Ty: VaListTy, ArraySize: nullptr, Name: "va_argument");
785 Builder.SetInsertPoint(CB);
786 Builder.CreateLifetimeStart(Ptr: VaList);
787 }
788 Builder.SetInsertPoint(CB);
789 Args.push_back(Elt: ABI->initializeVaList(M, Ctx, Builder, VaList, Buffer: Alloced));
790 }
791
792 // Attributes excluding any on the vararg arguments
793 AttributeList PAL = CB->getAttributes();
794 if (!PAL.isEmpty()) {
795 SmallVector<AttributeSet, 8> ArgAttrs;
796 for (unsigned ArgNo = 0; ArgNo < NumArgs; ArgNo++)
797 ArgAttrs.push_back(Elt: PAL.getParamAttrs(ArgNo));
798 PAL =
799 AttributeList::get(C&: Ctx, FnAttrs: PAL.getFnAttrs(), RetAttrs: PAL.getRetAttrs(), ArgAttrs);
800 }
801
802 SmallVector<OperandBundleDef, 1> OpBundles;
803 CB->getOperandBundlesAsDefs(Defs&: OpBundles);
804
805 CallBase *NewCB = nullptr;
806
807 if (CallInst *CI = dyn_cast<CallInst>(Val: CB)) {
808 Value *Dst = NF ? NF : CI->getCalledOperand();
809 // Use the type of the call site rather than the function type to ensure
810 // RAUW succeeds in the case of a mismatching return type.
811 FunctionType *NFTy =
812 inlinableVariadicFunctionType(M, FTy: VarargFunctionType, ReturnType: CB->getType());
813
814 NewCB = CallInst::Create(Ty: NFTy, Func: Dst, Args, Bundles: OpBundles, NameStr: "", InsertBefore: CI->getIterator());
815
816 CallInst::TailCallKind TCK = CI->getTailCallKind();
817 assert(TCK != CallInst::TCK_MustTail);
818
819 // Can't tail call a function that is being passed a pointer to an alloca
820 if (TCK == CallInst::TCK_Tail)
821 TCK = CallInst::TCK_None;
822 CI->setTailCallKind(TCK);
823
824 } else {
825 llvm_unreachable("Unreachable when !expansionApplicableToFunctionCall()");
826 }
827
828 if (VaList)
829 Builder.CreateLifetimeEnd(Ptr: VaList);
830
831 Builder.CreateLifetimeEnd(Ptr: Alloced);
832
833 NewCB->setAttributes(PAL);
834 NewCB->takeName(V: CB);
835 NewCB->setCallingConv(CB->getCallingConv());
836 NewCB->setDebugLoc(DebugLoc());
837
838 // DeadArgElim and ArgPromotion copy exactly this metadata
839 NewCB->copyProfileAndDebugMetadata(SrcInst: *CB);
840
841 CB->replaceAllUsesWith(V: NewCB);
842 CB->eraseFromParent();
843 return Changed;
844}
845
846bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder,
847 const DataLayout &DL,
848 VAStartInst *Inst) {
849 // Only removing va_start instructions that are not in variadic functions.
850 // Those would be rejected by the IR verifier before this pass.
851 // After splicing basic blocks from a variadic function into a fixed arity
852 // one the va_start that used to refer to the ... parameter still exist.
853 // There are also variadic functions that this pass did not change and
854 // va_start instances in the created single block wrapper functions.
855 // Replace exactly the instances in non-variadic functions as those are
856 // the ones to be fixed up to use the va_list passed as the final argument.
857
858 Function *ContainingFunction = Inst->getFunction();
859 if (ContainingFunction->isVarArg()) {
860 return false;
861 }
862
863 // The last argument is a vaListParameterType, either a va_list
864 // or a pointer to one depending on the target.
865 bool PassedByValue = ABI->vaListPassedInSSARegister();
866 Argument *PassedVaList =
867 ContainingFunction->getArg(i: ContainingFunction->arg_size() - 1);
868
869 // va_start takes a pointer to a va_list, e.g. one on the stack
870 Value *VaStartArg = Inst->getArgList();
871
872 Builder.SetInsertPoint(Inst);
873
874 if (PassedByValue) {
875 // The general thing to do is create an alloca, store the va_list argument
876 // to it, then create a va_copy. When vaCopyIsMemcpy(), this optimises to a
877 // store to the VaStartArg.
878 assert(ABI->vaCopyIsMemcpy());
879 // The va_list parameter may be passed in a different address space than
880 // the va_list object iterates in (e.g. NVPTX passes a local pointer but
881 // stores a generic cursor). Cast it to the type va_arg expects to load.
882 Value *Cursor = PassedVaList;
883 if (Cursor->getType() != VaStartArg->getType())
884 Cursor = Builder.CreateAddrSpaceCast(V: Cursor, DestTy: VaStartArg->getType());
885 Builder.CreateStore(Val: Cursor, Ptr: VaStartArg);
886 } else {
887
888 // Otherwise emit a vacopy to pick up target-specific handling if any
889 auto &Ctx = Builder.getContext();
890
891 Builder.CreateIntrinsic(ID: Intrinsic::vacopy, OverloadTypes: {DL.getAllocaPtrType(Ctx)},
892 Args: {VaStartArg, PassedVaList});
893 }
894
895 Inst->eraseFromParent();
896 return true;
897}
898
899bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &,
900 VAEndInst *Inst) {
901 assert(ABI->vaEndIsNop());
902 Inst->eraseFromParent();
903 return true;
904}
905
906bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder,
907 const DataLayout &DL,
908 VACopyInst *Inst) {
909 assert(ABI->vaCopyIsMemcpy());
910 Builder.SetInsertPoint(Inst);
911
912 auto &Ctx = Builder.getContext();
913 Type *VaListTy = ABI->vaListType(Ctx);
914 uint64_t Size = DL.getTypeAllocSize(Ty: VaListTy).getFixedValue();
915
916 Builder.CreateMemCpy(Dst: Inst->getDest(), DstAlign: {}, Src: Inst->getSrc(), SrcAlign: {},
917 Size: Builder.getInt32(C: Size));
918
919 Inst->eraseFromParent();
920 return true;
921}
922
923bool ExpandVariadics::expandVAArgInst(IRBuilder<> &Builder,
924 const DataLayout &DL, VAArgInst *Inst) {
925 Builder.SetInsertPoint(Inst);
926
927 auto &Ctx = Builder.getContext();
928 Type *ValTy = Inst->getType();
929 Value *VaListPtr = Inst->getPointerOperand();
930
931 const VariadicABIInfo::VAArgSlotInfo SlotInfo = ABI->slotInfo(DL, Parameter: ValTy);
932 Type *FrameFieldType = SlotInfo.Indirect ? DL.getAllocaPtrType(Ctx) : ValTy;
933 const uint64_t SlotSize = DL.getTypeAllocSize(Ty: FrameFieldType).getFixedValue();
934 const Align SlotAlign = SlotInfo.DataAlign;
935
936 Type *PtrTy = VaListPtr->getType();
937 Type *IdxTy = DL.getIndexType(PtrTy);
938
939 Value *Cur = Builder.CreateLoad(Ty: PtrTy, Ptr: VaListPtr);
940
941 // Round the cursor up to the slot alignment used by the caller.
942 Value *Aligned = Cur;
943 if (SlotAlign > Align(1)) {
944 Value *RoundUp = Builder.CreateInBoundsPtrAdd(
945 Ptr: Cur, Offset: ConstantInt::get(Ty: IdxTy, V: SlotAlign.value() - 1));
946 Aligned = Builder.CreateIntrinsic(
947 ID: Intrinsic::ptrmask, OverloadTypes: {PtrTy, IdxTy},
948 Args: {RoundUp, ConstantInt::getSigned(Ty: IdxTy, V: -(int64_t)SlotAlign.value())});
949 }
950
951 // Advance past the slot and write the iterator back.
952 Value *Next =
953 Builder.CreateInBoundsPtrAdd(Ptr: Aligned, Offset: ConstantInt::get(Ty: IdxTy, V: SlotSize));
954 Builder.CreateStore(Val: Next, Ptr: VaListPtr);
955
956 // Load the slot contents: the value itself for direct arguments, or a
957 // pointer to the value for indirect ones.
958 Value *Result = Builder.CreateAlignedLoad(Ty: FrameFieldType, Ptr: Aligned, Align: SlotAlign);
959
960 if (SlotInfo.Indirect)
961 Result = Builder.CreateLoad(Ty: ValTy, Ptr: Result);
962
963 Result->takeName(V: Inst);
964 Inst->replaceAllUsesWith(V: Result);
965 Inst->eraseFromParent();
966 return true;
967}
968
969struct Amdgpu final : public VariadicABIInfo {
970
971 bool enableForTarget() override { return true; }
972
973 bool vaListPassedInSSARegister() override { return true; }
974
975 Type *vaListType(LLVMContext &Ctx) override {
976 return PointerType::getUnqual(C&: Ctx);
977 }
978
979 Type *vaListParameterType(Module &M) override {
980 return PointerType::getUnqual(C&: M.getContext());
981 }
982
983 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
984 AllocaInst * /*va_list*/, Value *Buffer) override {
985 // Given Buffer, which is an AllocInst of vararg_buffer
986 // need to return something usable as parameter type
987 return Builder.CreateAddrSpaceCast(V: Buffer, DestTy: vaListParameterType(M));
988 }
989
990 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
991 return {.DataAlign: Align(4), .Indirect: false};
992 }
993};
994
995struct NVPTX final : public VariadicABIInfo {
996
997 bool enableForTarget() override { return true; }
998
999 bool vaListPassedInSSARegister() override { return true; }
1000
1001 Type *vaListType(LLVMContext &Ctx) override {
1002 return PointerType::getUnqual(C&: Ctx);
1003 }
1004
1005 Type *vaListParameterType(Module &M) override {
1006 return PointerType::get(C&: M.getContext(), AddressSpace: NVPTXAS::ADDRESS_SPACE_LOCAL);
1007 }
1008
1009 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
1010 AllocaInst *, Value *Buffer) override {
1011 return Builder.CreateAddrSpaceCast(V: Buffer, DestTy: vaListParameterType(M));
1012 }
1013
1014 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
1015 // NVPTX expects natural alignment in all cases. The variadic call ABI will
1016 // handle promoting types to their appropriate size and alignment.
1017 Align A = DL.getABITypeAlign(Ty: Parameter);
1018 return {.DataAlign: A, .Indirect: false};
1019 }
1020};
1021
1022struct SPIRV final : public VariadicABIInfo {
1023
1024 bool enableForTarget() override { return true; }
1025
1026 bool vaListPassedInSSARegister() override { return true; }
1027
1028 Type *vaListType(LLVMContext &Ctx) override {
1029 return PointerType::getUnqual(C&: Ctx);
1030 }
1031
1032 Type *vaListParameterType(Module &M) override {
1033 return PointerType::getUnqual(C&: M.getContext());
1034 }
1035
1036 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
1037 AllocaInst *, Value *Buffer) override {
1038 return Builder.CreateAddrSpaceCast(V: Buffer, DestTy: vaListParameterType(M));
1039 }
1040
1041 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
1042 // Expects natural alignment in all cases. The variadic call ABI will handle
1043 // promoting types to their appropriate size and alignment.
1044 Align A = DL.getABITypeAlign(Ty: Parameter);
1045 return {.DataAlign: A, .Indirect: false};
1046 }
1047
1048 // The SPIR-V backend has special handling for builtins.
1049 bool ignoreFunction(const Function *F) override {
1050 if (!F->isDeclaration())
1051 return false;
1052
1053 std::string Demangled = llvm::demangle(MangledName: F->getName());
1054 StringRef DemangledName(Demangled);
1055
1056 // Skip any SPIR-V builtins.
1057 // Note: an unmangled C `printf` declaration demangles to "printf" with no
1058 // argument list, so the "printf(" prefix check below misses it. Match the
1059 // bare name as well so OpenCL/HIP printf (emitted unmangled) is left as a
1060 // variadic call for the backend's OpenCL.std printf lowering to expand
1061 // inline, rather than being packed into a vararg buffer here.
1062 if (DemangledName.starts_with(Prefix: "__spirv_") ||
1063 DemangledName.starts_with(Prefix: "printf(") || F->getName() == "printf")
1064 return true;
1065
1066 return false;
1067 }
1068
1069 // We will likely see va intrinsics in the generic addrspace (4).
1070 SmallVector<unsigned> getTargetSpecificVaIntrinAddrSpaces() const override {
1071 return {4};
1072 }
1073};
1074
1075struct Wasm final : public VariadicABIInfo {
1076
1077 // Currently wasm is only used for testing, enabled by
1078 // -expand-variadics-override.
1079 bool enableForTarget() override { return false; }
1080
1081 bool vaListPassedInSSARegister() override { return true; }
1082
1083 Type *vaListType(LLVMContext &Ctx) override {
1084 return PointerType::getUnqual(C&: Ctx);
1085 }
1086
1087 Type *vaListParameterType(Module &M) override {
1088 return PointerType::getUnqual(C&: M.getContext());
1089 }
1090
1091 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
1092 AllocaInst * /*va_list*/, Value *Buffer) override {
1093 return Buffer;
1094 }
1095
1096 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
1097 LLVMContext &Ctx = Parameter->getContext();
1098 const unsigned MinAlign = 4;
1099 Align A = DL.getABITypeAlign(Ty: Parameter);
1100 if (A < MinAlign)
1101 A = Align(MinAlign);
1102
1103 if (auto *S = dyn_cast<StructType>(Val: Parameter)) {
1104 if (S->getNumElements() > 1) {
1105 return {.DataAlign: DL.getABITypeAlign(Ty: PointerType::getUnqual(C&: Ctx)), .Indirect: true};
1106 }
1107 }
1108
1109 return {.DataAlign: A, .Indirect: false};
1110 }
1111};
1112
1113std::unique_ptr<VariadicABIInfo> VariadicABIInfo::create(const Triple &T) {
1114 switch (T.getArch()) {
1115 case Triple::amdgpu:
1116 case Triple::r600: {
1117 return std::make_unique<Amdgpu>();
1118 }
1119
1120 case Triple::wasm32: {
1121 return std::make_unique<Wasm>();
1122 }
1123
1124 case Triple::nvptx:
1125 case Triple::nvptx64: {
1126 return std::make_unique<NVPTX>();
1127 }
1128
1129 case Triple::spirv:
1130 case Triple::spirv32:
1131 case Triple::spirv64: {
1132 return std::make_unique<SPIRV>();
1133 }
1134
1135 default:
1136 return {};
1137 }
1138}
1139
1140} // namespace
1141
1142char ExpandVariadics::ID = 0;
1143
1144INITIALIZE_PASS(ExpandVariadics, DEBUG_TYPE, "Expand variadic functions", false,
1145 false)
1146
1147ModulePass *llvm::createExpandVariadicsPass(ExpandVariadicsMode M) {
1148 return new ExpandVariadics(M);
1149}
1150
1151PreservedAnalyses ExpandVariadicsPass::run(Module &M, ModuleAnalysisManager &) {
1152 return ExpandVariadics(Mode).runOnModule(M) ? PreservedAnalyses::none()
1153 : PreservedAnalyses::all();
1154}
1155
1156ExpandVariadicsPass::ExpandVariadicsPass(ExpandVariadicsMode M) : Mode(M) {}
1157