1//===-- ExpandVariadicsPass.cpp --------------------------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is an optimization pass for variadic functions. If called from codegen,
10// it can serve as the implementation of variadic functions for a given target.
11//
12// The strategy is to turn the ... part of a variadic function into a va_list
13// and fix up the call sites. The majority of the pass is target independent.
14// The exceptions are the va_list type itself and the rules for where to store
15// variables in memory such that va_arg can iterate over them given a va_list.
16//
17// The majority of the plumbing is splitting the variadic function into a
18// single basic block that packs the variadic arguments into a va_list and
19// a second function that does the work of the original. That packing is
20// exactly what is done by va_start. Further, the transform from ... to va_list
21// replaced va_start with an operation to copy a va_list from the new argument,
22// which is exactly a va_copy. This is useful for reducing target-dependence.
23//
24// A va_list instance is a forward iterator, where the primary operation va_arg
25// is dereference-then-increment. This interface forces significant convergent
26// evolution between target specific implementations. The variation in runtime
27// data layout is limited to that representable by the iterator, parameterised
28// by the type passed to the va_arg instruction.
29//
30// Therefore the majority of the target specific subtlety is packing arguments
31// into a stack allocated buffer such that a va_list can be initialised with it
32// and the va_arg expansion for the target will find the arguments at runtime.
33//
34// The aggregate effect is to unblock other transforms, most critically the
35// general purpose inliner. Known calls to variadic functions become zero cost.
36//
37// Consistency with clang is primarily tested by emitting va_arg using clang
38// then expanding the variadic functions using this pass, followed by trying
39// to constant fold the functions to no-ops.
40//
41// Target specific behaviour is tested in IR - mainly checking that values are
42// put into positions in call frames that make sense for that particular target.
43//
44// There is one "clever" invariant in use. va_start intrinsics that are not
45// within a varidic functions are an error in the IR verifier. When this
46// transform moves blocks from a variadic function into a fixed arity one, it
47// moves va_start intrinsics along with everything else. That means that the
48// va_start intrinsics that need to be rewritten to use the trailing argument
49// are exactly those that are in non-variadic functions so no further state
50// is needed to distinguish those that need to be rewritten.
51//
52//===----------------------------------------------------------------------===//
53
54#include "llvm/Transforms/IPO/ExpandVariadics.h"
55#include "llvm/ADT/SmallVector.h"
56#include "llvm/IR/IRBuilder.h"
57#include "llvm/IR/IntrinsicInst.h"
58#include "llvm/IR/Module.h"
59#include "llvm/IR/PassManager.h"
60#include "llvm/InitializePasses.h"
61#include "llvm/Pass.h"
62#include "llvm/Support/CommandLine.h"
63#include "llvm/TargetParser/Triple.h"
64#include "llvm/Transforms/Utils/ModuleUtils.h"
65
66#define DEBUG_TYPE "expand-variadics"
67
68using namespace llvm;
69
70namespace {
71
72cl::opt<ExpandVariadicsMode> ExpandVariadicsModeOption(
73 DEBUG_TYPE "-override", cl::desc("Override the behaviour of " DEBUG_TYPE),
74 cl::init(Val: ExpandVariadicsMode::Unspecified),
75 cl::values(clEnumValN(ExpandVariadicsMode::Unspecified, "unspecified",
76 "Use the implementation defaults"),
77 clEnumValN(ExpandVariadicsMode::Disable, "disable",
78 "Disable the pass entirely"),
79 clEnumValN(ExpandVariadicsMode::Optimize, "optimize",
80 "Optimise without changing ABI"),
81 clEnumValN(ExpandVariadicsMode::Lowering, "lowering",
82 "Change variadic calling convention")));
83
84bool commandLineOverride() {
85 return ExpandVariadicsModeOption != ExpandVariadicsMode::Unspecified;
86}
87
88// Instances of this class encapsulate the target-dependant behaviour as a
89// function of triple. Implementing a new ABI is adding a case to the switch
90// in create(llvm::Triple) at the end of this file.
91// This class may end up instantiated in TargetMachine instances, keeping it
92// here for now until enough targets are implemented for the API to evolve.
93class VariadicABIInfo {
94protected:
95 VariadicABIInfo() = default;
96
97public:
98 static std::unique_ptr<VariadicABIInfo> create(const Triple &T);
99
100 // Allow overriding whether the pass runs on a per-target basis
101 virtual bool enableForTarget() = 0;
102
103 // Whether a valist instance is passed by value or by address
104 // I.e. does it need to be alloca'ed and stored into, or can
105 // it be passed directly in a SSA register
106 virtual bool vaListPassedInSSARegister() = 0;
107
108 // The type of a va_list iterator object
109 virtual Type *vaListType(LLVMContext &Ctx) = 0;
110
111 // The type of a va_list as a function argument as lowered by C
112 virtual Type *vaListParameterType(Module &M) = 0;
113
114 // Initialize an allocated va_list object to point to an already
115 // initialized contiguous memory region.
116 // Return the value to pass as the va_list argument
117 virtual Value *initializeVaList(Module &M, LLVMContext &Ctx,
118 IRBuilder<> &Builder, AllocaInst *VaList,
119 Value *Buffer) = 0;
120
121 struct VAArgSlotInfo {
122 Align DataAlign; // With respect to the call frame
123 bool Indirect; // Passed via a pointer
124 };
125 virtual VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) = 0;
126
127 // Per-target overrides of special symbols.
128 virtual bool ignoreFunction(Function *F) { return false; }
129
130 // Targets implemented so far all have the same trivial lowering for these
131 bool vaEndIsNop() { return true; }
132 bool vaCopyIsMemcpy() { return true; }
133
134 virtual ~VariadicABIInfo() = default;
135};
136
137class ExpandVariadics : public ModulePass {
138
139 // The pass construction sets the default to optimize when called from middle
140 // end and lowering when called from the backend. The command line variable
141 // overrides that. This is useful for testing and debugging. It also allows
142 // building an applications with variadic functions wholly removed if one
143 // has sufficient control over the dependencies, e.g. a statically linked
144 // clang that has no variadic function calls remaining in the binary.
145
146public:
147 static char ID;
148 const ExpandVariadicsMode Mode;
149 std::unique_ptr<VariadicABIInfo> ABI;
150
151 ExpandVariadics(ExpandVariadicsMode Mode)
152 : ModulePass(ID),
153 Mode(commandLineOverride() ? ExpandVariadicsModeOption : Mode) {}
154
155 StringRef getPassName() const override { return "Expand variadic functions"; }
156
157 bool rewriteABI() { return Mode == ExpandVariadicsMode::Lowering; }
158
159 template <typename T> bool isValidCallingConv(T *F) {
160 return F->getCallingConv() == CallingConv::C ||
161 F->getCallingConv() == CallingConv::SPIR_FUNC;
162 }
163
164 bool runOnModule(Module &M) override;
165
166 bool runOnFunction(Module &M, IRBuilder<> &Builder, Function *F);
167
168 Function *replaceAllUsesWithNewDeclaration(Module &M,
169 Function *OriginalFunction);
170
171 Function *deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder,
172 Function *OriginalFunction);
173
174 Function *defineVariadicWrapper(Module &M, IRBuilder<> &Builder,
175 Function *VariadicWrapper,
176 Function *FixedArityReplacement);
177
178 bool expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB, FunctionType *,
179 Function *NF);
180
181 // The intrinsic functions va_copy and va_end are removed unconditionally.
182 // They correspond to a memcpy and a no-op on all implemented targets.
183 // The va_start intrinsic is removed from basic blocks that were not created
184 // by this pass, some may remain if needed to maintain the external ABI.
185
186 template <Intrinsic::ID ID, typename InstructionType>
187 bool expandIntrinsicUsers(Module &M, IRBuilder<> &Builder,
188 PointerType *IntrinsicArgType) {
189 bool Changed = false;
190 const DataLayout &DL = M.getDataLayout();
191 if (Function *Intrinsic =
192 Intrinsic::getDeclarationIfExists(M: &M, id: ID, Tys: {IntrinsicArgType})) {
193 for (User *U : make_early_inc_range(Range: Intrinsic->users()))
194 if (auto *I = dyn_cast<InstructionType>(U))
195 Changed |= expandVAIntrinsicCall(Builder, DL, I);
196
197 if (Intrinsic->use_empty())
198 Intrinsic->eraseFromParent();
199 }
200 return Changed;
201 }
202
203 bool expandVAIntrinsicUsersWithAddrspace(Module &M, IRBuilder<> &Builder,
204 unsigned Addrspace) {
205 auto &Ctx = M.getContext();
206 PointerType *IntrinsicArgType = PointerType::get(C&: Ctx, AddressSpace: Addrspace);
207 bool Changed = false;
208
209 // expand vastart before vacopy as vastart may introduce a vacopy
210 Changed |= expandIntrinsicUsers<Intrinsic::vastart, VAStartInst>(
211 M, Builder, IntrinsicArgType);
212 Changed |= expandIntrinsicUsers<Intrinsic::vaend, VAEndInst>(
213 M, Builder, IntrinsicArgType);
214 Changed |= expandIntrinsicUsers<Intrinsic::vacopy, VACopyInst>(
215 M, Builder, IntrinsicArgType);
216 return Changed;
217 }
218
219 bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL,
220 VAStartInst *Inst);
221
222 bool expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &,
223 VAEndInst *Inst);
224
225 bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL,
226 VACopyInst *Inst);
227
228 FunctionType *inlinableVariadicFunctionType(Module &M, FunctionType *FTy) {
229 // The type of "FTy" with the ... removed and a va_list appended
230 SmallVector<Type *> ArgTypes(FTy->params());
231 ArgTypes.push_back(Elt: ABI->vaListParameterType(M));
232 return FunctionType::get(Result: FTy->getReturnType(), Params: ArgTypes,
233 /*IsVarArgs=*/isVarArg: false);
234 }
235
236 bool expansionApplicableToFunction(Module &M, Function *F) {
237 if (F->isIntrinsic() || !F->isVarArg() ||
238 F->hasFnAttribute(Kind: Attribute::Naked))
239 return false;
240
241 if (ABI->ignoreFunction(F))
242 return false;
243
244 if (!isValidCallingConv(F))
245 return false;
246
247 if (rewriteABI())
248 return true;
249
250 if (!F->hasExactDefinition())
251 return false;
252
253 return true;
254 }
255
256 bool expansionApplicableToFunctionCall(CallBase *CB) {
257 if (CallInst *CI = dyn_cast<CallInst>(Val: CB)) {
258 if (CI->isMustTailCall()) {
259 // Cannot expand musttail calls
260 return false;
261 }
262
263 if (!isValidCallingConv(F: CI))
264 return false;
265
266 return true;
267 }
268
269 if (isa<InvokeInst>(Val: CB)) {
270 // Invoke not implemented in initial implementation of pass
271 return false;
272 }
273
274 // Other unimplemented derivative of CallBase
275 return false;
276 }
277
278 class ExpandedCallFrame {
279 // Helper for constructing an alloca instance containing the arguments bound
280 // to the variadic ... parameter, rearranged to allow indexing through a
281 // va_list iterator
282 enum { N = 4 };
283 SmallVector<Type *, N> FieldTypes;
284 enum Tag { Store, Memcpy, Padding };
285 SmallVector<std::tuple<Value *, uint64_t, Tag>, N> Source;
286
287 template <Tag tag> void append(Type *FieldType, Value *V, uint64_t Bytes) {
288 FieldTypes.push_back(Elt: FieldType);
289 Source.push_back(Elt: {V, Bytes, tag});
290 }
291
292 public:
293 void store(LLVMContext &Ctx, Type *T, Value *V) { append<Store>(FieldType: T, V, Bytes: 0); }
294
295 void memcpy(LLVMContext &Ctx, Type *T, Value *V, uint64_t Bytes) {
296 append<Memcpy>(FieldType: T, V, Bytes);
297 }
298
299 void padding(LLVMContext &Ctx, uint64_t By) {
300 append<Padding>(FieldType: ArrayType::get(ElementType: Type::getInt8Ty(C&: Ctx), NumElements: By), V: nullptr, Bytes: 0);
301 }
302
303 size_t size() const { return FieldTypes.size(); }
304 bool empty() const { return FieldTypes.empty(); }
305
306 StructType *asStruct(LLVMContext &Ctx, StringRef Name) {
307 const bool IsPacked = true;
308 return StructType::create(Context&: Ctx, Elements: FieldTypes,
309 Name: (Twine(Name) + ".vararg").str(), isPacked: IsPacked);
310 }
311
312 void initializeStructAlloca(const DataLayout &DL, IRBuilder<> &Builder,
313 AllocaInst *Alloced, StructType *VarargsTy) {
314
315 for (size_t I = 0; I < size(); I++) {
316
317 auto [V, bytes, tag] = Source[I];
318
319 if (tag == Padding) {
320 assert(V == nullptr);
321 continue;
322 }
323
324 auto Dst = Builder.CreateStructGEP(Ty: VarargsTy, Ptr: Alloced, Idx: I);
325
326 assert(V != nullptr);
327
328 if (tag == Store)
329 Builder.CreateStore(Val: V, Ptr: Dst);
330
331 if (tag == Memcpy)
332 Builder.CreateMemCpy(Dst, DstAlign: {}, Src: V, SrcAlign: {}, Size: bytes);
333 }
334 }
335 };
336};
337
338bool ExpandVariadics::runOnModule(Module &M) {
339 bool Changed = false;
340 if (Mode == ExpandVariadicsMode::Disable)
341 return Changed;
342
343 Triple TT(M.getTargetTriple());
344 ABI = VariadicABIInfo::create(T: TT);
345 if (!ABI)
346 return Changed;
347
348 if (!ABI->enableForTarget())
349 return Changed;
350
351 auto &Ctx = M.getContext();
352 const DataLayout &DL = M.getDataLayout();
353 IRBuilder<> Builder(Ctx);
354
355 // Lowering needs to run on all functions exactly once.
356 // Optimize could run on functions containing va_start exactly once.
357 for (Function &F : make_early_inc_range(Range&: M))
358 Changed |= runOnFunction(M, Builder, F: &F);
359
360 // After runOnFunction, all known calls to known variadic functions have been
361 // replaced. va_start intrinsics are presently (and invalidly!) only present
362 // in functions that used to be variadic and have now been replaced to take a
363 // va_list instead. If lowering as opposed to optimising, calls to unknown
364 // variadic functions have also been replaced.
365
366 {
367 // 0 and AllocaAddrSpace are sufficient for the targets implemented so far
368 unsigned Addrspace = 0;
369 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);
370
371 Addrspace = DL.getAllocaAddrSpace();
372 if (Addrspace != 0)
373 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);
374 }
375
376 if (Mode != ExpandVariadicsMode::Lowering)
377 return Changed;
378
379 for (Function &F : make_early_inc_range(Range&: M)) {
380 if (F.isDeclaration())
381 continue;
382
383 // Now need to track down indirect calls. Can't find those
384 // by walking uses of variadic functions, need to crawl the instruction
385 // stream. Fortunately this is only necessary for the ABI rewrite case.
386 for (BasicBlock &BB : F) {
387 for (Instruction &I : make_early_inc_range(Range&: BB)) {
388 if (CallBase *CB = dyn_cast<CallBase>(Val: &I)) {
389 if (CB->isIndirectCall()) {
390 FunctionType *FTy = CB->getFunctionType();
391 if (FTy->isVarArg())
392 Changed |= expandCall(M, Builder, CB, FTy, /*NF=*/nullptr);
393 }
394 }
395 }
396 }
397 }
398
399 return Changed;
400}
401
402bool ExpandVariadics::runOnFunction(Module &M, IRBuilder<> &Builder,
403 Function *OriginalFunction) {
404 bool Changed = false;
405
406 if (!expansionApplicableToFunction(M, F: OriginalFunction))
407 return Changed;
408
409 [[maybe_unused]] const bool OriginalFunctionIsDeclaration =
410 OriginalFunction->isDeclaration();
411 assert(rewriteABI() || !OriginalFunctionIsDeclaration);
412
413 // Declare a new function and redirect every use to that new function
414 Function *VariadicWrapper =
415 replaceAllUsesWithNewDeclaration(M, OriginalFunction);
416 assert(VariadicWrapper->isDeclaration());
417 assert(OriginalFunction->use_empty());
418
419 // Create a new function taking va_list containing the implementation of the
420 // original
421 Function *FixedArityReplacement =
422 deriveFixedArityReplacement(M, Builder, OriginalFunction);
423 assert(OriginalFunction->isDeclaration());
424 assert(FixedArityReplacement->isDeclaration() ==
425 OriginalFunctionIsDeclaration);
426 assert(VariadicWrapper->isDeclaration());
427
428 // Create a single block forwarding wrapper that turns a ... into a va_list
429 [[maybe_unused]] Function *VariadicWrapperDefine =
430 defineVariadicWrapper(M, Builder, VariadicWrapper, FixedArityReplacement);
431 assert(VariadicWrapperDefine == VariadicWrapper);
432 assert(!VariadicWrapper->isDeclaration());
433
434 // Add the prof metadata from the original function to the wrapper. Because
435 // FixedArityReplacement is the owner of original function's prof metadata
436 // after the splice, we need to transfer it to VariadicWrapper.
437 VariadicWrapper->setMetadata(
438 KindID: LLVMContext::MD_prof,
439 Node: FixedArityReplacement->getMetadata(KindID: LLVMContext::MD_prof));
440
441 // We now have:
442 // 1. the original function, now as a declaration with no uses
443 // 2. a variadic function that unconditionally calls a fixed arity replacement
444 // 3. a fixed arity function equivalent to the original function
445
446 // Replace known calls to the variadic with calls to the va_list equivalent
447 for (User *U : make_early_inc_range(Range: VariadicWrapper->users())) {
448 if (CallBase *CB = dyn_cast<CallBase>(Val: U)) {
449 Value *CalledOperand = CB->getCalledOperand();
450 if (VariadicWrapper == CalledOperand)
451 Changed |=
452 expandCall(M, Builder, CB, VariadicWrapper->getFunctionType(),
453 NF: FixedArityReplacement);
454 }
455 }
456
457 // The original function will be erased.
458 // One of the two new functions will become a replacement for the original.
459 // When preserving the ABI, the other is an internal implementation detail.
460 // When rewriting the ABI, RAUW then the variadic one.
461 Function *const ExternallyAccessible =
462 rewriteABI() ? FixedArityReplacement : VariadicWrapper;
463 Function *const InternalOnly =
464 rewriteABI() ? VariadicWrapper : FixedArityReplacement;
465
466 // The external function is the replacement for the original
467 ExternallyAccessible->setLinkage(OriginalFunction->getLinkage());
468 ExternallyAccessible->setVisibility(OriginalFunction->getVisibility());
469 ExternallyAccessible->setComdat(OriginalFunction->getComdat());
470 ExternallyAccessible->takeName(V: OriginalFunction);
471
472 // Annotate the internal one as internal
473 InternalOnly->setVisibility(GlobalValue::DefaultVisibility);
474 InternalOnly->setLinkage(GlobalValue::InternalLinkage);
475
476 // The original is unused and obsolete
477 OriginalFunction->eraseFromParent();
478
479 InternalOnly->removeDeadConstantUsers();
480
481 if (rewriteABI()) {
482 // All known calls to the function have been removed by expandCall
483 // Resolve everything else by replaceAllUsesWith
484 VariadicWrapper->replaceAllUsesWith(V: FixedArityReplacement);
485 VariadicWrapper->eraseFromParent();
486 }
487
488 return Changed;
489}
490
491Function *
492ExpandVariadics::replaceAllUsesWithNewDeclaration(Module &M,
493 Function *OriginalFunction) {
494 auto &Ctx = M.getContext();
495 Function &F = *OriginalFunction;
496 FunctionType *FTy = F.getFunctionType();
497 Function *NF = Function::Create(Ty: FTy, Linkage: F.getLinkage(), AddrSpace: F.getAddressSpace());
498
499 NF->setName(F.getName() + ".varargs");
500
501 F.getParent()->getFunctionList().insert(where: F.getIterator(), New: NF);
502
503 AttrBuilder ParamAttrs(Ctx);
504 AttributeList Attrs = NF->getAttributes();
505 Attrs = Attrs.addParamAttributes(C&: Ctx, ArgNo: FTy->getNumParams(), B: ParamAttrs);
506 NF->setAttributes(Attrs);
507
508 OriginalFunction->replaceAllUsesWith(V: NF);
509 return NF;
510}
511
512Function *
513ExpandVariadics::deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder,
514 Function *OriginalFunction) {
515 Function &F = *OriginalFunction;
516 // The purpose here is split the variadic function F into two functions
517 // One is a variadic function that bundles the passed argument into a va_list
518 // and passes it to the second function. The second function does whatever
519 // the original F does, except that it takes a va_list instead of the ...
520
521 assert(expansionApplicableToFunction(M, &F));
522
523 auto &Ctx = M.getContext();
524
525 // Returned value isDeclaration() is equal to F.isDeclaration()
526 // but that property is not invariant throughout this function
527 const bool FunctionIsDefinition = !F.isDeclaration();
528
529 FunctionType *FTy = F.getFunctionType();
530 SmallVector<Type *> ArgTypes(FTy->params());
531 ArgTypes.push_back(Elt: ABI->vaListParameterType(M));
532
533 FunctionType *NFTy = inlinableVariadicFunctionType(M, FTy);
534 Function *NF = Function::Create(Ty: NFTy, Linkage: F.getLinkage(), AddrSpace: F.getAddressSpace());
535
536 // Note - same attribute handling as DeadArgumentElimination
537 NF->copyAttributesFrom(Src: &F);
538 NF->setComdat(F.getComdat());
539 F.getParent()->getFunctionList().insert(where: F.getIterator(), New: NF);
540 NF->setName(F.getName() + ".valist");
541
542 AttrBuilder ParamAttrs(Ctx);
543
544 AttributeList Attrs = NF->getAttributes();
545 Attrs = Attrs.addParamAttributes(C&: Ctx, ArgNo: NFTy->getNumParams() - 1, B: ParamAttrs);
546 NF->setAttributes(Attrs);
547
548 // Splice the implementation into the new function with minimal changes
549 if (FunctionIsDefinition) {
550 NF->splice(ToIt: NF->begin(), FromF: &F);
551
552 auto NewArg = NF->arg_begin();
553 for (Argument &Arg : F.args()) {
554 Arg.replaceAllUsesWith(V: NewArg);
555 NewArg->setName(Arg.getName()); // takeName without killing the old one
556 ++NewArg;
557 }
558 NewArg->setName("varargs");
559 }
560
561 SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;
562 F.getAllMetadata(MDs);
563 for (auto [KindID, Node] : MDs)
564 NF->addMetadata(KindID, MD&: *Node);
565 F.clearMetadata();
566
567 return NF;
568}
569
570Function *
571ExpandVariadics::defineVariadicWrapper(Module &M, IRBuilder<> &Builder,
572 Function *VariadicWrapper,
573 Function *FixedArityReplacement) {
574 auto &Ctx = Builder.getContext();
575 const DataLayout &DL = M.getDataLayout();
576 assert(VariadicWrapper->isDeclaration());
577 Function &F = *VariadicWrapper;
578
579 assert(F.isDeclaration());
580 Type *VaListTy = ABI->vaListType(Ctx);
581
582 auto *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: &F);
583 Builder.SetInsertPoint(BB);
584
585 AllocaInst *VaListInstance =
586 Builder.CreateAlloca(Ty: VaListTy, ArraySize: nullptr, Name: "va_start");
587
588 Builder.CreateLifetimeStart(Ptr: VaListInstance);
589
590 Builder.CreateIntrinsic(ID: Intrinsic::vastart, Types: {DL.getAllocaPtrType(Ctx)},
591 Args: {VaListInstance});
592
593 SmallVector<Value *> Args(llvm::make_pointer_range(Range: F.args()));
594
595 Type *ParameterType = ABI->vaListParameterType(M);
596 if (ABI->vaListPassedInSSARegister())
597 Args.push_back(Elt: Builder.CreateLoad(Ty: ParameterType, Ptr: VaListInstance));
598 else
599 Args.push_back(Elt: Builder.CreateAddrSpaceCast(V: VaListInstance, DestTy: ParameterType));
600
601 CallInst *Result = Builder.CreateCall(Callee: FixedArityReplacement, Args);
602
603 Builder.CreateIntrinsic(ID: Intrinsic::vaend, Types: {DL.getAllocaPtrType(Ctx)},
604 Args: {VaListInstance});
605 Builder.CreateLifetimeEnd(Ptr: VaListInstance);
606
607 if (Result->getType()->isVoidTy())
608 Builder.CreateRetVoid();
609 else
610 Builder.CreateRet(V: Result);
611
612 return VariadicWrapper;
613}
614
615bool ExpandVariadics::expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB,
616 FunctionType *VarargFunctionType,
617 Function *NF) {
618 bool Changed = false;
619 const DataLayout &DL = M.getDataLayout();
620
621 if (ABI->ignoreFunction(F: CB->getCalledFunction()))
622 return Changed;
623
624 if (!expansionApplicableToFunctionCall(CB)) {
625 if (rewriteABI())
626 report_fatal_error(reason: "Cannot lower callbase instruction");
627 return Changed;
628 }
629
630 // This is tricky. The call instruction's function type might not match
631 // the type of the caller. When optimising, can leave it unchanged.
632 // Webassembly detects that inconsistency and repairs it.
633 FunctionType *FuncType = CB->getFunctionType();
634 if (FuncType != VarargFunctionType) {
635 if (!rewriteABI())
636 return Changed;
637 FuncType = VarargFunctionType;
638 }
639
640 auto &Ctx = CB->getContext();
641
642 Align MaxFieldAlign(1);
643
644 // The strategy is to allocate a call frame containing the variadic
645 // arguments laid out such that a target specific va_list can be initialized
646 // with it, such that target specific va_arg instructions will correctly
647 // iterate over it. This means getting the alignment right and sometimes
648 // embedding a pointer to the value instead of embedding the value itself.
649
650 Function *CBF = CB->getParent()->getParent();
651
652 ExpandedCallFrame Frame;
653
654 uint64_t CurrentOffset = 0;
655
656 for (unsigned I = FuncType->getNumParams(), E = CB->arg_size(); I < E; ++I) {
657 Value *ArgVal = CB->getArgOperand(i: I);
658 const bool IsByVal = CB->paramHasAttr(ArgNo: I, Kind: Attribute::ByVal);
659 const bool IsByRef = CB->paramHasAttr(ArgNo: I, Kind: Attribute::ByRef);
660
661 // The type of the value being passed, decoded from byval/byref metadata if
662 // required
663 Type *const UnderlyingType = IsByVal ? CB->getParamByValType(ArgNo: I)
664 : IsByRef ? CB->getParamByRefType(ArgNo: I)
665 : ArgVal->getType();
666 const uint64_t UnderlyingSize =
667 DL.getTypeAllocSize(Ty: UnderlyingType).getFixedValue();
668
669 // The type to be written into the call frame
670 Type *FrameFieldType = UnderlyingType;
671
672 // The value to copy from when initialising the frame alloca
673 Value *SourceValue = ArgVal;
674
675 VariadicABIInfo::VAArgSlotInfo SlotInfo = ABI->slotInfo(DL, Parameter: UnderlyingType);
676
677 if (SlotInfo.Indirect) {
678 // The va_arg lowering loads through a pointer. Set up an alloca to aim
679 // that pointer at.
680 Builder.SetInsertPointPastAllocas(CBF);
681 Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());
682 Value *CallerCopy =
683 Builder.CreateAlloca(Ty: UnderlyingType, ArraySize: nullptr, Name: "IndirectAlloca");
684
685 Builder.SetInsertPoint(CB);
686 if (IsByVal)
687 Builder.CreateMemCpy(Dst: CallerCopy, DstAlign: {}, Src: ArgVal, SrcAlign: {}, Size: UnderlyingSize);
688 else
689 Builder.CreateStore(Val: ArgVal, Ptr: CallerCopy);
690
691 // Indirection now handled, pass the alloca ptr by value
692 FrameFieldType = DL.getAllocaPtrType(Ctx);
693 SourceValue = CallerCopy;
694 }
695
696 // Alignment of the value within the frame
697 // This probably needs to be controllable as a function of type
698 Align DataAlign = SlotInfo.DataAlign;
699
700 MaxFieldAlign = std::max(a: MaxFieldAlign, b: DataAlign);
701
702 uint64_t DataAlignV = DataAlign.value();
703 if (uint64_t Rem = CurrentOffset % DataAlignV) {
704 // Inject explicit padding to deal with alignment requirements
705 uint64_t Padding = DataAlignV - Rem;
706 Frame.padding(Ctx, By: Padding);
707 CurrentOffset += Padding;
708 }
709
710 if (SlotInfo.Indirect) {
711 Frame.store(Ctx, T: FrameFieldType, V: SourceValue);
712 } else {
713 if (IsByVal)
714 Frame.memcpy(Ctx, T: FrameFieldType, V: SourceValue, Bytes: UnderlyingSize);
715 else
716 Frame.store(Ctx, T: FrameFieldType, V: SourceValue);
717 }
718
719 CurrentOffset += DL.getTypeAllocSize(Ty: FrameFieldType).getFixedValue();
720 }
721
722 if (Frame.empty()) {
723 // Not passing any arguments, hopefully va_arg won't try to read any
724 // Creating a single byte frame containing nothing to point the va_list
725 // instance as that is less special-casey in the compiler and probably
726 // easier to interpret in a debugger.
727 Frame.padding(Ctx, By: 1);
728 }
729
730 StructType *VarargsTy = Frame.asStruct(Ctx, Name: CBF->getName());
731
732 // The struct instance needs to be at least MaxFieldAlign for the alignment of
733 // the fields to be correct at runtime. Use the native stack alignment instead
734 // if that's greater as that tends to give better codegen.
735 // This is an awkward way to guess whether there is a known stack alignment
736 // without hitting an assert in DL.getStackAlignment, 1024 is an arbitrary
737 // number likely to be greater than the natural stack alignment.
738 Align AllocaAlign = MaxFieldAlign;
739 if (MaybeAlign StackAlign = DL.getStackAlignment();
740 StackAlign && *StackAlign > AllocaAlign)
741 AllocaAlign = *StackAlign;
742
743 // Put the alloca to hold the variadic args in the entry basic block.
744 Builder.SetInsertPointPastAllocas(CBF);
745
746 // SetCurrentDebugLocation when the builder SetInsertPoint method does not
747 Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());
748
749 // The awkward construction here is to set the alignment on the instance
750 AllocaInst *Alloced = Builder.Insert(
751 I: new AllocaInst(VarargsTy, DL.getAllocaAddrSpace(), nullptr, AllocaAlign),
752 Name: "vararg_buffer");
753 Changed = true;
754 assert(Alloced->getAllocatedType() == VarargsTy);
755
756 // Initialize the fields in the struct
757 Builder.SetInsertPoint(CB);
758 Builder.CreateLifetimeStart(Ptr: Alloced);
759 Frame.initializeStructAlloca(DL, Builder, Alloced, VarargsTy);
760
761 const unsigned NumArgs = FuncType->getNumParams();
762 SmallVector<Value *> Args(CB->arg_begin(), CB->arg_begin() + NumArgs);
763
764 // Initialize a va_list pointing to that struct and pass it as the last
765 // argument
766 AllocaInst *VaList = nullptr;
767 {
768 if (!ABI->vaListPassedInSSARegister()) {
769 Type *VaListTy = ABI->vaListType(Ctx);
770 Builder.SetInsertPointPastAllocas(CBF);
771 Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());
772 VaList = Builder.CreateAlloca(Ty: VaListTy, ArraySize: nullptr, Name: "va_argument");
773 Builder.SetInsertPoint(CB);
774 Builder.CreateLifetimeStart(Ptr: VaList);
775 }
776 Builder.SetInsertPoint(CB);
777 Args.push_back(Elt: ABI->initializeVaList(M, Ctx, Builder, VaList, Buffer: Alloced));
778 }
779
780 // Attributes excluding any on the vararg arguments
781 AttributeList PAL = CB->getAttributes();
782 if (!PAL.isEmpty()) {
783 SmallVector<AttributeSet, 8> ArgAttrs;
784 for (unsigned ArgNo = 0; ArgNo < NumArgs; ArgNo++)
785 ArgAttrs.push_back(Elt: PAL.getParamAttrs(ArgNo));
786 PAL =
787 AttributeList::get(C&: Ctx, FnAttrs: PAL.getFnAttrs(), RetAttrs: PAL.getRetAttrs(), ArgAttrs);
788 }
789
790 SmallVector<OperandBundleDef, 1> OpBundles;
791 CB->getOperandBundlesAsDefs(Defs&: OpBundles);
792
793 CallBase *NewCB = nullptr;
794
795 if (CallInst *CI = dyn_cast<CallInst>(Val: CB)) {
796 Value *Dst = NF ? NF : CI->getCalledOperand();
797 FunctionType *NFTy = inlinableVariadicFunctionType(M, FTy: VarargFunctionType);
798
799 NewCB = CallInst::Create(Ty: NFTy, Func: Dst, Args, Bundles: OpBundles, NameStr: "", InsertBefore: CI->getIterator());
800
801 CallInst::TailCallKind TCK = CI->getTailCallKind();
802 assert(TCK != CallInst::TCK_MustTail);
803
804 // Can't tail call a function that is being passed a pointer to an alloca
805 if (TCK == CallInst::TCK_Tail)
806 TCK = CallInst::TCK_None;
807 CI->setTailCallKind(TCK);
808
809 } else {
810 llvm_unreachable("Unreachable when !expansionApplicableToFunctionCall()");
811 }
812
813 if (VaList)
814 Builder.CreateLifetimeEnd(Ptr: VaList);
815
816 Builder.CreateLifetimeEnd(Ptr: Alloced);
817
818 NewCB->setAttributes(PAL);
819 NewCB->takeName(V: CB);
820 NewCB->setCallingConv(CB->getCallingConv());
821 NewCB->setDebugLoc(DebugLoc());
822
823 // DeadArgElim and ArgPromotion copy exactly this metadata
824 NewCB->copyMetadata(SrcInst: *CB, WL: {LLVMContext::MD_prof, LLVMContext::MD_dbg});
825
826 CB->replaceAllUsesWith(V: NewCB);
827 CB->eraseFromParent();
828 return Changed;
829}
830
831bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder,
832 const DataLayout &DL,
833 VAStartInst *Inst) {
834 // Only removing va_start instructions that are not in variadic functions.
835 // Those would be rejected by the IR verifier before this pass.
836 // After splicing basic blocks from a variadic function into a fixed arity
837 // one the va_start that used to refer to the ... parameter still exist.
838 // There are also variadic functions that this pass did not change and
839 // va_start instances in the created single block wrapper functions.
840 // Replace exactly the instances in non-variadic functions as those are
841 // the ones to be fixed up to use the va_list passed as the final argument.
842
843 Function *ContainingFunction = Inst->getFunction();
844 if (ContainingFunction->isVarArg()) {
845 return false;
846 }
847
848 // The last argument is a vaListParameterType, either a va_list
849 // or a pointer to one depending on the target.
850 bool PassedByValue = ABI->vaListPassedInSSARegister();
851 Argument *PassedVaList =
852 ContainingFunction->getArg(i: ContainingFunction->arg_size() - 1);
853
854 // va_start takes a pointer to a va_list, e.g. one on the stack
855 Value *VaStartArg = Inst->getArgList();
856
857 Builder.SetInsertPoint(Inst);
858
859 if (PassedByValue) {
860 // The general thing to do is create an alloca, store the va_list argument
861 // to it, then create a va_copy. When vaCopyIsMemcpy(), this optimises to a
862 // store to the VaStartArg.
863 assert(ABI->vaCopyIsMemcpy());
864 Builder.CreateStore(Val: PassedVaList, Ptr: VaStartArg);
865 } else {
866
867 // Otherwise emit a vacopy to pick up target-specific handling if any
868 auto &Ctx = Builder.getContext();
869
870 Builder.CreateIntrinsic(ID: Intrinsic::vacopy, Types: {DL.getAllocaPtrType(Ctx)},
871 Args: {VaStartArg, PassedVaList});
872 }
873
874 Inst->eraseFromParent();
875 return true;
876}
877
878bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &,
879 VAEndInst *Inst) {
880 assert(ABI->vaEndIsNop());
881 Inst->eraseFromParent();
882 return true;
883}
884
885bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder,
886 const DataLayout &DL,
887 VACopyInst *Inst) {
888 assert(ABI->vaCopyIsMemcpy());
889 Builder.SetInsertPoint(Inst);
890
891 auto &Ctx = Builder.getContext();
892 Type *VaListTy = ABI->vaListType(Ctx);
893 uint64_t Size = DL.getTypeAllocSize(Ty: VaListTy).getFixedValue();
894
895 Builder.CreateMemCpy(Dst: Inst->getDest(), DstAlign: {}, Src: Inst->getSrc(), SrcAlign: {},
896 Size: Builder.getInt32(C: Size));
897
898 Inst->eraseFromParent();
899 return true;
900}
901
902struct Amdgpu final : public VariadicABIInfo {
903
904 bool enableForTarget() override { return true; }
905
906 bool vaListPassedInSSARegister() override { return true; }
907
908 Type *vaListType(LLVMContext &Ctx) override {
909 return PointerType::getUnqual(C&: Ctx);
910 }
911
912 Type *vaListParameterType(Module &M) override {
913 return PointerType::getUnqual(C&: M.getContext());
914 }
915
916 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
917 AllocaInst * /*va_list*/, Value *Buffer) override {
918 // Given Buffer, which is an AllocInst of vararg_buffer
919 // need to return something usable as parameter type
920 return Builder.CreateAddrSpaceCast(V: Buffer, DestTy: vaListParameterType(M));
921 }
922
923 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
924 return {.DataAlign: Align(4), .Indirect: false};
925 }
926};
927
928struct NVPTX final : public VariadicABIInfo {
929
930 bool enableForTarget() override { return true; }
931
932 bool vaListPassedInSSARegister() override { return true; }
933
934 Type *vaListType(LLVMContext &Ctx) override {
935 return PointerType::getUnqual(C&: Ctx);
936 }
937
938 Type *vaListParameterType(Module &M) override {
939 return PointerType::getUnqual(C&: M.getContext());
940 }
941
942 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
943 AllocaInst *, Value *Buffer) override {
944 return Builder.CreateAddrSpaceCast(V: Buffer, DestTy: vaListParameterType(M));
945 }
946
947 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
948 // NVPTX expects natural alignment in all cases. The variadic call ABI will
949 // handle promoting types to their appropriate size and alignment.
950 Align A = DL.getABITypeAlign(Ty: Parameter);
951 return {.DataAlign: A, .Indirect: false};
952 }
953};
954
955struct SPIRV final : public VariadicABIInfo {
956
957 bool enableForTarget() override { return true; }
958
959 bool vaListPassedInSSARegister() override { return true; }
960
961 Type *vaListType(LLVMContext &Ctx) override {
962 return PointerType::getUnqual(C&: Ctx);
963 }
964
965 Type *vaListParameterType(Module &M) override {
966 return PointerType::getUnqual(C&: M.getContext());
967 }
968
969 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
970 AllocaInst *, Value *Buffer) override {
971 return Builder.CreateAddrSpaceCast(V: Buffer, DestTy: vaListParameterType(M));
972 }
973
974 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
975 // Expects natural alignment in all cases. The variadic call ABI will handle
976 // promoting types to their appropriate size and alignment.
977 Align A = DL.getABITypeAlign(Ty: Parameter);
978 return {.DataAlign: A, .Indirect: false};
979 }
980
981 // The SPIR-V backend has special handling for SPIR-V mangled printf
982 // functions.
983 bool ignoreFunction(Function *F) override {
984 return F->getName().starts_with(Prefix: '_') && F->getName().contains(Other: "printf");
985 }
986};
987
988struct Wasm final : public VariadicABIInfo {
989
990 bool enableForTarget() override {
991 // Currently wasm is only used for testing.
992 return commandLineOverride();
993 }
994
995 bool vaListPassedInSSARegister() override { return true; }
996
997 Type *vaListType(LLVMContext &Ctx) override {
998 return PointerType::getUnqual(C&: Ctx);
999 }
1000
1001 Type *vaListParameterType(Module &M) override {
1002 return PointerType::getUnqual(C&: M.getContext());
1003 }
1004
1005 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
1006 AllocaInst * /*va_list*/, Value *Buffer) override {
1007 return Buffer;
1008 }
1009
1010 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
1011 LLVMContext &Ctx = Parameter->getContext();
1012 const unsigned MinAlign = 4;
1013 Align A = DL.getABITypeAlign(Ty: Parameter);
1014 if (A < MinAlign)
1015 A = Align(MinAlign);
1016
1017 if (auto *S = dyn_cast<StructType>(Val: Parameter)) {
1018 if (S->getNumElements() > 1) {
1019 return {.DataAlign: DL.getABITypeAlign(Ty: PointerType::getUnqual(C&: Ctx)), .Indirect: true};
1020 }
1021 }
1022
1023 return {.DataAlign: A, .Indirect: false};
1024 }
1025};
1026
1027std::unique_ptr<VariadicABIInfo> VariadicABIInfo::create(const Triple &T) {
1028 switch (T.getArch()) {
1029 case Triple::r600:
1030 case Triple::amdgcn: {
1031 return std::make_unique<Amdgpu>();
1032 }
1033
1034 case Triple::wasm32: {
1035 return std::make_unique<Wasm>();
1036 }
1037
1038 case Triple::nvptx:
1039 case Triple::nvptx64: {
1040 return std::make_unique<NVPTX>();
1041 }
1042
1043 case Triple::spirv:
1044 case Triple::spirv64: {
1045 return std::make_unique<SPIRV>();
1046 }
1047
1048 default:
1049 return {};
1050 }
1051}
1052
1053} // namespace
1054
1055char ExpandVariadics::ID = 0;
1056
1057INITIALIZE_PASS(ExpandVariadics, DEBUG_TYPE, "Expand variadic functions", false,
1058 false)
1059
1060ModulePass *llvm::createExpandVariadicsPass(ExpandVariadicsMode M) {
1061 return new ExpandVariadics(M);
1062}
1063
1064PreservedAnalyses ExpandVariadicsPass::run(Module &M, ModuleAnalysisManager &) {
1065 return ExpandVariadics(Mode).runOnModule(M) ? PreservedAnalyses::none()
1066 : PreservedAnalyses::all();
1067}
1068
1069ExpandVariadicsPass::ExpandVariadicsPass(ExpandVariadicsMode M) : Mode(M) {}
1070