1//===-- ExpandVariadicsPass.cpp --------------------------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is an optimization pass for variadic functions. If called from codegen,
10// it can serve as the implementation of variadic functions for a given target.
11//
12// The strategy is to turn the ... part of a variadic function into a va_list
13// and fix up the call sites. The majority of the pass is target independent.
14// The exceptions are the va_list type itself and the rules for where to store
15// variables in memory such that va_arg can iterate over them given a va_list.
16//
17// The majority of the plumbing is splitting the variadic function into a
18// single basic block that packs the variadic arguments into a va_list and
19// a second function that does the work of the original. That packing is
20// exactly what is done by va_start. Further, the transform from ... to va_list
21// replaced va_start with an operation to copy a va_list from the new argument,
22// which is exactly a va_copy. This is useful for reducing target-dependence.
23//
24// A va_list instance is a forward iterator, where the primary operation va_arg
25// is dereference-then-increment. This interface forces significant convergent
26// evolution between target specific implementations. The variation in runtime
27// data layout is limited to that representable by the iterator, parameterised
28// by the type passed to the va_arg instruction.
29//
30// Therefore the majority of the target specific subtlety is packing arguments
31// into a stack allocated buffer such that a va_list can be initialised with it
32// and the va_arg expansion for the target will find the arguments at runtime.
33//
34// The aggregate effect is to unblock other transforms, most critically the
35// general purpose inliner. Known calls to variadic functions become zero cost.
36//
37// Consistency with clang is primarily tested by emitting va_arg using clang
38// then expanding the variadic functions using this pass, followed by trying
39// to constant fold the functions to no-ops.
40//
41// Target specific behaviour is tested in IR - mainly checking that values are
42// put into positions in call frames that make sense for that particular target.
43//
44// There is one "clever" invariant in use. va_start intrinsics that are not
45// within a varidic functions are an error in the IR verifier. When this
46// transform moves blocks from a variadic function into a fixed arity one, it
47// moves va_start intrinsics along with everything else. That means that the
48// va_start intrinsics that need to be rewritten to use the trailing argument
49// are exactly those that are in non-variadic functions so no further state
50// is needed to distinguish those that need to be rewritten.
51//
52//===----------------------------------------------------------------------===//
53
54#include "llvm/Transforms/IPO/ExpandVariadics.h"
55#include "llvm/ADT/SmallVector.h"
56#include "llvm/Demangle/Demangle.h"
57#include "llvm/IR/IRBuilder.h"
58#include "llvm/IR/IntrinsicInst.h"
59#include "llvm/IR/Module.h"
60#include "llvm/IR/PassManager.h"
61#include "llvm/InitializePasses.h"
62#include "llvm/Pass.h"
63#include "llvm/Support/CommandLine.h"
64#include "llvm/TargetParser/Triple.h"
65#include "llvm/Transforms/Utils/ModuleUtils.h"
66
67#define DEBUG_TYPE "expand-variadics"
68
69using namespace llvm;
70
71namespace {
72
73cl::opt<ExpandVariadicsMode> ExpandVariadicsModeOption(
74 DEBUG_TYPE "-override", cl::desc("Override the behaviour of " DEBUG_TYPE),
75 cl::init(Val: ExpandVariadicsMode::Unspecified),
76 cl::values(clEnumValN(ExpandVariadicsMode::Unspecified, "unspecified",
77 "Use the implementation defaults"),
78 clEnumValN(ExpandVariadicsMode::Disable, "disable",
79 "Disable the pass entirely"),
80 clEnumValN(ExpandVariadicsMode::Optimize, "optimize",
81 "Optimise without changing ABI"),
82 clEnumValN(ExpandVariadicsMode::Lowering, "lowering",
83 "Change variadic calling convention")));
84
85bool commandLineOverride() {
86 return ExpandVariadicsModeOption != ExpandVariadicsMode::Unspecified;
87}
88
89// Instances of this class encapsulate the target-dependant behaviour as a
90// function of triple. Implementing a new ABI is adding a case to the switch
91// in create(llvm::Triple) at the end of this file.
92// This class may end up instantiated in TargetMachine instances, keeping it
93// here for now until enough targets are implemented for the API to evolve.
94class VariadicABIInfo {
95protected:
96 VariadicABIInfo() = default;
97
98public:
99 static std::unique_ptr<VariadicABIInfo> create(const Triple &T);
100
101 // Allow overriding whether the pass runs on a per-target basis
102 virtual bool enableForTarget() = 0;
103
104 // Whether a valist instance is passed by value or by address
105 // I.e. does it need to be alloca'ed and stored into, or can
106 // it be passed directly in a SSA register
107 virtual bool vaListPassedInSSARegister() = 0;
108
109 // The type of a va_list iterator object
110 virtual Type *vaListType(LLVMContext &Ctx) = 0;
111
112 // The type of a va_list as a function argument as lowered by C
113 virtual Type *vaListParameterType(Module &M) = 0;
114
115 // Initialize an allocated va_list object to point to an already
116 // initialized contiguous memory region.
117 // Return the value to pass as the va_list argument
118 virtual Value *initializeVaList(Module &M, LLVMContext &Ctx,
119 IRBuilder<> &Builder, AllocaInst *VaList,
120 Value *Buffer) = 0;
121
122 struct VAArgSlotInfo {
123 Align DataAlign; // With respect to the call frame
124 bool Indirect; // Passed via a pointer
125 };
126 virtual VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) = 0;
127
128 // Targets implemented so far all have the same trivial lowering for these
129 bool vaEndIsNop() { return true; }
130 bool vaCopyIsMemcpy() { return true; }
131
132 // Any additional address spaces used in va intrinsics that should be
133 // expanded.
134 virtual SmallVector<unsigned> getTargetSpecificVaIntrinAddrSpaces() const {
135 return {};
136 }
137
138 virtual ~VariadicABIInfo() = default;
139};
140
141class ExpandVariadics : public ModulePass {
142
143 // The pass construction sets the default to optimize when called from middle
144 // end and lowering when called from the backend. The command line variable
145 // overrides that. This is useful for testing and debugging. It also allows
146 // building an applications with variadic functions wholly removed if one
147 // has sufficient control over the dependencies, e.g. a statically linked
148 // clang that has no variadic function calls remaining in the binary.
149
150public:
151 static char ID;
152 const ExpandVariadicsMode Mode;
153 std::unique_ptr<VariadicABIInfo> ABI;
154
155 ExpandVariadics(ExpandVariadicsMode Mode)
156 : ModulePass(ID),
157 Mode(commandLineOverride() ? ExpandVariadicsModeOption : Mode) {}
158
159 StringRef getPassName() const override { return "Expand variadic functions"; }
160
161 bool rewriteABI() { return Mode == ExpandVariadicsMode::Lowering; }
162
163 template <typename T> bool isValidCallingConv(T *F) {
164 return F->getCallingConv() == CallingConv::C ||
165 F->getCallingConv() == CallingConv::SPIR_FUNC;
166 }
167
168 bool runOnModule(Module &M) override;
169
170 bool runOnFunction(Module &M, IRBuilder<> &Builder, Function *F);
171
172 Function *replaceAllUsesWithNewDeclaration(Module &M,
173 Function *OriginalFunction);
174
175 Function *deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder,
176 Function *OriginalFunction);
177
178 Function *defineVariadicWrapper(Module &M, IRBuilder<> &Builder,
179 Function *VariadicWrapper,
180 Function *FixedArityReplacement);
181
182 bool expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB, FunctionType *,
183 Function *NF);
184
185 // The intrinsic functions va_copy and va_end are removed unconditionally.
186 // They correspond to a memcpy and a no-op on all implemented targets.
187 // The va_start intrinsic is removed from basic blocks that were not created
188 // by this pass, some may remain if needed to maintain the external ABI.
189
190 template <Intrinsic::ID ID, typename InstructionType>
191 bool expandIntrinsicUsers(Module &M, IRBuilder<> &Builder,
192 PointerType *IntrinsicArgType) {
193 bool Changed = false;
194 const DataLayout &DL = M.getDataLayout();
195 if (Function *Intrinsic =
196 Intrinsic::getDeclarationIfExists(M: &M, id: ID, Tys: {IntrinsicArgType})) {
197 for (User *U : make_early_inc_range(Range: Intrinsic->users()))
198 if (auto *I = dyn_cast<InstructionType>(U))
199 Changed |= expandVAIntrinsicCall(Builder, DL, I);
200
201 if (Intrinsic->use_empty())
202 Intrinsic->eraseFromParent();
203 }
204 return Changed;
205 }
206
207 bool expandVAIntrinsicUsersWithAddrspace(Module &M, IRBuilder<> &Builder,
208 unsigned Addrspace) {
209 auto &Ctx = M.getContext();
210 PointerType *IntrinsicArgType = PointerType::get(C&: Ctx, AddressSpace: Addrspace);
211 bool Changed = false;
212
213 // expand vastart before vacopy as vastart may introduce a vacopy
214 Changed |= expandIntrinsicUsers<Intrinsic::vastart, VAStartInst>(
215 M, Builder, IntrinsicArgType);
216 Changed |= expandIntrinsicUsers<Intrinsic::vaend, VAEndInst>(
217 M, Builder, IntrinsicArgType);
218 Changed |= expandIntrinsicUsers<Intrinsic::vacopy, VACopyInst>(
219 M, Builder, IntrinsicArgType);
220 return Changed;
221 }
222
223 bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL,
224 VAStartInst *Inst);
225
226 bool expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &,
227 VAEndInst *Inst);
228
229 bool expandVAIntrinsicCall(IRBuilder<> &Builder, const DataLayout &DL,
230 VACopyInst *Inst);
231
232 FunctionType *inlinableVariadicFunctionType(Module &M, FunctionType *FTy) {
233 // The type of "FTy" with the ... removed and a va_list appended
234 SmallVector<Type *> ArgTypes(FTy->params());
235 ArgTypes.push_back(Elt: ABI->vaListParameterType(M));
236 return FunctionType::get(Result: FTy->getReturnType(), Params: ArgTypes,
237 /*IsVarArgs=*/isVarArg: false);
238 }
239
240 bool expansionApplicableToFunction(Module &M, Function *F) {
241 if (F->isIntrinsic() || !F->isVarArg() ||
242 F->hasFnAttribute(Kind: Attribute::Naked))
243 return false;
244
245 if (!isValidCallingConv(F))
246 return false;
247
248 if (rewriteABI())
249 return true;
250
251 if (!F->hasExactDefinition())
252 return false;
253
254 return true;
255 }
256
257 bool expansionApplicableToFunctionCall(CallBase *CB) {
258 if (CallInst *CI = dyn_cast<CallInst>(Val: CB)) {
259 if (CI->isMustTailCall()) {
260 // Cannot expand musttail calls
261 return false;
262 }
263
264 if (!isValidCallingConv(F: CI))
265 return false;
266
267 return true;
268 }
269
270 if (isa<InvokeInst>(Val: CB)) {
271 // Invoke not implemented in initial implementation of pass
272 return false;
273 }
274
275 // Other unimplemented derivative of CallBase
276 return false;
277 }
278
279 class ExpandedCallFrame {
280 // Helper for constructing an alloca instance containing the arguments bound
281 // to the variadic ... parameter, rearranged to allow indexing through a
282 // va_list iterator
283 enum { N = 4 };
284 SmallVector<Type *, N> FieldTypes;
285 enum Tag { Store, Memcpy, Padding };
286 SmallVector<std::tuple<Value *, uint64_t, Tag>, N> Source;
287
288 template <Tag tag> void append(Type *FieldType, Value *V, uint64_t Bytes) {
289 FieldTypes.push_back(Elt: FieldType);
290 Source.push_back(Elt: {V, Bytes, tag});
291 }
292
293 public:
294 void store(LLVMContext &Ctx, Type *T, Value *V) { append<Store>(FieldType: T, V, Bytes: 0); }
295
296 void memcpy(LLVMContext &Ctx, Type *T, Value *V, uint64_t Bytes) {
297 append<Memcpy>(FieldType: T, V, Bytes);
298 }
299
300 void padding(LLVMContext &Ctx, uint64_t By) {
301 append<Padding>(FieldType: ArrayType::get(ElementType: Type::getInt8Ty(C&: Ctx), NumElements: By), V: nullptr, Bytes: 0);
302 }
303
304 size_t size() const { return FieldTypes.size(); }
305 bool empty() const { return FieldTypes.empty(); }
306
307 StructType *asStruct(LLVMContext &Ctx, StringRef Name) {
308 const bool IsPacked = true;
309 return StructType::create(Context&: Ctx, Elements: FieldTypes,
310 Name: (Twine(Name) + ".vararg").str(), isPacked: IsPacked);
311 }
312
313 void initializeStructAlloca(const DataLayout &DL, IRBuilder<> &Builder,
314 AllocaInst *Alloced, StructType *VarargsTy) {
315
316 for (size_t I = 0; I < size(); I++) {
317
318 auto [V, bytes, tag] = Source[I];
319
320 if (tag == Padding) {
321 assert(V == nullptr);
322 continue;
323 }
324
325 auto Dst = Builder.CreateStructGEP(Ty: VarargsTy, Ptr: Alloced, Idx: I);
326
327 assert(V != nullptr);
328
329 if (tag == Store)
330 Builder.CreateStore(Val: V, Ptr: Dst);
331
332 if (tag == Memcpy)
333 Builder.CreateMemCpy(Dst, DstAlign: {}, Src: V, SrcAlign: {}, Size: bytes);
334 }
335 }
336 };
337};
338
339bool ExpandVariadics::runOnModule(Module &M) {
340 bool Changed = false;
341 if (Mode == ExpandVariadicsMode::Disable)
342 return Changed;
343
344 Triple TT(M.getTargetTriple());
345 ABI = VariadicABIInfo::create(T: TT);
346 if (!ABI)
347 return Changed;
348
349 if (!ABI->enableForTarget())
350 return Changed;
351
352 auto &Ctx = M.getContext();
353 const DataLayout &DL = M.getDataLayout();
354 IRBuilder<> Builder(Ctx);
355
356 // Lowering needs to run on all functions exactly once.
357 // Optimize could run on functions containing va_start exactly once.
358 for (Function &F : make_early_inc_range(Range&: M))
359 Changed |= runOnFunction(M, Builder, F: &F);
360
361 // After runOnFunction, all known calls to known variadic functions have been
362 // replaced. va_start intrinsics are presently (and invalidly!) only present
363 // in functions that used to be variadic and have now been replaced to take a
364 // va_list instead. If lowering as opposed to optimising, calls to unknown
365 // variadic functions have also been replaced.
366
367 {
368 unsigned Addrspace = 0;
369 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);
370
371 Addrspace = DL.getAllocaAddrSpace();
372 if (Addrspace != 0)
373 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace);
374
375 // Process any addrspaces targets declare to be important.
376 const SmallVector<unsigned> &TargetASVec =
377 ABI->getTargetSpecificVaIntrinAddrSpaces();
378 for (unsigned TargetAS : TargetASVec) {
379 if (TargetAS == 0 || TargetAS == DL.getAllocaAddrSpace())
380 continue;
381 Changed |= expandVAIntrinsicUsersWithAddrspace(M, Builder, Addrspace: TargetAS);
382 }
383 }
384
385 if (Mode != ExpandVariadicsMode::Lowering)
386 return Changed;
387
388 for (Function &F : make_early_inc_range(Range&: M)) {
389 if (F.isDeclaration())
390 continue;
391
392 // Now need to track down indirect calls. Can't find those
393 // by walking uses of variadic functions, need to crawl the instruction
394 // stream. Fortunately this is only necessary for the ABI rewrite case.
395 for (BasicBlock &BB : F) {
396 for (Instruction &I : make_early_inc_range(Range&: BB)) {
397 if (CallBase *CB = dyn_cast<CallBase>(Val: &I)) {
398 if (CB->isIndirectCall()) {
399 FunctionType *FTy = CB->getFunctionType();
400 if (FTy->isVarArg())
401 Changed |= expandCall(M, Builder, CB, FTy, /*NF=*/nullptr);
402 }
403 }
404 }
405 }
406 }
407
408 return Changed;
409}
410
411bool ExpandVariadics::runOnFunction(Module &M, IRBuilder<> &Builder,
412 Function *OriginalFunction) {
413 bool Changed = false;
414
415 if (!expansionApplicableToFunction(M, F: OriginalFunction))
416 return Changed;
417
418 [[maybe_unused]] const bool OriginalFunctionIsDeclaration =
419 OriginalFunction->isDeclaration();
420 assert(rewriteABI() || !OriginalFunctionIsDeclaration);
421
422 // Declare a new function and redirect every use to that new function
423 Function *VariadicWrapper =
424 replaceAllUsesWithNewDeclaration(M, OriginalFunction);
425 assert(VariadicWrapper->isDeclaration());
426 assert(OriginalFunction->use_empty());
427
428 // Create a new function taking va_list containing the implementation of the
429 // original
430 Function *FixedArityReplacement =
431 deriveFixedArityReplacement(M, Builder, OriginalFunction);
432 assert(OriginalFunction->isDeclaration());
433 assert(FixedArityReplacement->isDeclaration() ==
434 OriginalFunctionIsDeclaration);
435 assert(VariadicWrapper->isDeclaration());
436
437 // Create a single block forwarding wrapper that turns a ... into a va_list
438 [[maybe_unused]] Function *VariadicWrapperDefine =
439 defineVariadicWrapper(M, Builder, VariadicWrapper, FixedArityReplacement);
440 assert(VariadicWrapperDefine == VariadicWrapper);
441 assert(!VariadicWrapper->isDeclaration());
442
443 // Add the prof metadata from the original function to the wrapper. Because
444 // FixedArityReplacement is the owner of original function's prof metadata
445 // after the splice, we need to transfer it to VariadicWrapper.
446 VariadicWrapper->setMetadata(
447 KindID: LLVMContext::MD_prof,
448 Node: FixedArityReplacement->getMetadata(KindID: LLVMContext::MD_prof));
449
450 // We now have:
451 // 1. the original function, now as a declaration with no uses
452 // 2. a variadic function that unconditionally calls a fixed arity replacement
453 // 3. a fixed arity function equivalent to the original function
454
455 // Replace known calls to the variadic with calls to the va_list equivalent
456 for (User *U : make_early_inc_range(Range: VariadicWrapper->users())) {
457 if (CallBase *CB = dyn_cast<CallBase>(Val: U)) {
458 Value *CalledOperand = CB->getCalledOperand();
459 if (VariadicWrapper == CalledOperand)
460 Changed |=
461 expandCall(M, Builder, CB, VariadicWrapper->getFunctionType(),
462 NF: FixedArityReplacement);
463 }
464 }
465
466 // The original function will be erased.
467 // One of the two new functions will become a replacement for the original.
468 // When preserving the ABI, the other is an internal implementation detail.
469 // When rewriting the ABI, RAUW then the variadic one.
470 Function *const ExternallyAccessible =
471 rewriteABI() ? FixedArityReplacement : VariadicWrapper;
472 Function *const InternalOnly =
473 rewriteABI() ? VariadicWrapper : FixedArityReplacement;
474
475 // The external function is the replacement for the original
476 ExternallyAccessible->setLinkage(OriginalFunction->getLinkage());
477 ExternallyAccessible->setVisibility(OriginalFunction->getVisibility());
478 ExternallyAccessible->setComdat(OriginalFunction->getComdat());
479 ExternallyAccessible->takeName(V: OriginalFunction);
480
481 // Annotate the internal one as internal
482 InternalOnly->setVisibility(GlobalValue::DefaultVisibility);
483 InternalOnly->setLinkage(GlobalValue::InternalLinkage);
484
485 // The original is unused and obsolete
486 OriginalFunction->eraseFromParent();
487
488 InternalOnly->removeDeadConstantUsers();
489
490 if (rewriteABI()) {
491 // All known calls to the function have been removed by expandCall
492 // Resolve everything else by replaceAllUsesWith
493 VariadicWrapper->replaceAllUsesWith(V: FixedArityReplacement);
494 VariadicWrapper->eraseFromParent();
495 }
496
497 return Changed;
498}
499
500Function *
501ExpandVariadics::replaceAllUsesWithNewDeclaration(Module &M,
502 Function *OriginalFunction) {
503 auto &Ctx = M.getContext();
504 Function &F = *OriginalFunction;
505 FunctionType *FTy = F.getFunctionType();
506 Function *NF = Function::Create(Ty: FTy, Linkage: F.getLinkage(), AddrSpace: F.getAddressSpace());
507
508 NF->setName(F.getName() + ".varargs");
509
510 F.getParent()->getFunctionList().insert(where: F.getIterator(), New: NF);
511
512 AttrBuilder ParamAttrs(Ctx);
513 AttributeList Attrs = NF->getAttributes();
514 Attrs = Attrs.addParamAttributes(C&: Ctx, ArgNo: FTy->getNumParams(), B: ParamAttrs);
515 NF->setAttributes(Attrs);
516
517 OriginalFunction->replaceAllUsesWith(V: NF);
518 return NF;
519}
520
521Function *
522ExpandVariadics::deriveFixedArityReplacement(Module &M, IRBuilder<> &Builder,
523 Function *OriginalFunction) {
524 Function &F = *OriginalFunction;
525 // The purpose here is split the variadic function F into two functions
526 // One is a variadic function that bundles the passed argument into a va_list
527 // and passes it to the second function. The second function does whatever
528 // the original F does, except that it takes a va_list instead of the ...
529
530 assert(expansionApplicableToFunction(M, &F));
531
532 auto &Ctx = M.getContext();
533
534 // Returned value isDeclaration() is equal to F.isDeclaration()
535 // but that property is not invariant throughout this function
536 const bool FunctionIsDefinition = !F.isDeclaration();
537
538 FunctionType *FTy = F.getFunctionType();
539 SmallVector<Type *> ArgTypes(FTy->params());
540 ArgTypes.push_back(Elt: ABI->vaListParameterType(M));
541
542 FunctionType *NFTy = inlinableVariadicFunctionType(M, FTy);
543 Function *NF = Function::Create(Ty: NFTy, Linkage: F.getLinkage(), AddrSpace: F.getAddressSpace());
544
545 // Note - same attribute handling as DeadArgumentElimination
546 NF->copyAttributesFrom(Src: &F);
547 NF->setComdat(F.getComdat());
548 F.getParent()->getFunctionList().insert(where: F.getIterator(), New: NF);
549 NF->setName(F.getName() + ".valist");
550
551 AttrBuilder ParamAttrs(Ctx);
552
553 AttributeList Attrs = NF->getAttributes();
554 Attrs = Attrs.addParamAttributes(C&: Ctx, ArgNo: NFTy->getNumParams() - 1, B: ParamAttrs);
555 NF->setAttributes(Attrs);
556
557 // Splice the implementation into the new function with minimal changes
558 if (FunctionIsDefinition) {
559 NF->splice(ToIt: NF->begin(), FromF: &F);
560
561 auto NewArg = NF->arg_begin();
562 for (Argument &Arg : F.args()) {
563 Arg.replaceAllUsesWith(V: NewArg);
564 NewArg->setName(Arg.getName()); // takeName without killing the old one
565 ++NewArg;
566 }
567 NewArg->setName("varargs");
568 }
569
570 SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;
571 F.getAllMetadata(MDs);
572 for (auto [KindID, Node] : MDs)
573 NF->addMetadata(KindID, MD&: *Node);
574 F.clearMetadata();
575
576 return NF;
577}
578
579Function *
580ExpandVariadics::defineVariadicWrapper(Module &M, IRBuilder<> &Builder,
581 Function *VariadicWrapper,
582 Function *FixedArityReplacement) {
583 auto &Ctx = Builder.getContext();
584 const DataLayout &DL = M.getDataLayout();
585 assert(VariadicWrapper->isDeclaration());
586 Function &F = *VariadicWrapper;
587
588 assert(F.isDeclaration());
589 Type *VaListTy = ABI->vaListType(Ctx);
590
591 auto *BB = BasicBlock::Create(Context&: Ctx, Name: "entry", Parent: &F);
592 Builder.SetInsertPoint(BB);
593
594 AllocaInst *VaListInstance =
595 Builder.CreateAlloca(Ty: VaListTy, ArraySize: nullptr, Name: "va_start");
596
597 Builder.CreateLifetimeStart(Ptr: VaListInstance);
598
599 Builder.CreateIntrinsic(ID: Intrinsic::vastart, Types: {DL.getAllocaPtrType(Ctx)},
600 Args: {VaListInstance});
601
602 SmallVector<Value *> Args(llvm::make_pointer_range(Range: F.args()));
603
604 Type *ParameterType = ABI->vaListParameterType(M);
605 if (ABI->vaListPassedInSSARegister())
606 Args.push_back(Elt: Builder.CreateLoad(Ty: ParameterType, Ptr: VaListInstance));
607 else
608 Args.push_back(Elt: Builder.CreateAddrSpaceCast(V: VaListInstance, DestTy: ParameterType));
609
610 CallInst *Result = Builder.CreateCall(Callee: FixedArityReplacement, Args);
611
612 Builder.CreateIntrinsic(ID: Intrinsic::vaend, Types: {DL.getAllocaPtrType(Ctx)},
613 Args: {VaListInstance});
614 Builder.CreateLifetimeEnd(Ptr: VaListInstance);
615
616 if (Result->getType()->isVoidTy())
617 Builder.CreateRetVoid();
618 else
619 Builder.CreateRet(V: Result);
620
621 return VariadicWrapper;
622}
623
624bool ExpandVariadics::expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB,
625 FunctionType *VarargFunctionType,
626 Function *NF) {
627 bool Changed = false;
628 const DataLayout &DL = M.getDataLayout();
629
630 if (!expansionApplicableToFunctionCall(CB)) {
631 if (rewriteABI())
632 report_fatal_error(reason: "Cannot lower callbase instruction");
633 return Changed;
634 }
635
636 // This is tricky. The call instruction's function type might not match
637 // the type of the caller. When optimising, can leave it unchanged.
638 // Webassembly detects that inconsistency and repairs it.
639 FunctionType *FuncType = CB->getFunctionType();
640 if (FuncType != VarargFunctionType) {
641 if (!rewriteABI())
642 return Changed;
643 FuncType = VarargFunctionType;
644 }
645
646 auto &Ctx = CB->getContext();
647
648 Align MaxFieldAlign(1);
649
650 // The strategy is to allocate a call frame containing the variadic
651 // arguments laid out such that a target specific va_list can be initialized
652 // with it, such that target specific va_arg instructions will correctly
653 // iterate over it. This means getting the alignment right and sometimes
654 // embedding a pointer to the value instead of embedding the value itself.
655
656 Function *CBF = CB->getParent()->getParent();
657
658 ExpandedCallFrame Frame;
659
660 uint64_t CurrentOffset = 0;
661
662 for (unsigned I = FuncType->getNumParams(), E = CB->arg_size(); I < E; ++I) {
663 Value *ArgVal = CB->getArgOperand(i: I);
664 const bool IsByVal = CB->paramHasAttr(ArgNo: I, Kind: Attribute::ByVal);
665 const bool IsByRef = CB->paramHasAttr(ArgNo: I, Kind: Attribute::ByRef);
666
667 // The type of the value being passed, decoded from byval/byref metadata if
668 // required
669 Type *const UnderlyingType = IsByVal ? CB->getParamByValType(ArgNo: I)
670 : IsByRef ? CB->getParamByRefType(ArgNo: I)
671 : ArgVal->getType();
672 const uint64_t UnderlyingSize =
673 DL.getTypeAllocSize(Ty: UnderlyingType).getFixedValue();
674
675 // The type to be written into the call frame
676 Type *FrameFieldType = UnderlyingType;
677
678 // The value to copy from when initialising the frame alloca
679 Value *SourceValue = ArgVal;
680
681 VariadicABIInfo::VAArgSlotInfo SlotInfo = ABI->slotInfo(DL, Parameter: UnderlyingType);
682
683 if (SlotInfo.Indirect) {
684 // The va_arg lowering loads through a pointer. Set up an alloca to aim
685 // that pointer at.
686 Builder.SetInsertPointPastAllocas(CBF);
687 Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());
688 Value *CallerCopy =
689 Builder.CreateAlloca(Ty: UnderlyingType, ArraySize: nullptr, Name: "IndirectAlloca");
690
691 Builder.SetInsertPoint(CB);
692 if (IsByVal)
693 Builder.CreateMemCpy(Dst: CallerCopy, DstAlign: {}, Src: ArgVal, SrcAlign: {}, Size: UnderlyingSize);
694 else
695 Builder.CreateStore(Val: ArgVal, Ptr: CallerCopy);
696
697 // Indirection now handled, pass the alloca ptr by value
698 FrameFieldType = DL.getAllocaPtrType(Ctx);
699 SourceValue = CallerCopy;
700 }
701
702 // Alignment of the value within the frame
703 // This probably needs to be controllable as a function of type
704 Align DataAlign = SlotInfo.DataAlign;
705
706 MaxFieldAlign = std::max(a: MaxFieldAlign, b: DataAlign);
707
708 uint64_t DataAlignV = DataAlign.value();
709 if (uint64_t Rem = CurrentOffset % DataAlignV) {
710 // Inject explicit padding to deal with alignment requirements
711 uint64_t Padding = DataAlignV - Rem;
712 Frame.padding(Ctx, By: Padding);
713 CurrentOffset += Padding;
714 }
715
716 if (SlotInfo.Indirect) {
717 Frame.store(Ctx, T: FrameFieldType, V: SourceValue);
718 } else {
719 if (IsByVal)
720 Frame.memcpy(Ctx, T: FrameFieldType, V: SourceValue, Bytes: UnderlyingSize);
721 else
722 Frame.store(Ctx, T: FrameFieldType, V: SourceValue);
723 }
724
725 CurrentOffset += DL.getTypeAllocSize(Ty: FrameFieldType).getFixedValue();
726 }
727
728 if (Frame.empty()) {
729 // Not passing any arguments, hopefully va_arg won't try to read any
730 // Creating a single byte frame containing nothing to point the va_list
731 // instance as that is less special-casey in the compiler and probably
732 // easier to interpret in a debugger.
733 Frame.padding(Ctx, By: 1);
734 }
735
736 StructType *VarargsTy = Frame.asStruct(Ctx, Name: CBF->getName());
737
738 // The struct instance needs to be at least MaxFieldAlign for the alignment of
739 // the fields to be correct at runtime. Use the native stack alignment instead
740 // if that's greater as that tends to give better codegen.
741 // This is an awkward way to guess whether there is a known stack alignment
742 // without hitting an assert in DL.getStackAlignment, 1024 is an arbitrary
743 // number likely to be greater than the natural stack alignment.
744 Align AllocaAlign = MaxFieldAlign;
745 if (MaybeAlign StackAlign = DL.getStackAlignment();
746 StackAlign && *StackAlign > AllocaAlign)
747 AllocaAlign = *StackAlign;
748
749 // Put the alloca to hold the variadic args in the entry basic block.
750 Builder.SetInsertPointPastAllocas(CBF);
751
752 // SetCurrentDebugLocation when the builder SetInsertPoint method does not
753 Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());
754
755 // The awkward construction here is to set the alignment on the instance
756 AllocaInst *Alloced = Builder.Insert(
757 I: new AllocaInst(VarargsTy, DL.getAllocaAddrSpace(), nullptr, AllocaAlign),
758 Name: "vararg_buffer");
759 Changed = true;
760 assert(Alloced->getAllocatedType() == VarargsTy);
761
762 // Initialize the fields in the struct
763 Builder.SetInsertPoint(CB);
764 Builder.CreateLifetimeStart(Ptr: Alloced);
765 Frame.initializeStructAlloca(DL, Builder, Alloced, VarargsTy);
766
767 const unsigned NumArgs = FuncType->getNumParams();
768 SmallVector<Value *> Args(CB->arg_begin(), CB->arg_begin() + NumArgs);
769
770 // Initialize a va_list pointing to that struct and pass it as the last
771 // argument
772 AllocaInst *VaList = nullptr;
773 {
774 if (!ABI->vaListPassedInSSARegister()) {
775 Type *VaListTy = ABI->vaListType(Ctx);
776 Builder.SetInsertPointPastAllocas(CBF);
777 Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());
778 VaList = Builder.CreateAlloca(Ty: VaListTy, ArraySize: nullptr, Name: "va_argument");
779 Builder.SetInsertPoint(CB);
780 Builder.CreateLifetimeStart(Ptr: VaList);
781 }
782 Builder.SetInsertPoint(CB);
783 Args.push_back(Elt: ABI->initializeVaList(M, Ctx, Builder, VaList, Buffer: Alloced));
784 }
785
786 // Attributes excluding any on the vararg arguments
787 AttributeList PAL = CB->getAttributes();
788 if (!PAL.isEmpty()) {
789 SmallVector<AttributeSet, 8> ArgAttrs;
790 for (unsigned ArgNo = 0; ArgNo < NumArgs; ArgNo++)
791 ArgAttrs.push_back(Elt: PAL.getParamAttrs(ArgNo));
792 PAL =
793 AttributeList::get(C&: Ctx, FnAttrs: PAL.getFnAttrs(), RetAttrs: PAL.getRetAttrs(), ArgAttrs);
794 }
795
796 SmallVector<OperandBundleDef, 1> OpBundles;
797 CB->getOperandBundlesAsDefs(Defs&: OpBundles);
798
799 CallBase *NewCB = nullptr;
800
801 if (CallInst *CI = dyn_cast<CallInst>(Val: CB)) {
802 Value *Dst = NF ? NF : CI->getCalledOperand();
803 FunctionType *NFTy = inlinableVariadicFunctionType(M, FTy: VarargFunctionType);
804
805 NewCB = CallInst::Create(Ty: NFTy, Func: Dst, Args, Bundles: OpBundles, NameStr: "", InsertBefore: CI->getIterator());
806
807 CallInst::TailCallKind TCK = CI->getTailCallKind();
808 assert(TCK != CallInst::TCK_MustTail);
809
810 // Can't tail call a function that is being passed a pointer to an alloca
811 if (TCK == CallInst::TCK_Tail)
812 TCK = CallInst::TCK_None;
813 CI->setTailCallKind(TCK);
814
815 } else {
816 llvm_unreachable("Unreachable when !expansionApplicableToFunctionCall()");
817 }
818
819 if (VaList)
820 Builder.CreateLifetimeEnd(Ptr: VaList);
821
822 Builder.CreateLifetimeEnd(Ptr: Alloced);
823
824 NewCB->setAttributes(PAL);
825 NewCB->takeName(V: CB);
826 NewCB->setCallingConv(CB->getCallingConv());
827 NewCB->setDebugLoc(DebugLoc());
828
829 // DeadArgElim and ArgPromotion copy exactly this metadata
830 NewCB->copyMetadata(SrcInst: *CB, WL: {LLVMContext::MD_prof, LLVMContext::MD_dbg});
831
832 CB->replaceAllUsesWith(V: NewCB);
833 CB->eraseFromParent();
834 return Changed;
835}
836
837bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder,
838 const DataLayout &DL,
839 VAStartInst *Inst) {
840 // Only removing va_start instructions that are not in variadic functions.
841 // Those would be rejected by the IR verifier before this pass.
842 // After splicing basic blocks from a variadic function into a fixed arity
843 // one the va_start that used to refer to the ... parameter still exist.
844 // There are also variadic functions that this pass did not change and
845 // va_start instances in the created single block wrapper functions.
846 // Replace exactly the instances in non-variadic functions as those are
847 // the ones to be fixed up to use the va_list passed as the final argument.
848
849 Function *ContainingFunction = Inst->getFunction();
850 if (ContainingFunction->isVarArg()) {
851 return false;
852 }
853
854 // The last argument is a vaListParameterType, either a va_list
855 // or a pointer to one depending on the target.
856 bool PassedByValue = ABI->vaListPassedInSSARegister();
857 Argument *PassedVaList =
858 ContainingFunction->getArg(i: ContainingFunction->arg_size() - 1);
859
860 // va_start takes a pointer to a va_list, e.g. one on the stack
861 Value *VaStartArg = Inst->getArgList();
862
863 Builder.SetInsertPoint(Inst);
864
865 if (PassedByValue) {
866 // The general thing to do is create an alloca, store the va_list argument
867 // to it, then create a va_copy. When vaCopyIsMemcpy(), this optimises to a
868 // store to the VaStartArg.
869 assert(ABI->vaCopyIsMemcpy());
870 Builder.CreateStore(Val: PassedVaList, Ptr: VaStartArg);
871 } else {
872
873 // Otherwise emit a vacopy to pick up target-specific handling if any
874 auto &Ctx = Builder.getContext();
875
876 Builder.CreateIntrinsic(ID: Intrinsic::vacopy, Types: {DL.getAllocaPtrType(Ctx)},
877 Args: {VaStartArg, PassedVaList});
878 }
879
880 Inst->eraseFromParent();
881 return true;
882}
883
884bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &, const DataLayout &,
885 VAEndInst *Inst) {
886 assert(ABI->vaEndIsNop());
887 Inst->eraseFromParent();
888 return true;
889}
890
891bool ExpandVariadics::expandVAIntrinsicCall(IRBuilder<> &Builder,
892 const DataLayout &DL,
893 VACopyInst *Inst) {
894 assert(ABI->vaCopyIsMemcpy());
895 Builder.SetInsertPoint(Inst);
896
897 auto &Ctx = Builder.getContext();
898 Type *VaListTy = ABI->vaListType(Ctx);
899 uint64_t Size = DL.getTypeAllocSize(Ty: VaListTy).getFixedValue();
900
901 Builder.CreateMemCpy(Dst: Inst->getDest(), DstAlign: {}, Src: Inst->getSrc(), SrcAlign: {},
902 Size: Builder.getInt32(C: Size));
903
904 Inst->eraseFromParent();
905 return true;
906}
907
908struct Amdgpu final : public VariadicABIInfo {
909
910 bool enableForTarget() override { return true; }
911
912 bool vaListPassedInSSARegister() override { return true; }
913
914 Type *vaListType(LLVMContext &Ctx) override {
915 return PointerType::getUnqual(C&: Ctx);
916 }
917
918 Type *vaListParameterType(Module &M) override {
919 return PointerType::getUnqual(C&: M.getContext());
920 }
921
922 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
923 AllocaInst * /*va_list*/, Value *Buffer) override {
924 // Given Buffer, which is an AllocInst of vararg_buffer
925 // need to return something usable as parameter type
926 return Builder.CreateAddrSpaceCast(V: Buffer, DestTy: vaListParameterType(M));
927 }
928
929 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
930 return {.DataAlign: Align(4), .Indirect: false};
931 }
932};
933
934struct NVPTX final : public VariadicABIInfo {
935
936 bool enableForTarget() override { return true; }
937
938 bool vaListPassedInSSARegister() override { return true; }
939
940 Type *vaListType(LLVMContext &Ctx) override {
941 return PointerType::getUnqual(C&: Ctx);
942 }
943
944 Type *vaListParameterType(Module &M) override {
945 return PointerType::getUnqual(C&: M.getContext());
946 }
947
948 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
949 AllocaInst *, Value *Buffer) override {
950 return Builder.CreateAddrSpaceCast(V: Buffer, DestTy: vaListParameterType(M));
951 }
952
953 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
954 // NVPTX expects natural alignment in all cases. The variadic call ABI will
955 // handle promoting types to their appropriate size and alignment.
956 Align A = DL.getABITypeAlign(Ty: Parameter);
957 return {.DataAlign: A, .Indirect: false};
958 }
959};
960
961struct SPIRV final : public VariadicABIInfo {
962
963 bool enableForTarget() override { return true; }
964
965 bool vaListPassedInSSARegister() override { return true; }
966
967 Type *vaListType(LLVMContext &Ctx) override {
968 return PointerType::getUnqual(C&: Ctx);
969 }
970
971 Type *vaListParameterType(Module &M) override {
972 return PointerType::getUnqual(C&: M.getContext());
973 }
974
975 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
976 AllocaInst *, Value *Buffer) override {
977 return Builder.CreateAddrSpaceCast(V: Buffer, DestTy: vaListParameterType(M));
978 }
979
980 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
981 // Expects natural alignment in all cases. The variadic call ABI will handle
982 // promoting types to their appropriate size and alignment.
983 Align A = DL.getABITypeAlign(Ty: Parameter);
984 return {.DataAlign: A, .Indirect: false};
985 }
986
987 // We will likely see va intrinsics in the generic addrspace (4).
988 SmallVector<unsigned> getTargetSpecificVaIntrinAddrSpaces() const override {
989 return {4};
990 }
991};
992
993struct Wasm final : public VariadicABIInfo {
994
995 bool enableForTarget() override {
996 // Currently wasm is only used for testing.
997 return commandLineOverride();
998 }
999
1000 bool vaListPassedInSSARegister() override { return true; }
1001
1002 Type *vaListType(LLVMContext &Ctx) override {
1003 return PointerType::getUnqual(C&: Ctx);
1004 }
1005
1006 Type *vaListParameterType(Module &M) override {
1007 return PointerType::getUnqual(C&: M.getContext());
1008 }
1009
1010 Value *initializeVaList(Module &M, LLVMContext &Ctx, IRBuilder<> &Builder,
1011 AllocaInst * /*va_list*/, Value *Buffer) override {
1012 return Buffer;
1013 }
1014
1015 VAArgSlotInfo slotInfo(const DataLayout &DL, Type *Parameter) override {
1016 LLVMContext &Ctx = Parameter->getContext();
1017 const unsigned MinAlign = 4;
1018 Align A = DL.getABITypeAlign(Ty: Parameter);
1019 if (A < MinAlign)
1020 A = Align(MinAlign);
1021
1022 if (auto *S = dyn_cast<StructType>(Val: Parameter)) {
1023 if (S->getNumElements() > 1) {
1024 return {.DataAlign: DL.getABITypeAlign(Ty: PointerType::getUnqual(C&: Ctx)), .Indirect: true};
1025 }
1026 }
1027
1028 return {.DataAlign: A, .Indirect: false};
1029 }
1030};
1031
1032std::unique_ptr<VariadicABIInfo> VariadicABIInfo::create(const Triple &T) {
1033 switch (T.getArch()) {
1034 case Triple::r600:
1035 case Triple::amdgcn: {
1036 return std::make_unique<Amdgpu>();
1037 }
1038
1039 case Triple::wasm32: {
1040 return std::make_unique<Wasm>();
1041 }
1042
1043 case Triple::nvptx:
1044 case Triple::nvptx64: {
1045 return std::make_unique<NVPTX>();
1046 }
1047
1048 case Triple::spirv:
1049 case Triple::spirv32:
1050 case Triple::spirv64: {
1051 return std::make_unique<SPIRV>();
1052 }
1053
1054 default:
1055 return {};
1056 }
1057}
1058
1059} // namespace
1060
1061char ExpandVariadics::ID = 0;
1062
1063INITIALIZE_PASS(ExpandVariadics, DEBUG_TYPE, "Expand variadic functions", false,
1064 false)
1065
1066ModulePass *llvm::createExpandVariadicsPass(ExpandVariadicsMode M) {
1067 return new ExpandVariadics(M);
1068}
1069
1070PreservedAnalyses ExpandVariadicsPass::run(Module &M, ModuleAnalysisManager &) {
1071 return ExpandVariadics(Mode).runOnModule(M) ? PreservedAnalyses::none()
1072 : PreservedAnalyses::all();
1073}
1074
1075ExpandVariadicsPass::ExpandVariadicsPass(ExpandVariadicsMode M) : Mode(M) {}
1076