1//===-- AMDGPUCtorDtorLowering.cpp - Handle global ctors and dtors --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This pass creates a unified init and fini kernel with the required metadata
11//===----------------------------------------------------------------------===//
12
13#include "AMDGPUCtorDtorLowering.h"
14#include "AMDGPU.h"
15#include "llvm/IR/Constants.h"
16#include "llvm/IR/Function.h"
17#include "llvm/IR/GlobalVariable.h"
18#include "llvm/IR/IRBuilder.h"
19#include "llvm/IR/Module.h"
20#include "llvm/IR/Value.h"
21#include "llvm/Pass.h"
22#include "llvm/Transforms/Utils/ModuleUtils.h"
23
24using namespace llvm;
25
26#define DEBUG_TYPE "amdgpu-lower-ctor-dtor"
27
28namespace {
29
30static Function *createInitOrFiniKernelFunction(Module &M, bool IsCtor) {
31 StringRef InitOrFiniKernelName = "amdgcn.device.init";
32 if (!IsCtor)
33 InitOrFiniKernelName = "amdgcn.device.fini";
34 if (M.getFunction(Name: InitOrFiniKernelName))
35 return nullptr;
36
37 Function *InitOrFiniKernel = Function::createWithDefaultAttr(
38 Ty: FunctionType::get(Result: Type::getVoidTy(C&: M.getContext()), isVarArg: false),
39 Linkage: GlobalValue::WeakODRLinkage, AddrSpace: 0, N: InitOrFiniKernelName, M: &M);
40 InitOrFiniKernel->setCallingConv(CallingConv::AMDGPU_KERNEL);
41 InitOrFiniKernel->addFnAttr(Kind: "amdgpu-flat-work-group-size", Val: "1,1");
42 if (IsCtor)
43 InitOrFiniKernel->addFnAttr(Kind: "device-init");
44 else
45 InitOrFiniKernel->addFnAttr(Kind: "device-fini");
46 return InitOrFiniKernel;
47}
48
49// The linker will provide the associated symbols to allow us to traverse the
50// global constructors / destructors in priority order. We create the IR
51// required to call each callback in this section. This is equivalent to the
52// following code.
53//
54// extern "C" void * __init_array_start[];
55// extern "C" void * __init_array_end[];
56// extern "C" void * __fini_array_start[];
57// extern "C" void * __fini_array_end[];
58//
59// using InitCallback = void();
60// using FiniCallback = void(void);
61//
62// void call_init_array_callbacks() {
63// for (auto start = __init_array_start; start != __init_array_end; ++start)
64// reinterpret_cast<InitCallback *>(*start)();
65// }
66//
67// void call_fini_array_callbacks() {
68// size_t fini_array_size = __fini_array_end - __fini_array_start;
69// for (size_t i = fini_array_size; i > 0; --i)
70// reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
71// }
72static void createInitOrFiniCalls(Function &F, bool IsCtor) {
73 Module &M = *F.getParent();
74 LLVMContext &C = M.getContext();
75
76 IRBuilder<> IRB(BasicBlock::Create(Context&: C, Name: "entry", Parent: &F));
77 auto *LoopBB = BasicBlock::Create(Context&: C, Name: "while.entry", Parent: &F);
78 auto *ExitBB = BasicBlock::Create(Context&: C, Name: "while.end", Parent: &F);
79 Type *PtrTy = IRB.getPtrTy(AddrSpace: AMDGPUAS::GLOBAL_ADDRESS);
80 ArrayType *PtrArrayTy = ArrayType::get(ElementType: PtrTy, NumElements: 0);
81
82 auto *Begin = M.getOrInsertGlobal(
83 Name: IsCtor ? "__init_array_start" : "__fini_array_start", Ty: PtrArrayTy, CreateGlobalCallback: [&]() {
84 return new GlobalVariable(
85 M, PtrArrayTy,
86 /*isConstant=*/true, GlobalValue::ExternalLinkage,
87 /*Initializer=*/nullptr,
88 IsCtor ? "__init_array_start" : "__fini_array_start",
89 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
90 /*AddressSpace=*/AMDGPUAS::GLOBAL_ADDRESS);
91 });
92 auto *End = M.getOrInsertGlobal(
93 Name: IsCtor ? "__init_array_end" : "__fini_array_end", Ty: PtrArrayTy, CreateGlobalCallback: [&]() {
94 return new GlobalVariable(
95 M, PtrArrayTy,
96 /*isConstant=*/true, GlobalValue::ExternalLinkage,
97 /*Initializer=*/nullptr,
98 IsCtor ? "__init_array_end" : "__fini_array_end",
99 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
100 /*AddressSpace=*/AMDGPUAS::GLOBAL_ADDRESS);
101 });
102
103 // The constructor type is suppoed to allow using the argument vectors, but
104 // for now we just call them with no arguments.
105 auto *CallBackTy = FunctionType::get(Result: IRB.getVoidTy(), isVarArg: {});
106
107 Value *Start = Begin;
108 Value *Stop = End;
109 // The destructor array must be called in reverse order. Get a constant
110 // expression to the end of the array and iterate backwards instead.
111 if (!IsCtor) {
112 Start = IRB.CreateInBoundsGEP(
113 Ty: PtrTy, Ptr: End, IdxList: ConstantInt::getAllOnesValue(Ty: IRB.getInt64Ty()));
114 Stop = Begin;
115 }
116
117 IRB.CreateCondBr(
118 Cond: IRB.CreateCmp(Pred: IsCtor ? ICmpInst::ICMP_NE : ICmpInst::ICMP_UGE, LHS: Start,
119 RHS: Stop),
120 True: LoopBB, False: ExitBB);
121 IRB.SetInsertPoint(LoopBB);
122 auto *CallBackPHI = IRB.CreatePHI(Ty: PtrTy, NumReservedValues: 2, Name: "ptr");
123 auto *CallBack = IRB.CreateLoad(Ty: F.getType(), Ptr: CallBackPHI, Name: "callback");
124 IRB.CreateCall(FTy: CallBackTy, Callee: CallBack);
125 auto *NewCallBack =
126 IRB.CreateConstGEP1_64(Ty: PtrTy, Ptr: CallBackPHI, Idx0: IsCtor ? 1 : -1, Name: "next");
127 auto *EndCmp = IRB.CreateCmp(Pred: IsCtor ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_ULT,
128 LHS: NewCallBack, RHS: Stop, Name: "end");
129 CallBackPHI->addIncoming(V: Start, BB: &F.getEntryBlock());
130 CallBackPHI->addIncoming(V: NewCallBack, BB: LoopBB);
131 IRB.CreateCondBr(Cond: EndCmp, True: ExitBB, False: LoopBB);
132 IRB.SetInsertPoint(ExitBB);
133 IRB.CreateRetVoid();
134}
135
136static bool createInitOrFiniKernel(Module &M, StringRef GlobalName,
137 bool IsCtor) {
138 GlobalVariable *GV = M.getGlobalVariable(Name: GlobalName);
139 if (!GV || !GV->hasInitializer())
140 return false;
141 ConstantArray *GA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
142 if (!GA || GA->getNumOperands() == 0)
143 return false;
144
145 Function *InitOrFiniKernel = createInitOrFiniKernelFunction(M, IsCtor);
146 if (!InitOrFiniKernel)
147 return false;
148
149 createInitOrFiniCalls(F&: *InitOrFiniKernel, IsCtor);
150
151 appendToUsed(M, Values: {InitOrFiniKernel});
152 return true;
153}
154
155static bool lowerCtorsAndDtors(Module &M) {
156 bool Modified = false;
157 Modified |= createInitOrFiniKernel(M, GlobalName: "llvm.global_ctors", /*IsCtor =*/true);
158 Modified |= createInitOrFiniKernel(M, GlobalName: "llvm.global_dtors", /*IsCtor =*/false);
159 return Modified;
160}
161
162class AMDGPUCtorDtorLoweringLegacy final : public ModulePass {
163public:
164 static char ID;
165 AMDGPUCtorDtorLoweringLegacy() : ModulePass(ID) {}
166 bool runOnModule(Module &M) override { return lowerCtorsAndDtors(M); }
167};
168
169} // End anonymous namespace
170
171PreservedAnalyses AMDGPUCtorDtorLoweringPass::run(Module &M,
172 ModuleAnalysisManager &AM) {
173 return lowerCtorsAndDtors(M) ? PreservedAnalyses::none()
174 : PreservedAnalyses::all();
175}
176
177char AMDGPUCtorDtorLoweringLegacy::ID = 0;
178char &llvm::AMDGPUCtorDtorLoweringLegacyPassID =
179 AMDGPUCtorDtorLoweringLegacy::ID;
180INITIALIZE_PASS(AMDGPUCtorDtorLoweringLegacy, DEBUG_TYPE,
181 "Lower ctors and dtors for AMDGPU", false, false)
182
183ModulePass *llvm::createAMDGPUCtorDtorLoweringLegacyPass() {
184 return new AMDGPUCtorDtorLoweringLegacy();
185}
186