1//===-- NVPTXCtorDtorLowering.cpp - Handle global ctors and dtors --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This pass creates a unified init and fini kernel with the required metadata
11//===----------------------------------------------------------------------===//
12
13#include "MCTargetDesc/NVPTXBaseInfo.h"
14#include "NVPTX.h"
15#include "NVPTXSubtarget.h"
16#include "llvm/ADT/StringExtras.h"
17#include "llvm/IR/CallingConv.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/Function.h"
20#include "llvm/IR/GlobalVariable.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/Module.h"
23#include "llvm/IR/Value.h"
24#include "llvm/Pass.h"
25#include "llvm/Support/MD5.h"
26#include "llvm/Support/NVVMAttributes.h"
27#include "llvm/Transforms/Utils/ModuleUtils.h"
28
29using namespace llvm;
30
31#define DEBUG_TYPE "nvptx-lower-ctor-dtor"
32
33namespace {
34
35static std::string getHash(StringRef Str) {
36 llvm::MD5 Hasher;
37 llvm::MD5::MD5Result Hash;
38 Hasher.update(Str);
39 Hasher.final(Result&: Hash);
40 return llvm::utohexstr(X: Hash.low(), /*LowerCase=*/true);
41}
42
43static void addKernelAttrs(Function *F) {
44 F->addFnAttr(Kind: NVVMAttr::MaxClusterRank, Val: "1");
45 F->addFnAttr(Kind: NVVMAttr::MaxNTID, Val: "1");
46 F->setCallingConv(CallingConv::PTX_Kernel);
47}
48
49static Function *createInitOrFiniKernelFunction(Module &M, bool IsCtor) {
50 StringRef InitOrFiniKernelName =
51 IsCtor ? "nvptx$device$init" : "nvptx$device$fini";
52 if (M.getFunction(Name: InitOrFiniKernelName))
53 return nullptr;
54
55 Function *InitOrFiniKernel = Function::createWithDefaultAttr(
56 Ty: FunctionType::get(Result: Type::getVoidTy(C&: M.getContext()), isVarArg: false),
57 Linkage: GlobalValue::WeakODRLinkage, AddrSpace: 0, N: InitOrFiniKernelName, M: &M);
58 addKernelAttrs(F: InitOrFiniKernel);
59
60 return InitOrFiniKernel;
61}
62
63// We create the IR required to call each callback in this section. This is
64// equivalent to the following code. Normally, the linker would provide us with
65// the definitions of the init and fini array sections. The 'nvlink' linker does
66// not do this so initializing these values is done by the runtime.
67//
68// extern "C" void **__init_array_start = nullptr;
69// extern "C" void **__init_array_end = nullptr;
70// extern "C" void **__fini_array_start = nullptr;
71// extern "C" void **__fini_array_end = nullptr;
72//
73// using InitCallback = void();
74// using FiniCallback = void();
75//
76// void call_init_array_callbacks() {
77// for (auto start = __init_array_start; start != __init_array_end; ++start)
78// reinterpret_cast<InitCallback *>(*start)();
79// }
80//
81// void call_fini_array_callbacks() {
82// size_t fini_array_size = __fini_array_end - __fini_array_start;
83// for (size_t i = fini_array_size; i > 0; --i)
84// reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
85// }
86static void createInitOrFiniCalls(Function &F, bool IsCtor) {
87 Module &M = *F.getParent();
88 LLVMContext &C = M.getContext();
89
90 IRBuilder<> IRB(BasicBlock::Create(Context&: C, Name: "entry", Parent: &F));
91 IRB.disableGEPCanonicalization();
92 auto *LoopBB = BasicBlock::Create(Context&: C, Name: "while.entry", Parent: &F);
93 auto *ExitBB = BasicBlock::Create(Context&: C, Name: "while.end", Parent: &F);
94 Type *PtrTy = IRB.getPtrTy(AddrSpace: llvm::ADDRESS_SPACE_GLOBAL);
95
96 auto *Begin = M.getOrInsertGlobal(
97 Name: IsCtor ? "__init_array_start" : "__fini_array_start",
98 Ty: PointerType::get(C, AddressSpace: 0), CreateGlobalCallback: [&]() {
99 auto *GV = new GlobalVariable(
100 M, PointerType::get(C, AddressSpace: 0),
101 /*isConstant=*/false, GlobalValue::WeakAnyLinkage,
102 Constant::getNullValue(Ty: PointerType::get(C, AddressSpace: 0)),
103 IsCtor ? "__init_array_start" : "__fini_array_start",
104 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
105 /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL);
106 GV->setVisibility(GlobalVariable::ProtectedVisibility);
107 return GV;
108 });
109 auto *End = M.getOrInsertGlobal(
110 Name: IsCtor ? "__init_array_end" : "__fini_array_end", Ty: PointerType::get(C, AddressSpace: 0),
111 CreateGlobalCallback: [&]() {
112 auto *GV = new GlobalVariable(
113 M, PointerType::get(C, AddressSpace: 0),
114 /*isConstant=*/false, GlobalValue::WeakAnyLinkage,
115 Constant::getNullValue(Ty: PointerType::get(C, AddressSpace: 0)),
116 IsCtor ? "__init_array_end" : "__fini_array_end",
117 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
118 /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL);
119 GV->setVisibility(GlobalVariable::ProtectedVisibility);
120 return GV;
121 });
122
123 // The constructor type is suppoed to allow using the argument vectors, but
124 // for now we just call them with no arguments.
125 auto *CallBackTy = FunctionType::get(Result: IRB.getVoidTy(), isVarArg: {});
126
127 // The destructor array must be called in reverse order. Get an expression to
128 // the end of the array and iterate backwards in that case.
129 Value *BeginVal = IRB.CreateLoad(Ty: Begin->getType(), Ptr: Begin, Name: "begin");
130 Value *EndVal = IRB.CreateLoad(Ty: Begin->getType(), Ptr: End, Name: "stop");
131 if (!IsCtor) {
132 Value *OldBeginVal = BeginVal;
133 BeginVal =
134 IRB.CreateInBoundsGEP(Ty: PointerType::get(C, AddressSpace: 0), Ptr: EndVal,
135 IdxList: ArrayRef<Value *>(ConstantInt::getAllOnesValue(
136 Ty: IntegerType::getInt64Ty(C))),
137 Name: "start");
138 EndVal = OldBeginVal;
139 }
140 IRB.CreateCondBr(
141 Cond: IRB.CreateCmp(Pred: IsCtor ? ICmpInst::ICMP_NE : ICmpInst::ICMP_UGE, LHS: BeginVal,
142 RHS: EndVal),
143 True: LoopBB, False: ExitBB);
144 IRB.SetInsertPoint(LoopBB);
145 auto *CallBackPHI = IRB.CreatePHI(Ty: PtrTy, NumReservedValues: 2, Name: "ptr");
146 auto *CallBack = IRB.CreateLoad(Ty: IRB.getPtrTy(AddrSpace: F.getAddressSpace()),
147 Ptr: CallBackPHI, Name: "callback");
148 IRB.CreateCall(FTy: CallBackTy, Callee: CallBack);
149 auto *NewCallBack =
150 IRB.CreateConstGEP1_64(Ty: PtrTy, Ptr: CallBackPHI, Idx0: IsCtor ? 1 : -1, Name: "next");
151 auto *EndCmp = IRB.CreateCmp(Pred: IsCtor ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_ULT,
152 LHS: NewCallBack, RHS: EndVal, Name: "end");
153 CallBackPHI->addIncoming(V: BeginVal, BB: &F.getEntryBlock());
154 CallBackPHI->addIncoming(V: NewCallBack, BB: LoopBB);
155 IRB.CreateCondBr(Cond: EndCmp, True: ExitBB, False: LoopBB);
156 IRB.SetInsertPoint(ExitBB);
157 IRB.CreateRetVoid();
158}
159
160static bool createInitOrFiniGlobals(Module &M, GlobalVariable *GV, bool IsCtor,
161 const NVPTXOptions &CLOpts) {
162 ConstantArray *GA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
163 if (!GA || GA->getNumOperands() == 0)
164 return false;
165
166 // NVPTX has no way to emit variables at specific sections or support for
167 // the traditional constructor sections. Instead, we emit mangled global
168 // names so the runtime can build the list manually.
169 for (Value *V : GA->operands()) {
170 auto *CS = cast<ConstantStruct>(Val: V);
171 auto *F = cast<Constant>(Val: CS->getOperand(i_nocapture: 1));
172 uint64_t Priority = cast<ConstantInt>(Val: CS->getOperand(i_nocapture: 0))->getSExtValue();
173 std::string PriorityStr = "." + std::to_string(val: Priority);
174 // We append a semi-unique hash and the priority to the global name.
175 std::string GlobalID = CLOpts.lower_global_ctor_dtor_id.empty()
176 ? getHash(Str: M.getSourceFileName())
177 : CLOpts.lower_global_ctor_dtor_id.str();
178 std::string NameStr =
179 ((IsCtor ? "__init_array_object_" : "__fini_array_object_") +
180 F->getName() + "_" + GlobalID + "_" + std::to_string(val: Priority))
181 .str();
182 // PTX does not support exported names with '.' in them.
183 llvm::transform(Range&: NameStr, d_first: NameStr.begin(),
184 F: [](char c) { return c == '.' ? '_' : c; });
185
186 auto *GV = new GlobalVariable(M, F->getType(), /*IsConstant=*/true,
187 GlobalValue::ExternalLinkage, F, NameStr,
188 nullptr, GlobalValue::NotThreadLocal,
189 /*AddressSpace=*/4);
190 // This isn't respected by Nvidia, simply put here for clarity.
191 GV->setSection(IsCtor ? ".init_array" + PriorityStr
192 : ".fini_array" + PriorityStr);
193 GV->setVisibility(GlobalVariable::ProtectedVisibility);
194 appendToUsed(M, Values: {GV});
195 }
196
197 return true;
198}
199
200static bool createInitOrFiniKernel(Module &M, StringRef GlobalName, bool IsCtor,
201 const NVPTXOptions &CLOpts) {
202 GlobalVariable *GV = M.getGlobalVariable(Name: GlobalName);
203 if (!GV || !GV->hasInitializer())
204 return false;
205
206 if (!createInitOrFiniGlobals(M, GV, IsCtor, CLOpts))
207 return false;
208
209 if (!CLOpts.emit_init_fini_kernel)
210 return true;
211
212 Function *InitOrFiniKernel = createInitOrFiniKernelFunction(M, IsCtor);
213 if (!InitOrFiniKernel)
214 return false;
215
216 createInitOrFiniCalls(F&: *InitOrFiniKernel, IsCtor);
217
218 GV->eraseFromParent();
219 return true;
220}
221
222static bool lowerCtorsAndDtors(Module &M) {
223 const NVPTXOptions &CLOpts = getNVPTXOptions(M);
224 bool Modified = false;
225 Modified |=
226 createInitOrFiniKernel(M, GlobalName: "llvm.global_ctors", /*IsCtor =*/true, CLOpts);
227 Modified |=
228 createInitOrFiniKernel(M, GlobalName: "llvm.global_dtors", /*IsCtor =*/false, CLOpts);
229 return Modified;
230}
231
232class NVPTXCtorDtorLoweringLegacy final : public ModulePass {
233public:
234 static char ID;
235 NVPTXCtorDtorLoweringLegacy() : ModulePass(ID) {}
236 bool runOnModule(Module &M) override { return lowerCtorsAndDtors(M); }
237};
238
239} // End anonymous namespace
240
241PreservedAnalyses NVPTXCtorDtorLoweringPass::run(Module &M,
242 ModuleAnalysisManager &MAM) {
243 return lowerCtorsAndDtors(M) ? PreservedAnalyses::none()
244 : PreservedAnalyses::all();
245}
246
247char NVPTXCtorDtorLoweringLegacy::ID = 0;
248INITIALIZE_PASS(NVPTXCtorDtorLoweringLegacy, DEBUG_TYPE,
249 "Lower ctors and dtors for NVPTX", false, false)
250
251ModulePass *llvm::createNVPTXCtorDtorLoweringLegacyPass() {
252 return new NVPTXCtorDtorLoweringLegacy();
253}
254