1//===-- SPIRVCtorDtorLowering.cpp - Handle global ctors and dtors --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass creates a unified init and fini kernel with the required metadata
10// to call global constructors and destructors on SPIR-V targets.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SPIRVCtorDtorLowering.h"
15#include "MCTargetDesc/SPIRVBaseInfo.h"
16#include "SPIRV.h"
17#include "llvm/ADT/STLFunctionalExtras.h"
18#include "llvm/ADT/StringExtras.h"
19#include "llvm/IR/CallingConv.h"
20#include "llvm/IR/Constants.h"
21#include "llvm/IR/Function.h"
22#include "llvm/IR/GlobalVariable.h"
23#include "llvm/IR/IRBuilder.h"
24#include "llvm/IR/Module.h"
25#include "llvm/IR/Value.h"
26#include "llvm/Pass.h"
27#include "llvm/Support/CommandLine.h"
28#include "llvm/Support/MD5.h"
29#include "llvm/Transforms/IPO/OpenMPOpt.h"
30#include "llvm/Transforms/Utils/ModuleUtils.h"
31
32using namespace llvm;
33
34#define DEBUG_TYPE "spirv-lower-ctor-dtor"
35
36static cl::opt<std::string>
37 GlobalStr("spirv-lower-global-ctor-dtor-id",
38 cl::desc("Override unique ID of ctor/dtor globals."),
39 cl::init(Val: ""), cl::Hidden);
40
41static cl::opt<bool>
42 CreateKernels("spirv-emit-init-fini-kernel",
43 cl::desc("Emit kernels to call ctor/dtor globals."),
44 cl::init(Val: true), cl::Hidden);
45
46namespace {
47constexpr int SPIRV_GLOBAL_AS = 1;
48
49std::string getHash(StringRef Str) {
50 llvm::MD5 Hasher;
51 llvm::MD5::MD5Result Hash;
52 Hasher.update(Str);
53 Hasher.final(Result&: Hash);
54 return llvm::utohexstr(X: Hash.low(), /*LowerCase=*/true);
55}
56
57void addKernelAttrs(Function *F) {
58 F->setCallingConv(CallingConv::SPIR_KERNEL);
59 F->addFnAttr(Kind: "uniform-work-group-size", Val: "true");
60}
61
62Function *createInitOrFiniKernelFunction(Module &M, bool IsCtor) {
63 StringRef InitOrFiniKernelName =
64 IsCtor ? "spirv$device$init" : "spirv$device$fini";
65 if (M.getFunction(Name: InitOrFiniKernelName))
66 return nullptr;
67
68 Function *InitOrFiniKernel = Function::createWithDefaultAttr(
69 Ty: FunctionType::get(Result: Type::getVoidTy(C&: M.getContext()), isVarArg: false),
70 Linkage: GlobalValue::WeakODRLinkage, AddrSpace: 0, N: InitOrFiniKernelName, M: &M);
71 addKernelAttrs(F: InitOrFiniKernel);
72
73 return InitOrFiniKernel;
74}
75
76// We create the IR required to call each callback in this section. This is
77// equivalent to the following code. Normally, the linker would provide us with
78// the definitions of the init and fini array sections. The 'spirv-link' linker
79// does not do this so initializing these values is done by the offload runtime.
80//
81// extern "C" void **__init_array_start = nullptr;
82// extern "C" void **__init_array_end = nullptr;
83// extern "C" void **__fini_array_start = nullptr;
84// extern "C" void **__fini_array_end = nullptr;
85//
86// using InitCallback = void();
87// using FiniCallback = void();
88//
89// void call_init_array_callbacks() {
90// for (auto start = __init_array_start; start != __init_array_end; ++start)
91// reinterpret_cast<InitCallback *>(*start)();
92// }
93//
94// void call_fini_array_callbacks() {
95// size_t fini_array_size = __fini_array_end - __fini_array_start;
96// for (size_t i = fini_array_size; i > 0; --i)
97// reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
98// }
99void createInitOrFiniCalls(Function &F, bool IsCtor) {
100 Module &M = *F.getParent();
101 LLVMContext &C = M.getContext();
102
103 IRBuilder<> IRB(BasicBlock::Create(Context&: C, Name: "entry", Parent: &F));
104 auto *LoopBB = BasicBlock::Create(Context&: C, Name: "while.entry", Parent: &F);
105 auto *ExitBB = BasicBlock::Create(Context&: C, Name: "while.end", Parent: &F);
106 Type *PtrTy = IRB.getPtrTy(AddrSpace: SPIRV_GLOBAL_AS);
107
108 auto CreateGlobal = [&](const char *Name) -> GlobalVariable * {
109 auto *GV = new GlobalVariable(
110 M, PointerType::getUnqual(C),
111 /*isConstant=*/false, GlobalValue::WeakAnyLinkage,
112 Constant::getNullValue(Ty: PointerType::getUnqual(C)), Name,
113 /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
114 /*AddressSpace=*/SPIRV_GLOBAL_AS);
115 GV->setVisibility(GlobalVariable::ProtectedVisibility);
116 return GV;
117 };
118
119 auto *Begin = M.getOrInsertGlobal(
120 Name: IsCtor ? "__init_array_start" : "__fini_array_start",
121 Ty: PointerType::getUnqual(C), CreateGlobalCallback: function_ref<GlobalVariable *()>([&]() {
122 return CreateGlobal(IsCtor ? "__init_array_start"
123 : "__fini_array_start");
124 }));
125 auto *End = M.getOrInsertGlobal(
126 Name: IsCtor ? "__init_array_end" : "__fini_array_end",
127 Ty: PointerType::getUnqual(C), CreateGlobalCallback: function_ref<GlobalVariable *()>([&]() {
128 return CreateGlobal(IsCtor ? "__init_array_end" : "__fini_array_end");
129 }));
130 auto *CallBackTy = FunctionType::get(Result: IRB.getVoidTy(), isVarArg: {});
131
132 // The destructor array must be called in reverse order. Get an expression to
133 // the end of the array and iterate backwards in that case.
134 Value *BeginVal = IRB.CreateLoad(Ty: Begin->getType(), Ptr: Begin, Name: "begin");
135 Value *EndVal = IRB.CreateLoad(Ty: Begin->getType(), Ptr: End, Name: "stop");
136 if (!IsCtor) {
137 Value *OldBeginVal = BeginVal;
138 BeginVal =
139 IRB.CreateInBoundsGEP(Ty: PointerType::getUnqual(C), Ptr: EndVal,
140 IdxList: ArrayRef<Value *>(ConstantInt::getAllOnesValue(
141 Ty: IntegerType::getInt64Ty(C))),
142 Name: "start");
143 EndVal = OldBeginVal;
144 }
145 IRB.CreateCondBr(
146 Cond: IRB.CreateCmp(Pred: IsCtor ? ICmpInst::ICMP_NE : ICmpInst::ICMP_UGE, LHS: BeginVal,
147 RHS: EndVal),
148 True: LoopBB, False: ExitBB);
149 IRB.SetInsertPoint(LoopBB);
150 auto *CallBackPHI = IRB.CreatePHI(Ty: PtrTy, NumReservedValues: 2, Name: "ptr");
151 auto *CallBack = IRB.CreateLoad(Ty: IRB.getPtrTy(AddrSpace: F.getAddressSpace()),
152 Ptr: CallBackPHI, Name: "callback");
153 IRB.CreateCall(FTy: CallBackTy, Callee: CallBack);
154 auto *NewCallBack =
155 IRB.CreateConstGEP1_64(Ty: PtrTy, Ptr: CallBackPHI, Idx0: IsCtor ? 1 : -1, Name: "next");
156 auto *EndCmp = IRB.CreateCmp(Pred: IsCtor ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_ULT,
157 LHS: NewCallBack, RHS: EndVal, Name: "end");
158 CallBackPHI->addIncoming(V: BeginVal, BB: &F.getEntryBlock());
159 CallBackPHI->addIncoming(V: NewCallBack, BB: LoopBB);
160 IRB.CreateCondBr(Cond: EndCmp, True: ExitBB, False: LoopBB);
161 IRB.SetInsertPoint(ExitBB);
162 IRB.CreateRetVoid();
163}
164
165bool createInitOrFiniGlobals(Module &M, GlobalVariable *GV, bool IsCtor) {
166 ConstantArray *GA = dyn_cast<ConstantArray>(Val: GV->getInitializer());
167 if (!GA || GA->getNumOperands() == 0)
168 return false;
169
170 // SPIR-V has no way to emit variables at specific sections or support for
171 // the traditional constructor sections. Instead, we emit mangled global
172 // names so the runtime can build the list manually.
173 for (Value *V : GA->operands()) {
174 auto *CS = cast<ConstantStruct>(Val: V);
175 auto *F = cast<Constant>(Val: CS->getOperand(i_nocapture: 1));
176 uint64_t Priority = cast<ConstantInt>(Val: CS->getOperand(i_nocapture: 0))->getSExtValue();
177 std::string PriorityStr = "." + std::to_string(val: Priority);
178 // We append a semi-unique hash and the priority to the global name.
179 std::string GlobalID =
180 !GlobalStr.empty() ? GlobalStr : getHash(Str: M.getSourceFileName());
181 std::string NameStr =
182 ((IsCtor ? "__init_array_object_" : "__fini_array_object_") +
183 F->getName() + "_" + GlobalID + "_" + std::to_string(val: Priority))
184 .str();
185 llvm::transform(Range&: NameStr, d_first: NameStr.begin(),
186 F: [](char c) { return c == '.' ? '_' : c; });
187
188 auto *GV = new GlobalVariable(M, F->getType(), /*IsConstant=*/true,
189 GlobalValue::ExternalLinkage, F, NameStr,
190 nullptr, GlobalValue::NotThreadLocal,
191 /*AddressSpace=*/SPIRV_GLOBAL_AS);
192 GV->setSection(IsCtor ? ".init_array" + PriorityStr
193 : ".fini_array" + PriorityStr);
194 GV->setVisibility(GlobalVariable::ProtectedVisibility);
195 }
196
197 return true;
198}
199
200bool createInitOrFiniKernel(Module &M, StringRef GlobalName, bool IsCtor) {
201 GlobalVariable *GV = M.getGlobalVariable(Name: GlobalName);
202 if (!GV || !GV->hasInitializer())
203 return false;
204
205 if (!createInitOrFiniGlobals(M, GV, IsCtor))
206 return false;
207
208 if (!CreateKernels)
209 return true;
210
211 Function *InitOrFiniKernel = createInitOrFiniKernelFunction(M, IsCtor);
212 if (!InitOrFiniKernel)
213 return false;
214
215 createInitOrFiniCalls(F&: *InitOrFiniKernel, IsCtor);
216
217 GV->eraseFromParent();
218 return true;
219}
220
221bool lowerCtorsAndDtors(Module &M) {
222 // Only run this pass for OpenMP offload compilation
223 if (!llvm::omp::isOpenMPDevice(M))
224 return false;
225
226 bool Modified = false;
227 Modified |= createInitOrFiniKernel(M, GlobalName: "llvm.global_ctors", /*IsCtor =*/true);
228 Modified |= createInitOrFiniKernel(M, GlobalName: "llvm.global_dtors", /*IsCtor =*/false);
229 return Modified;
230}
231
232class SPIRVCtorDtorLoweringLegacy final : public ModulePass {
233public:
234 static char ID;
235 SPIRVCtorDtorLoweringLegacy() : ModulePass(ID) {}
236 bool runOnModule(Module &M) override { return lowerCtorsAndDtors(M); }
237};
238
239} // End anonymous namespace
240
241PreservedAnalyses SPIRVCtorDtorLoweringPass::run(Module &M,
242 ModuleAnalysisManager &AM) {
243 return lowerCtorsAndDtors(M) ? PreservedAnalyses::none()
244 : PreservedAnalyses::all();
245}
246
247char SPIRVCtorDtorLoweringLegacy::ID = 0;
248INITIALIZE_PASS(SPIRVCtorDtorLoweringLegacy, DEBUG_TYPE,
249 "SPIRV lower ctors and dtors", false, false)
250
251ModulePass *llvm::createSPIRVCtorDtorLoweringLegacyPass() {
252 return new SPIRVCtorDtorLoweringLegacy();
253}
254