1#include "llvm/ExecutionEngine/Orc/ReOptimizeLayer.h"
2#include "llvm/ExecutionEngine/Orc/BindCallControllerHandler.h"
3#include "llvm/ExecutionEngine/Orc/LookupAndApply.h"
4#include "llvm/ExecutionEngine/Orc/Mangling.h"
5#include "llvm/ExecutionEngine/Orc/SPSHandlerSpec.h"
6#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
7
8using namespace llvm;
9using namespace orc;
10
11// Spec for the call-controller handler that the ORC runtime calls to
12// request reoptimization.
13namespace llvm::orc::reoptimize_sps_handlers {
14struct Reoptimize : sps::HandlerSpec<shared::SPSError(uint64_t, uint32_t)> {
15 static constexpr SymbolNameSpec Name =
16 SymbolNameSpec::c(Name: "__orc_rt_reoptimize_tag");
17};
18} // namespace llvm::orc::reoptimize_sps_handlers
19
20bool ReOptimizeLayer::ReOptMaterializationUnitState::tryStartReoptimize() {
21 std::unique_lock<std::mutex> Lock(Mutex);
22 if (Reoptimizing)
23 return false;
24
25 Reoptimizing = true;
26 return true;
27}
28
29void ReOptimizeLayer::ReOptMaterializationUnitState::reoptimizeSucceeded() {
30 std::unique_lock<std::mutex> Lock(Mutex);
31 assert(Reoptimizing && "Tried to mark unstarted reoptimization as done");
32 Reoptimizing = false;
33 CurVersion++;
34}
35
36void ReOptimizeLayer::ReOptMaterializationUnitState::reoptimizeFailed() {
37 std::unique_lock<std::mutex> Lock(Mutex);
38 assert(Reoptimizing && "Tried to mark unstarted reoptimization as done");
39 Reoptimizing = false;
40}
41
42static void orc_rt_lite_reoptimize_helper(
43 shared::CWrapperFunctionBuffer (*JITDispatch)(void *Ctx, const void *Tag,
44 const char *Data,
45 size_t Size),
46 void *JITDispatchCtx, const void *Tag, uint64_t MUID, uint32_t CurVersion) {
47 // Serialize the arguments into a WrapperFunctionBuffer and call dispatch.
48 using SPSArgs = shared::SPSArgList<uint64_t, uint32_t>;
49 auto ArgBytes =
50 shared::WrapperFunctionBuffer::allocate(Size: SPSArgs::size(Arg: MUID, Args: CurVersion));
51 shared::SPSOutputBuffer OB(ArgBytes.data(), ArgBytes.size());
52 if (!SPSArgs::serialize(OB, Arg: MUID, Args: CurVersion)) {
53 errs()
54 << "Reoptimization error: could not serialize reoptimization arguments";
55 abort();
56 }
57 shared::WrapperFunctionBuffer Buf{
58 JITDispatch(JITDispatchCtx, Tag, ArgBytes.data(), ArgBytes.size())};
59
60 if (const char *ErrMsg = Buf.getOutOfBandError()) {
61 errs() << "Reoptimization error: " << ErrMsg << "\naborting.\n";
62 abort();
63 }
64}
65
66Error ReOptimizeLayer::addOrcRTLiteSupport(JITDylib &PlatformJD,
67 const DataLayout &DL) {
68 auto Ctx = std::make_unique<LLVMContext>();
69 auto Mod = std::make_unique<Module>(args: "orc-rt-lite-reoptimize.ll", args&: *Ctx);
70 Mod->setDataLayout(DL);
71 Mod->setTargetTriple(ES.getTargetTriple());
72
73 IRBuilder<> Builder(*Mod);
74
75 // Create basic types portably
76 Type *VoidTy = Type::getVoidTy(C&: *Ctx);
77 Type *Int8Ty = Type::getInt8Ty(C&: *Ctx);
78 Type *Int32Ty = Type::getInt32Ty(C&: *Ctx);
79 Type *Int64Ty = Type::getInt64Ty(C&: *Ctx);
80 Type *VoidPtrTy = PointerType::getUnqual(C&: *Ctx);
81
82 // Helper function type: void (void*, void*, void*, uint64_t, uint32_t)
83 FunctionType *HelperFnTy = FunctionType::get(
84 Result: VoidTy, Params: {VoidPtrTy, VoidPtrTy, VoidPtrTy, Int64Ty, Int32Ty}, isVarArg: false);
85
86 // Define ReoptimizeTag with initializer = 0
87 GlobalVariable *ReoptimizeTag = new GlobalVariable(
88 *Mod, Int8Ty, false, GlobalValue::ExternalLinkage,
89 ConstantInt::get(Ty: Int8Ty, V: 0), "__orc_rt_reoptimize_tag");
90
91 // Define orc_rt_lite_reoptimize function: void (uint64_t, uint32_t)
92 FunctionType *ReOptimizeFnTy =
93 FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int32Ty}, isVarArg: false);
94
95 Function *ReOptimizeFn =
96 Function::Create(Ty: ReOptimizeFnTy, Linkage: Function::ExternalLinkage,
97 N: "__orc_rt_reoptimize", M: Mod.get());
98
99 // Set parameter names
100 auto ArgIt = ReOptimizeFn->arg_begin();
101 Value *MUID = &*ArgIt++;
102 MUID->setName("MUID");
103 Value *CurVersion = &*ArgIt;
104 CurVersion->setName("CurVersion");
105
106 // Build function body
107 BasicBlock *Entry = BasicBlock::Create(Context&: *Ctx, Name: "entry", Parent: ReOptimizeFn);
108 Builder.SetInsertPoint(Entry);
109
110 ExecutorAddr JITDispatchSym, JITDispatchCtxSym;
111 if (auto Err =
112 lookupAndApply(JD&: ES.getBootstrapJITDylib(),
113 PrepareFns: {recordAddr(Name: rt::DispatchName, A: &JITDispatchSym),
114 recordAddr(Name: rt::DispatchCtxName, A: &JITDispatchCtxSym)}))
115 return Err;
116
117 Type *IntPtrTy = DL.getIntPtrType(C&: *Ctx);
118 Constant *JITDispatchPtr = ConstantExpr::getIntToPtr(
119 C: ConstantInt::get(Ty: IntPtrTy, V: JITDispatchSym.getValue()), Ty: VoidPtrTy);
120 Constant *JITDispatchCtxPtr = ConstantExpr::getIntToPtr(
121 C: ConstantInt::get(Ty: IntPtrTy, V: JITDispatchCtxSym.getValue()), Ty: VoidPtrTy);
122 Constant *HelperFnAddr = ConstantExpr::getIntToPtr(
123 C: ConstantInt::get(Ty: IntPtrTy, V: reinterpret_cast<uintptr_t>(
124 &orc_rt_lite_reoptimize_helper)),
125 Ty: PointerType::getUnqual(C&: *Ctx));
126
127 // Cast ReoptimizeTag to void*
128 Value *ReoptimizeTagPtr = Builder.CreatePointerCast(V: ReoptimizeTag, DestTy: VoidPtrTy);
129
130 // Call the helper function
131 Builder.CreateCall(
132 FTy: HelperFnTy, Callee: HelperFnAddr,
133 Args: {JITDispatchPtr, JITDispatchCtxPtr, ReoptimizeTagPtr, MUID, CurVersion});
134
135 // Return void
136 Builder.CreateRetVoid();
137
138 return BaseLayer.add(JD&: PlatformJD,
139 TSM: ThreadSafeModule(std::move(Mod), std::move(Ctx)));
140}
141
142Error ReOptimizeLayer::registerRuntimeFunctions(JITDylib &PlatformJD) {
143 return ES.registerCallControllerHandlers(
144 JD&: PlatformJD,
145 Hs: bindCallControllerHandler<reoptimize_sps_handlers::Reoptimize>(
146 Instance: this, Method: &ReOptimizeLayer::rt_reoptimize));
147}
148
149void ReOptimizeLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
150 ThreadSafeModule TSM) {
151 auto &JD = R->getTargetJITDylib();
152
153 bool HasNonCallable = false;
154 for (auto &KV : R->getSymbols()) {
155 auto &Flags = KV.second;
156 if (!Flags.isCallable())
157 HasNonCallable = true;
158 }
159
160 if (HasNonCallable) {
161 BaseLayer.emit(R: std::move(R), TSM: std::move(TSM));
162 return;
163 }
164
165 auto &MUState = createMaterializationUnitState(TSM);
166
167 if (auto Err = R->withResourceKeyDo(F: [&](ResourceKey Key) {
168 registerMaterializationUnitResource(Key, State&: MUState);
169 })) {
170 ES.reportError(Err: std::move(Err));
171 R->failMaterialization();
172 return;
173 }
174
175 if (auto Err =
176 ProfilerFunc(*this, MUState.getID(), MUState.getCurVersion(), TSM)) {
177 ES.reportError(Err: std::move(Err));
178 R->failMaterialization();
179 return;
180 }
181
182 auto InitialDests =
183 emitMUImplSymbols(MUState, Version: MUState.getCurVersion(), JD, TSM: std::move(TSM));
184 if (!InitialDests) {
185 ES.reportError(Err: InitialDests.takeError());
186 R->failMaterialization();
187 return;
188 }
189
190 RSManager.emitRedirectableSymbols(MR: std::move(R), InitialDests: std::move(*InitialDests));
191}
192
193Error ReOptimizeLayer::reoptimizeIfCallFrequent(ReOptimizeLayer &Parent,
194 ReOptMaterializationUnitID MUID,
195 unsigned CurVersion,
196 ThreadSafeModule &TSM) {
197 return TSM.withModuleDo(F: [&](Module &M) -> Error {
198 Type *I64Ty = Type::getInt64Ty(C&: M.getContext());
199 GlobalVariable *Counter = new GlobalVariable(
200 M, I64Ty, false, GlobalValue::InternalLinkage,
201 Constant::getNullValue(Ty: I64Ty), "__orc_reopt_counter");
202 for (auto &F : M) {
203 if (F.isDeclaration())
204 continue;
205 auto &BB = F.getEntryBlock();
206 auto *IP = &*BB.getFirstInsertionPt();
207 IRBuilder<> IRB(IP);
208 Value *Threshold = ConstantInt::get(Ty: I64Ty, V: CallCountThreshold, IsSigned: true);
209 Value *Cnt = IRB.CreateLoad(Ty: I64Ty, Ptr: Counter);
210 // Use EQ to prevent further reoptimize calls.
211 Value *Cmp = IRB.CreateICmpEQ(LHS: Cnt, RHS: Threshold);
212 Value *Added = IRB.CreateAdd(LHS: Cnt, RHS: ConstantInt::get(Ty: I64Ty, V: 1));
213 (void)IRB.CreateStore(Val: Added, Ptr: Counter);
214 Instruction *SplitTerminator = SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore: IP, Unreachable: false);
215 createReoptimizeCall(M, IP&: *SplitTerminator, MUID, CurVersion);
216 }
217 return Error::success();
218 });
219}
220
221Expected<SymbolMap>
222ReOptimizeLayer::emitMUImplSymbols(ReOptMaterializationUnitState &MUState,
223 uint32_t Version, JITDylib &JD,
224 ThreadSafeModule TSM) {
225 DenseMap<SymbolStringPtr, SymbolStringPtr> RenamedMap;
226 cantFail(Err: TSM.withModuleDo(F: [&](Module &M) -> Error {
227 MangleAndInterner Mangle(ES, M.getDataLayout());
228 for (auto &F : M)
229 if (!F.isDeclaration()) {
230 std::string NewName =
231 (F.getName() + ".__def__." + Twine(Version)).str();
232 RenamedMap[Mangle(F.getName())] = Mangle(NewName);
233 F.setName(NewName);
234 }
235 return Error::success();
236 }));
237
238 auto RT = JD.createResourceTracker();
239 if (auto Err =
240 JD.define(MU: std::make_unique<BasicIRLayerMaterializationUnit>(
241 args&: BaseLayer, args: *getManglingOptions(), args: std::move(TSM)),
242 RT))
243 return Err;
244 MUState.setResourceTracker(RT);
245
246 SymbolLookupSet LookupSymbols;
247 for (auto [K, V] : RenamedMap)
248 LookupSymbols.add(Name: V);
249
250 auto ImplSymbols =
251 ES.lookup(SearchOrder: {{&JD, JITDylibLookupFlags::MatchAllSymbols}}, Symbols: LookupSymbols,
252 K: LookupKind::Static, RequiredState: SymbolState::Resolved);
253 if (auto Err = ImplSymbols.takeError())
254 return Err;
255
256 SymbolMap Result;
257 for (auto [K, V] : RenamedMap)
258 Result[K] = (*ImplSymbols)[V];
259
260 return Result;
261}
262
263void ReOptimizeLayer::rt_reoptimize(SendErrorFn SendResult,
264 ReOptMaterializationUnitID MUID,
265 uint32_t CurVersion) {
266 auto &MUState = getMaterializationUnitState(MUID);
267 if (CurVersion < MUState.getCurVersion() || !MUState.tryStartReoptimize()) {
268 SendResult(Error::success());
269 return;
270 }
271
272 ThreadSafeModule TSM = cloneToNewContext(TSMW: MUState.getThreadSafeModule());
273 auto OldRT = MUState.getResourceTracker();
274 auto &JD = OldRT->getJITDylib();
275
276 if (auto Err = ReOptFunc(*this, MUID, CurVersion + 1, OldRT, TSM)) {
277 ES.reportError(Err: std::move(Err));
278 MUState.reoptimizeFailed();
279 SendResult(Error::success());
280 return;
281 }
282
283 auto SymbolDests =
284 emitMUImplSymbols(MUState, Version: CurVersion + 1, JD, TSM: std::move(TSM));
285 if (!SymbolDests) {
286 ES.reportError(Err: SymbolDests.takeError());
287 MUState.reoptimizeFailed();
288 SendResult(Error::success());
289 return;
290 }
291
292 if (auto Err = RSManager.redirect(JD, NewDests: std::move(*SymbolDests))) {
293 ES.reportError(Err: std::move(Err));
294 MUState.reoptimizeFailed();
295 SendResult(Error::success());
296 return;
297 }
298
299 MUState.reoptimizeSucceeded();
300 SendResult(Error::success());
301}
302
303void ReOptimizeLayer::createReoptimizeCall(Module &M, Instruction &IP,
304 ReOptMaterializationUnitID MUID,
305 uint32_t CurVersion) {
306 Type *MUIDTy = IntegerType::get(C&: M.getContext(), NumBits: 64);
307 Type *VersionTy = IntegerType::get(C&: M.getContext(), NumBits: 32);
308 Function *ReoptimizeFunc = M.getFunction(Name: "__orc_rt_reoptimize");
309 if (!ReoptimizeFunc) {
310 std::vector<Type *> ArgTys = {MUIDTy, VersionTy};
311 FunctionType *FuncTy =
312 FunctionType::get(Result: Type::getVoidTy(C&: M.getContext()), Params: ArgTys, isVarArg: false);
313 ReoptimizeFunc = Function::Create(Ty: FuncTy, Linkage: GlobalValue::ExternalLinkage,
314 N: "__orc_rt_reoptimize", M: &M);
315 }
316 Constant *MUIDArg = ConstantInt::get(Ty: MUIDTy, V: MUID, IsSigned: false);
317 Constant *CurVersionArg = ConstantInt::get(Ty: VersionTy, V: CurVersion, IsSigned: false);
318 IRBuilder<> IRB(&IP);
319 (void)IRB.CreateCall(Callee: ReoptimizeFunc, Args: {MUIDArg, CurVersionArg});
320}
321
322ReOptimizeLayer::ReOptMaterializationUnitState &
323ReOptimizeLayer::createMaterializationUnitState(const ThreadSafeModule &TSM) {
324 std::unique_lock<std::mutex> Lock(Mutex);
325 ReOptMaterializationUnitID MUID = NextID;
326 MUStates.emplace(args&: MUID,
327 args: ReOptMaterializationUnitState(MUID, cloneToNewContext(TSMW: TSM)));
328 ++NextID;
329 return MUStates.at(k: MUID);
330}
331
332ReOptimizeLayer::ReOptMaterializationUnitState &
333ReOptimizeLayer::getMaterializationUnitState(ReOptMaterializationUnitID MUID) {
334 std::unique_lock<std::mutex> Lock(Mutex);
335 return MUStates.at(k: MUID);
336}
337
338void ReOptimizeLayer::registerMaterializationUnitResource(
339 ResourceKey Key, ReOptMaterializationUnitState &State) {
340 std::unique_lock<std::mutex> Lock(Mutex);
341 MUResources[Key].insert(V: State.getID());
342}
343
344Error ReOptimizeLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) {
345 std::unique_lock<std::mutex> Lock(Mutex);
346 for (auto MUID : MUResources[K])
347 MUStates.erase(x: MUID);
348
349 MUResources.erase(Val: K);
350 return Error::success();
351}
352
353void ReOptimizeLayer::handleTransferResources(JITDylib &JD, ResourceKey DstK,
354 ResourceKey SrcK) {
355 std::unique_lock<std::mutex> Lock(Mutex);
356 MUResources[DstK].insert_range(R&: MUResources[SrcK]);
357 MUResources.erase(Val: SrcK);
358}
359