| 1 | #include "llvm/ExecutionEngine/Orc/ReOptimizeLayer.h" |
| 2 | #include "llvm/ExecutionEngine/Orc/BindCallControllerHandler.h" |
| 3 | #include "llvm/ExecutionEngine/Orc/LookupAndApply.h" |
| 4 | #include "llvm/ExecutionEngine/Orc/Mangling.h" |
| 5 | #include "llvm/ExecutionEngine/Orc/SPSHandlerSpec.h" |
| 6 | #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" |
| 7 | |
| 8 | using namespace llvm; |
| 9 | using namespace orc; |
| 10 | |
| 11 | // Spec for the call-controller handler that the ORC runtime calls to |
| 12 | // request reoptimization. |
| 13 | namespace llvm::orc::reoptimize_sps_handlers { |
| 14 | struct Reoptimize : sps::HandlerSpec<shared::SPSError(uint64_t, uint32_t)> { |
| 15 | static constexpr SymbolNameSpec Name = |
| 16 | SymbolNameSpec::c(Name: "__orc_rt_reoptimize_tag" ); |
| 17 | }; |
| 18 | } // namespace llvm::orc::reoptimize_sps_handlers |
| 19 | |
| 20 | bool ReOptimizeLayer::ReOptMaterializationUnitState::tryStartReoptimize() { |
| 21 | std::unique_lock<std::mutex> Lock(Mutex); |
| 22 | if (Reoptimizing) |
| 23 | return false; |
| 24 | |
| 25 | Reoptimizing = true; |
| 26 | return true; |
| 27 | } |
| 28 | |
| 29 | void ReOptimizeLayer::ReOptMaterializationUnitState::reoptimizeSucceeded() { |
| 30 | std::unique_lock<std::mutex> Lock(Mutex); |
| 31 | assert(Reoptimizing && "Tried to mark unstarted reoptimization as done" ); |
| 32 | Reoptimizing = false; |
| 33 | CurVersion++; |
| 34 | } |
| 35 | |
| 36 | void ReOptimizeLayer::ReOptMaterializationUnitState::reoptimizeFailed() { |
| 37 | std::unique_lock<std::mutex> Lock(Mutex); |
| 38 | assert(Reoptimizing && "Tried to mark unstarted reoptimization as done" ); |
| 39 | Reoptimizing = false; |
| 40 | } |
| 41 | |
| 42 | static void orc_rt_lite_reoptimize_helper( |
| 43 | shared::CWrapperFunctionBuffer (*JITDispatch)(void *Ctx, const void *Tag, |
| 44 | const char *Data, |
| 45 | size_t Size), |
| 46 | void *JITDispatchCtx, const void *Tag, uint64_t MUID, uint32_t CurVersion) { |
| 47 | // Serialize the arguments into a WrapperFunctionBuffer and call dispatch. |
| 48 | using SPSArgs = shared::SPSArgList<uint64_t, uint32_t>; |
| 49 | auto ArgBytes = |
| 50 | shared::WrapperFunctionBuffer::allocate(Size: SPSArgs::size(Arg: MUID, Args: CurVersion)); |
| 51 | shared::SPSOutputBuffer OB(ArgBytes.data(), ArgBytes.size()); |
| 52 | if (!SPSArgs::serialize(OB, Arg: MUID, Args: CurVersion)) { |
| 53 | errs() |
| 54 | << "Reoptimization error: could not serialize reoptimization arguments" ; |
| 55 | abort(); |
| 56 | } |
| 57 | shared::WrapperFunctionBuffer Buf{ |
| 58 | JITDispatch(JITDispatchCtx, Tag, ArgBytes.data(), ArgBytes.size())}; |
| 59 | |
| 60 | if (const char *ErrMsg = Buf.getOutOfBandError()) { |
| 61 | errs() << "Reoptimization error: " << ErrMsg << "\naborting.\n" ; |
| 62 | abort(); |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | Error ReOptimizeLayer::addOrcRTLiteSupport(JITDylib &PlatformJD, |
| 67 | const DataLayout &DL) { |
| 68 | auto Ctx = std::make_unique<LLVMContext>(); |
| 69 | auto Mod = std::make_unique<Module>(args: "orc-rt-lite-reoptimize.ll" , args&: *Ctx); |
| 70 | Mod->setDataLayout(DL); |
| 71 | Mod->setTargetTriple(ES.getTargetTriple()); |
| 72 | |
| 73 | IRBuilder<> Builder(*Mod); |
| 74 | |
| 75 | // Create basic types portably |
| 76 | Type *VoidTy = Type::getVoidTy(C&: *Ctx); |
| 77 | Type *Int8Ty = Type::getInt8Ty(C&: *Ctx); |
| 78 | Type *Int32Ty = Type::getInt32Ty(C&: *Ctx); |
| 79 | Type *Int64Ty = Type::getInt64Ty(C&: *Ctx); |
| 80 | Type *VoidPtrTy = PointerType::getUnqual(C&: *Ctx); |
| 81 | |
| 82 | // Helper function type: void (void*, void*, void*, uint64_t, uint32_t) |
| 83 | FunctionType *HelperFnTy = FunctionType::get( |
| 84 | Result: VoidTy, Params: {VoidPtrTy, VoidPtrTy, VoidPtrTy, Int64Ty, Int32Ty}, isVarArg: false); |
| 85 | |
| 86 | // Define ReoptimizeTag with initializer = 0 |
| 87 | GlobalVariable *ReoptimizeTag = new GlobalVariable( |
| 88 | *Mod, Int8Ty, false, GlobalValue::ExternalLinkage, |
| 89 | ConstantInt::get(Ty: Int8Ty, V: 0), "__orc_rt_reoptimize_tag" ); |
| 90 | |
| 91 | // Define orc_rt_lite_reoptimize function: void (uint64_t, uint32_t) |
| 92 | FunctionType *ReOptimizeFnTy = |
| 93 | FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int32Ty}, isVarArg: false); |
| 94 | |
| 95 | Function *ReOptimizeFn = |
| 96 | Function::Create(Ty: ReOptimizeFnTy, Linkage: Function::ExternalLinkage, |
| 97 | N: "__orc_rt_reoptimize" , M: Mod.get()); |
| 98 | |
| 99 | // Set parameter names |
| 100 | auto ArgIt = ReOptimizeFn->arg_begin(); |
| 101 | Value *MUID = &*ArgIt++; |
| 102 | MUID->setName("MUID" ); |
| 103 | Value *CurVersion = &*ArgIt; |
| 104 | CurVersion->setName("CurVersion" ); |
| 105 | |
| 106 | // Build function body |
| 107 | BasicBlock *Entry = BasicBlock::Create(Context&: *Ctx, Name: "entry" , Parent: ReOptimizeFn); |
| 108 | Builder.SetInsertPoint(Entry); |
| 109 | |
| 110 | ExecutorAddr JITDispatchSym, JITDispatchCtxSym; |
| 111 | if (auto Err = |
| 112 | lookupAndApply(JD&: ES.getBootstrapJITDylib(), |
| 113 | PrepareFns: {recordAddr(Name: rt::DispatchName, A: &JITDispatchSym), |
| 114 | recordAddr(Name: rt::DispatchCtxName, A: &JITDispatchCtxSym)})) |
| 115 | return Err; |
| 116 | |
| 117 | Type *IntPtrTy = DL.getIntPtrType(C&: *Ctx); |
| 118 | Constant *JITDispatchPtr = ConstantExpr::getIntToPtr( |
| 119 | C: ConstantInt::get(Ty: IntPtrTy, V: JITDispatchSym.getValue()), Ty: VoidPtrTy); |
| 120 | Constant *JITDispatchCtxPtr = ConstantExpr::getIntToPtr( |
| 121 | C: ConstantInt::get(Ty: IntPtrTy, V: JITDispatchCtxSym.getValue()), Ty: VoidPtrTy); |
| 122 | Constant *HelperFnAddr = ConstantExpr::getIntToPtr( |
| 123 | C: ConstantInt::get(Ty: IntPtrTy, V: reinterpret_cast<uintptr_t>( |
| 124 | &orc_rt_lite_reoptimize_helper)), |
| 125 | Ty: PointerType::getUnqual(C&: *Ctx)); |
| 126 | |
| 127 | // Cast ReoptimizeTag to void* |
| 128 | Value *ReoptimizeTagPtr = Builder.CreatePointerCast(V: ReoptimizeTag, DestTy: VoidPtrTy); |
| 129 | |
| 130 | // Call the helper function |
| 131 | Builder.CreateCall( |
| 132 | FTy: HelperFnTy, Callee: HelperFnAddr, |
| 133 | Args: {JITDispatchPtr, JITDispatchCtxPtr, ReoptimizeTagPtr, MUID, CurVersion}); |
| 134 | |
| 135 | // Return void |
| 136 | Builder.CreateRetVoid(); |
| 137 | |
| 138 | return BaseLayer.add(JD&: PlatformJD, |
| 139 | TSM: ThreadSafeModule(std::move(Mod), std::move(Ctx))); |
| 140 | } |
| 141 | |
| 142 | Error ReOptimizeLayer::registerRuntimeFunctions(JITDylib &PlatformJD) { |
| 143 | return ES.registerCallControllerHandlers( |
| 144 | JD&: PlatformJD, |
| 145 | Hs: bindCallControllerHandler<reoptimize_sps_handlers::Reoptimize>( |
| 146 | Instance: this, Method: &ReOptimizeLayer::rt_reoptimize)); |
| 147 | } |
| 148 | |
| 149 | void ReOptimizeLayer::emit(std::unique_ptr<MaterializationResponsibility> R, |
| 150 | ThreadSafeModule TSM) { |
| 151 | auto &JD = R->getTargetJITDylib(); |
| 152 | |
| 153 | bool HasNonCallable = false; |
| 154 | for (auto &KV : R->getSymbols()) { |
| 155 | auto &Flags = KV.second; |
| 156 | if (!Flags.isCallable()) |
| 157 | HasNonCallable = true; |
| 158 | } |
| 159 | |
| 160 | if (HasNonCallable) { |
| 161 | BaseLayer.emit(R: std::move(R), TSM: std::move(TSM)); |
| 162 | return; |
| 163 | } |
| 164 | |
| 165 | auto &MUState = createMaterializationUnitState(TSM); |
| 166 | |
| 167 | if (auto Err = R->withResourceKeyDo(F: [&](ResourceKey Key) { |
| 168 | registerMaterializationUnitResource(Key, State&: MUState); |
| 169 | })) { |
| 170 | ES.reportError(Err: std::move(Err)); |
| 171 | R->failMaterialization(); |
| 172 | return; |
| 173 | } |
| 174 | |
| 175 | if (auto Err = |
| 176 | ProfilerFunc(*this, MUState.getID(), MUState.getCurVersion(), TSM)) { |
| 177 | ES.reportError(Err: std::move(Err)); |
| 178 | R->failMaterialization(); |
| 179 | return; |
| 180 | } |
| 181 | |
| 182 | auto InitialDests = |
| 183 | emitMUImplSymbols(MUState, Version: MUState.getCurVersion(), JD, TSM: std::move(TSM)); |
| 184 | if (!InitialDests) { |
| 185 | ES.reportError(Err: InitialDests.takeError()); |
| 186 | R->failMaterialization(); |
| 187 | return; |
| 188 | } |
| 189 | |
| 190 | RSManager.emitRedirectableSymbols(MR: std::move(R), InitialDests: std::move(*InitialDests)); |
| 191 | } |
| 192 | |
| 193 | Error ReOptimizeLayer::reoptimizeIfCallFrequent(ReOptimizeLayer &Parent, |
| 194 | ReOptMaterializationUnitID MUID, |
| 195 | unsigned CurVersion, |
| 196 | ThreadSafeModule &TSM) { |
| 197 | return TSM.withModuleDo(F: [&](Module &M) -> Error { |
| 198 | Type *I64Ty = Type::getInt64Ty(C&: M.getContext()); |
| 199 | GlobalVariable *Counter = new GlobalVariable( |
| 200 | M, I64Ty, false, GlobalValue::InternalLinkage, |
| 201 | Constant::getNullValue(Ty: I64Ty), "__orc_reopt_counter" ); |
| 202 | for (auto &F : M) { |
| 203 | if (F.isDeclaration()) |
| 204 | continue; |
| 205 | auto &BB = F.getEntryBlock(); |
| 206 | auto *IP = &*BB.getFirstInsertionPt(); |
| 207 | IRBuilder<> IRB(IP); |
| 208 | Value *Threshold = ConstantInt::get(Ty: I64Ty, V: CallCountThreshold, IsSigned: true); |
| 209 | Value *Cnt = IRB.CreateLoad(Ty: I64Ty, Ptr: Counter); |
| 210 | // Use EQ to prevent further reoptimize calls. |
| 211 | Value *Cmp = IRB.CreateICmpEQ(LHS: Cnt, RHS: Threshold); |
| 212 | Value *Added = IRB.CreateAdd(LHS: Cnt, RHS: ConstantInt::get(Ty: I64Ty, V: 1)); |
| 213 | (void)IRB.CreateStore(Val: Added, Ptr: Counter); |
| 214 | Instruction *SplitTerminator = SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore: IP, Unreachable: false); |
| 215 | createReoptimizeCall(M, IP&: *SplitTerminator, MUID, CurVersion); |
| 216 | } |
| 217 | return Error::success(); |
| 218 | }); |
| 219 | } |
| 220 | |
| 221 | Expected<SymbolMap> |
| 222 | ReOptimizeLayer::emitMUImplSymbols(ReOptMaterializationUnitState &MUState, |
| 223 | uint32_t Version, JITDylib &JD, |
| 224 | ThreadSafeModule TSM) { |
| 225 | DenseMap<SymbolStringPtr, SymbolStringPtr> RenamedMap; |
| 226 | cantFail(Err: TSM.withModuleDo(F: [&](Module &M) -> Error { |
| 227 | MangleAndInterner Mangle(ES, M.getDataLayout()); |
| 228 | for (auto &F : M) |
| 229 | if (!F.isDeclaration()) { |
| 230 | std::string NewName = |
| 231 | (F.getName() + ".__def__." + Twine(Version)).str(); |
| 232 | RenamedMap[Mangle(F.getName())] = Mangle(NewName); |
| 233 | F.setName(NewName); |
| 234 | } |
| 235 | return Error::success(); |
| 236 | })); |
| 237 | |
| 238 | auto RT = JD.createResourceTracker(); |
| 239 | if (auto Err = |
| 240 | JD.define(MU: std::make_unique<BasicIRLayerMaterializationUnit>( |
| 241 | args&: BaseLayer, args: *getManglingOptions(), args: std::move(TSM)), |
| 242 | RT)) |
| 243 | return Err; |
| 244 | MUState.setResourceTracker(RT); |
| 245 | |
| 246 | SymbolLookupSet LookupSymbols; |
| 247 | for (auto [K, V] : RenamedMap) |
| 248 | LookupSymbols.add(Name: V); |
| 249 | |
| 250 | auto ImplSymbols = |
| 251 | ES.lookup(SearchOrder: {{&JD, JITDylibLookupFlags::MatchAllSymbols}}, Symbols: LookupSymbols, |
| 252 | K: LookupKind::Static, RequiredState: SymbolState::Resolved); |
| 253 | if (auto Err = ImplSymbols.takeError()) |
| 254 | return Err; |
| 255 | |
| 256 | SymbolMap Result; |
| 257 | for (auto [K, V] : RenamedMap) |
| 258 | Result[K] = (*ImplSymbols)[V]; |
| 259 | |
| 260 | return Result; |
| 261 | } |
| 262 | |
| 263 | void ReOptimizeLayer::rt_reoptimize(SendErrorFn SendResult, |
| 264 | ReOptMaterializationUnitID MUID, |
| 265 | uint32_t CurVersion) { |
| 266 | auto &MUState = getMaterializationUnitState(MUID); |
| 267 | if (CurVersion < MUState.getCurVersion() || !MUState.tryStartReoptimize()) { |
| 268 | SendResult(Error::success()); |
| 269 | return; |
| 270 | } |
| 271 | |
| 272 | ThreadSafeModule TSM = cloneToNewContext(TSMW: MUState.getThreadSafeModule()); |
| 273 | auto OldRT = MUState.getResourceTracker(); |
| 274 | auto &JD = OldRT->getJITDylib(); |
| 275 | |
| 276 | if (auto Err = ReOptFunc(*this, MUID, CurVersion + 1, OldRT, TSM)) { |
| 277 | ES.reportError(Err: std::move(Err)); |
| 278 | MUState.reoptimizeFailed(); |
| 279 | SendResult(Error::success()); |
| 280 | return; |
| 281 | } |
| 282 | |
| 283 | auto SymbolDests = |
| 284 | emitMUImplSymbols(MUState, Version: CurVersion + 1, JD, TSM: std::move(TSM)); |
| 285 | if (!SymbolDests) { |
| 286 | ES.reportError(Err: SymbolDests.takeError()); |
| 287 | MUState.reoptimizeFailed(); |
| 288 | SendResult(Error::success()); |
| 289 | return; |
| 290 | } |
| 291 | |
| 292 | if (auto Err = RSManager.redirect(JD, NewDests: std::move(*SymbolDests))) { |
| 293 | ES.reportError(Err: std::move(Err)); |
| 294 | MUState.reoptimizeFailed(); |
| 295 | SendResult(Error::success()); |
| 296 | return; |
| 297 | } |
| 298 | |
| 299 | MUState.reoptimizeSucceeded(); |
| 300 | SendResult(Error::success()); |
| 301 | } |
| 302 | |
| 303 | void ReOptimizeLayer::createReoptimizeCall(Module &M, Instruction &IP, |
| 304 | ReOptMaterializationUnitID MUID, |
| 305 | uint32_t CurVersion) { |
| 306 | Type *MUIDTy = IntegerType::get(C&: M.getContext(), NumBits: 64); |
| 307 | Type *VersionTy = IntegerType::get(C&: M.getContext(), NumBits: 32); |
| 308 | Function *ReoptimizeFunc = M.getFunction(Name: "__orc_rt_reoptimize" ); |
| 309 | if (!ReoptimizeFunc) { |
| 310 | std::vector<Type *> ArgTys = {MUIDTy, VersionTy}; |
| 311 | FunctionType *FuncTy = |
| 312 | FunctionType::get(Result: Type::getVoidTy(C&: M.getContext()), Params: ArgTys, isVarArg: false); |
| 313 | ReoptimizeFunc = Function::Create(Ty: FuncTy, Linkage: GlobalValue::ExternalLinkage, |
| 314 | N: "__orc_rt_reoptimize" , M: &M); |
| 315 | } |
| 316 | Constant *MUIDArg = ConstantInt::get(Ty: MUIDTy, V: MUID, IsSigned: false); |
| 317 | Constant *CurVersionArg = ConstantInt::get(Ty: VersionTy, V: CurVersion, IsSigned: false); |
| 318 | IRBuilder<> IRB(&IP); |
| 319 | (void)IRB.CreateCall(Callee: ReoptimizeFunc, Args: {MUIDArg, CurVersionArg}); |
| 320 | } |
| 321 | |
| 322 | ReOptimizeLayer::ReOptMaterializationUnitState & |
| 323 | ReOptimizeLayer::createMaterializationUnitState(const ThreadSafeModule &TSM) { |
| 324 | std::unique_lock<std::mutex> Lock(Mutex); |
| 325 | ReOptMaterializationUnitID MUID = NextID; |
| 326 | MUStates.emplace(args&: MUID, |
| 327 | args: ReOptMaterializationUnitState(MUID, cloneToNewContext(TSMW: TSM))); |
| 328 | ++NextID; |
| 329 | return MUStates.at(k: MUID); |
| 330 | } |
| 331 | |
| 332 | ReOptimizeLayer::ReOptMaterializationUnitState & |
| 333 | ReOptimizeLayer::getMaterializationUnitState(ReOptMaterializationUnitID MUID) { |
| 334 | std::unique_lock<std::mutex> Lock(Mutex); |
| 335 | return MUStates.at(k: MUID); |
| 336 | } |
| 337 | |
| 338 | void ReOptimizeLayer::registerMaterializationUnitResource( |
| 339 | ResourceKey Key, ReOptMaterializationUnitState &State) { |
| 340 | std::unique_lock<std::mutex> Lock(Mutex); |
| 341 | MUResources[Key].insert(V: State.getID()); |
| 342 | } |
| 343 | |
| 344 | Error ReOptimizeLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) { |
| 345 | std::unique_lock<std::mutex> Lock(Mutex); |
| 346 | for (auto MUID : MUResources[K]) |
| 347 | MUStates.erase(x: MUID); |
| 348 | |
| 349 | MUResources.erase(Val: K); |
| 350 | return Error::success(); |
| 351 | } |
| 352 | |
| 353 | void ReOptimizeLayer::handleTransferResources(JITDylib &JD, ResourceKey DstK, |
| 354 | ResourceKey SrcK) { |
| 355 | std::unique_lock<std::mutex> Lock(Mutex); |
| 356 | MUResources[DstK].insert_range(R&: MUResources[SrcK]); |
| 357 | MUResources.erase(Val: SrcK); |
| 358 | } |
| 359 | |