| 1 | //===- MLInlineAdvisor.cpp - machine learned InlineAdvisor ----------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements the interface between the inliner and a learned model. |
| 10 | // It delegates model evaluation to either the AOT compiled model (the |
| 11 | // 'release' mode) or a runtime-loaded model (the 'development' case). |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | #include "llvm/Analysis/MLInlineAdvisor.h" |
| 15 | #include "AnalysisOptions.h" |
| 16 | #include "llvm/ADT/SCCIterator.h" |
| 17 | #include "llvm/Analysis/AssumptionCache.h" |
| 18 | #include "llvm/Analysis/BlockFrequencyInfo.h" |
| 19 | #include "llvm/Analysis/CallGraph.h" |
| 20 | #include "llvm/Analysis/FunctionPropertiesAnalysis.h" |
| 21 | #include "llvm/Analysis/InlineCost.h" |
| 22 | #include "llvm/Analysis/InlineModelFeatureMaps.h" |
| 23 | #include "llvm/Analysis/LazyCallGraph.h" |
| 24 | #include "llvm/Analysis/LoopInfo.h" |
| 25 | #include "llvm/Analysis/MLModelRunner.h" |
| 26 | #include "llvm/Analysis/OptimizationRemarkEmitter.h" |
| 27 | #include "llvm/Analysis/ProfileSummaryInfo.h" |
| 28 | #include "llvm/Analysis/ReleaseModeModelRunner.h" |
| 29 | #include "llvm/Analysis/TargetTransformInfo.h" |
| 30 | #include "llvm/Analysis/TensorSpec.h" |
| 31 | #include "llvm/Analysis/Utils/MLGOUtils.h" |
| 32 | #include "llvm/IR/Dominators.h" |
| 33 | #include "llvm/IR/InstIterator.h" |
| 34 | #include "llvm/IR/Module.h" |
| 35 | #include "llvm/IR/PassManager.h" |
| 36 | #include "llvm/Support/CommandLine.h" |
| 37 | |
| 38 | using namespace llvm; |
| 39 | |
| 40 | #if defined(LLVM_HAVE_TF_AOT_INLINERSIZEMODEL) |
| 41 | // codegen-ed file |
| 42 | #include "InlinerSizeModel.h" // NOLINT |
| 43 | using CompiledModelType = llvm::InlinerSizeModel; |
| 44 | #else |
| 45 | using CompiledModelType = NoopSavedModelImpl; |
| 46 | #endif |
| 47 | |
| 48 | #include "llvm/Analysis/EmitCModelRunner.h" |
| 49 | #include "llvm/Analysis/InlinerModels.h" |
| 50 | |
| 51 | enum class EmitCModelChoice { |
| 52 | Default, |
| 53 | #define MLGO_MODEL(CLASS_NAME, CLI_FLAG) CLASS_NAME, |
| 54 | #include "llvm/Analysis/InlinerModels.def" |
| 55 | }; |
| 56 | |
| 57 | static llvm::cl::opt<EmitCModelChoice> SelectedMLGOModel( |
| 58 | "mlgo-model" , llvm::cl::desc("Select the MLGO model to execute:" ), |
| 59 | llvm::cl::init(Val: EmitCModelChoice::Default), |
| 60 | llvm::cl::values(clEnumValN(EmitCModelChoice::Default, "default" , |
| 61 | "Use standard heuristic" ) |
| 62 | #define MLGO_MODEL(CLASS_NAME, CLI_FLAG) \ |
| 63 | , clEnumValN(EmitCModelChoice::CLASS_NAME, CLI_FLAG, \ |
| 64 | "Use the " CLI_FLAG " MLGO model") |
| 65 | #include "llvm/Analysis/InlinerModels.def" |
| 66 | )); |
| 67 | |
| 68 | static std::unique_ptr<MLModelRunner> |
| 69 | createEmitCModelRunner(LLVMContext &Ctx, |
| 70 | const std::vector<TensorSpec> &InputFeatures) { |
| 71 | switch (SelectedMLGOModel) { |
| 72 | case EmitCModelChoice::Default: |
| 73 | return nullptr; |
| 74 | #define MLGO_MODEL(CLASS_NAME, CLI_FLAG) \ |
| 75 | case EmitCModelChoice::CLASS_NAME: \ |
| 76 | return std::make_unique<EmitCModelRunner<CLASS_NAME>>(Ctx, InputFeatures); |
| 77 | #include "llvm/Analysis/InlinerModels.def" |
| 78 | } |
| 79 | llvm_unreachable("Unknown MLGO model type!" ); |
| 80 | } |
| 81 | |
| 82 | std::unique_ptr<InlineAdvisor> |
| 83 | llvm::getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM, |
| 84 | std::function<bool(CallBase &)> GetDefaultAdvice) { |
| 85 | const AnalysisOptions &Opts = getAnalysisOptions(M); |
| 86 | if (!isReleaseModelValid<CompiledModelType>( |
| 87 | InteractiveChannelBaseName: Opts.inliner_interactive_channel_base, SelectedModel: SelectedMLGOModel)) |
| 88 | return nullptr; |
| 89 | auto RunnerFactory = [&](const std::vector<TensorSpec> &InputFeatures) |
| 90 | -> std::unique_ptr<MLModelRunner> { |
| 91 | return createReleaseModeModelRunner<CompiledModelType>( |
| 92 | Ctx&: M.getContext(), InputFeatures, DecisionName, |
| 93 | InteractiveChannelBaseName: Opts.inliner_interactive_channel_base.str(), InteractiveDecisionSpec: InlineDecisionSpec, |
| 94 | CreateEmitCModelRunner&: createEmitCModelRunner, |
| 95 | Options: EmbeddedModelRunnerOptions().setModelSelector( |
| 96 | Opts.ml_inliner_model_selector)); |
| 97 | }; |
| 98 | return std::make_unique<MLInlineAdvisor>(args&: M, args&: MAM, args&: RunnerFactory, |
| 99 | args&: GetDefaultAdvice); |
| 100 | } |
| 101 | |
| 102 | #define DEBUG_TYPE "inline-ml" |
| 103 | |
| 104 | const std::vector<TensorSpec> &MLInlineAdvisor::getInitialFeatureMap() { |
| 105 | // clang-format off |
| 106 | static std::vector<TensorSpec> FeatureMap{ |
| 107 | #define POPULATE_NAMES(DTYPE, SHAPE, NAME, __) TensorSpec::createSpec<DTYPE>(#NAME, SHAPE), |
| 108 | // InlineCost features - these must come first |
| 109 | INLINE_COST_FEATURE_ITERATOR(POPULATE_NAMES) |
| 110 | |
| 111 | // Non-cost features |
| 112 | INLINE_FEATURE_ITERATOR(POPULATE_NAMES) |
| 113 | #undef POPULATE_NAMES |
| 114 | }; |
| 115 | // clang-format on |
| 116 | return FeatureMap; |
| 117 | } |
| 118 | |
| 119 | const char *const llvm::DecisionName = "inlining_decision" ; |
| 120 | const TensorSpec llvm::InlineDecisionSpec = |
| 121 | TensorSpec::createSpec<int64_t>(Name: DecisionName, Shape: {1}); |
| 122 | const char *const llvm::DefaultDecisionName = "inlining_default" ; |
| 123 | const TensorSpec llvm::DefaultDecisionSpec = |
| 124 | TensorSpec::createSpec<int64_t>(Name: DefaultDecisionName, Shape: {1}); |
| 125 | const char *const llvm::RewardName = "delta_size" ; |
| 126 | |
| 127 | CallBase *getInlinableCS(Instruction &I) { |
| 128 | if (auto *CS = dyn_cast<CallBase>(Val: &I)) |
| 129 | if (Function *Callee = CS->getCalledFunction()) { |
| 130 | if (!Callee->isDeclaration()) { |
| 131 | return CS; |
| 132 | } |
| 133 | } |
| 134 | return nullptr; |
| 135 | } |
| 136 | |
| 137 | MLInlineAdvisor::MLInlineAdvisor( |
| 138 | Module &M, ModuleAnalysisManager &MAM, |
| 139 | std::function< |
| 140 | std::unique_ptr<MLModelRunner>(const std::vector<TensorSpec> &)> |
| 141 | GetModelRunner, |
| 142 | std::function<bool(CallBase &)> GetDefaultAdvice) |
| 143 | : InlineAdvisor( |
| 144 | M, MAM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager()), |
| 145 | Opts(getAnalysisOptions(M)), GetDefaultAdvice(GetDefaultAdvice), |
| 146 | FeatureMap(getInitialFeatureMap()), |
| 147 | CG(MAM.getResult<LazyCallGraphAnalysis>(IR&: M)), |
| 148 | UseIR2Vec(MAM.getCachedResult<IR2VecVocabAnalysis>(IR&: M) != nullptr), |
| 149 | InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize), |
| 150 | PSI(MAM.getResult<ProfileSummaryAnalysis>(IR&: M)) { |
| 151 | // Extract the 'call site height' feature - the position of a call site |
| 152 | // relative to the farthest statically reachable SCC node. We don't mutate |
| 153 | // this value while inlining happens. Empirically, this feature proved |
| 154 | // critical in behavioral cloning - i.e. training a model to mimic the manual |
| 155 | // heuristic's decisions - and, thus, equally important for training for |
| 156 | // improvement. |
| 157 | CallGraph CGraph(M); |
| 158 | for (auto I = scc_begin(G: &CGraph); !I.isAtEnd(); ++I) { |
| 159 | const std::vector<CallGraphNode *> &CGNodes = *I; |
| 160 | unsigned Level = 0; |
| 161 | for (auto *CGNode : CGNodes) { |
| 162 | Function *F = CGNode->getFunction(); |
| 163 | if (!F || F->isDeclaration()) |
| 164 | continue; |
| 165 | for (auto &I : instructions(F)) { |
| 166 | if (auto *CS = getInlinableCS(I)) { |
| 167 | auto *Called = CS->getCalledFunction(); |
| 168 | auto Pos = FunctionLevels.find(x: &CG.get(F&: *Called)); |
| 169 | // In bottom up traversal, an inlinable callee is either in the |
| 170 | // same SCC, or to a function in a visited SCC. So not finding its |
| 171 | // level means we haven't visited it yet, meaning it's in this SCC. |
| 172 | if (Pos == FunctionLevels.end()) |
| 173 | continue; |
| 174 | Level = std::max(a: Level, b: Pos->second + 1); |
| 175 | } |
| 176 | } |
| 177 | } |
| 178 | for (auto *CGNode : CGNodes) { |
| 179 | Function *F = CGNode->getFunction(); |
| 180 | if (F && !F->isDeclaration()) |
| 181 | FunctionLevels[&CG.get(F&: *F)] = Level; |
| 182 | } |
| 183 | } |
| 184 | for (auto KVP : FunctionLevels) { |
| 185 | AllNodes.insert(V: KVP.first); |
| 186 | EdgeCount += getLocalCalls(F&: KVP.first->getFunction()); |
| 187 | } |
| 188 | NodeCount = AllNodes.size(); |
| 189 | |
| 190 | if (auto *IR2VecVocabResult = MAM.getCachedResult<IR2VecVocabAnalysis>(IR&: M)) { |
| 191 | if (!IR2VecVocabResult->isValid()) { |
| 192 | M.getContext().emitError(ErrorStr: "IR2VecVocabAnalysis is not valid" ); |
| 193 | return; |
| 194 | } |
| 195 | // Add the IR2Vec features to the feature map |
| 196 | auto IR2VecDim = IR2VecVocabResult->getDimension(); |
| 197 | FeatureMap.push_back( |
| 198 | x: TensorSpec::createSpec<float>(Name: "callee_embedding" , Shape: {IR2VecDim})); |
| 199 | FeatureMap.push_back( |
| 200 | x: TensorSpec::createSpec<float>(Name: "caller_embedding" , Shape: {IR2VecDim})); |
| 201 | } |
| 202 | if (Opts.inliner_interactive_include_default) |
| 203 | FeatureMap.push_back(x: DefaultDecisionSpec); |
| 204 | |
| 205 | ModelRunner = GetModelRunner(getFeatureMap()); |
| 206 | if (!ModelRunner) { |
| 207 | M.getContext().emitError(ErrorStr: "Could not create model runner" ); |
| 208 | return; |
| 209 | } |
| 210 | ModelRunner->switchContext(Name: "" ); |
| 211 | ForceStop = Opts.ml_inliner_stop_immediately; |
| 212 | } |
| 213 | |
| 214 | unsigned MLInlineAdvisor::getInitialFunctionLevel(const Function &F) const { |
| 215 | return CG.lookup(F) ? FunctionLevels.at(k: CG.lookup(F)) : 0; |
| 216 | } |
| 217 | |
| 218 | void MLInlineAdvisor::onPassEntry(LazyCallGraph::SCC *CurSCC) { |
| 219 | if (!CurSCC || ForceStop) |
| 220 | return; |
| 221 | FPICache.clear(); |
| 222 | // Function passes executed between InlinerPass runs may have changed the |
| 223 | // module-wide features. |
| 224 | // The cgscc pass manager rules are such that: |
| 225 | // - if a pass leads to merging SCCs, then the pipeline is restarted on the |
| 226 | // merged SCC |
| 227 | // - if a pass leads to splitting the SCC, then we continue with one of the |
| 228 | // splits |
| 229 | // This means that the NodesInLastSCC is a superset (not strict) of the nodes |
| 230 | // that subsequent passes would have processed |
| 231 | // - in addition, if new Nodes were created by a pass (e.g. CoroSplit), |
| 232 | // they'd be adjacent to Nodes in the last SCC. So we just need to check the |
| 233 | // boundary of Nodes in NodesInLastSCC for Nodes we haven't seen. We don't |
| 234 | // care about the nature of the Edge (call or ref). `FunctionLevels`-wise, we |
| 235 | // record them at the same level as the original node (this is a choice, may |
| 236 | // need revisiting). |
| 237 | // - nodes are only deleted at the end of a call graph walk where they are |
| 238 | // batch deleted, so we shouldn't see any dead nodes here. |
| 239 | while (!NodesInLastSCC.empty()) { |
| 240 | const auto *N = *NodesInLastSCC.begin(); |
| 241 | assert(!N->isDead()); |
| 242 | NodesInLastSCC.erase(Ptr: N); |
| 243 | EdgeCount += getLocalCalls(F&: N->getFunction()); |
| 244 | const auto NLevel = FunctionLevels.at(k: N); |
| 245 | for (const auto &E : *(*N)) { |
| 246 | const auto *AdjNode = &E.getNode(); |
| 247 | assert(!AdjNode->isDead() && !AdjNode->getFunction().isDeclaration()); |
| 248 | auto I = AllNodes.insert(V: AdjNode); |
| 249 | // We've discovered a new function. |
| 250 | if (I.second) { |
| 251 | ++NodeCount; |
| 252 | NodesInLastSCC.insert(Ptr: AdjNode); |
| 253 | FunctionLevels[AdjNode] = NLevel; |
| 254 | } |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | EdgeCount -= EdgesOfLastSeenNodes; |
| 259 | EdgesOfLastSeenNodes = 0; |
| 260 | |
| 261 | // (Re)use NodesInLastSCC to remember the nodes in the SCC right now, |
| 262 | // in case the SCC is split before onPassExit and some nodes are split out |
| 263 | assert(NodesInLastSCC.empty()); |
| 264 | for (const auto &N : *CurSCC) |
| 265 | NodesInLastSCC.insert(Ptr: &N); |
| 266 | } |
| 267 | |
| 268 | void MLInlineAdvisor::onPassExit(LazyCallGraph::SCC *CurSCC) { |
| 269 | // No need to keep this around - function passes will invalidate it. |
| 270 | if (!Opts.ml_advisor_keep_fpi_cache) |
| 271 | FPICache.clear(); |
| 272 | if (!CurSCC || ForceStop) |
| 273 | return; |
| 274 | // Keep track of the nodes and edges we last saw. Then, in onPassEntry, |
| 275 | // we update the node count and edge count from the subset of these nodes that |
| 276 | // survived. |
| 277 | EdgesOfLastSeenNodes = 0; |
| 278 | |
| 279 | // Check on nodes that were in SCC onPassEntry |
| 280 | for (const LazyCallGraph::Node *N : NodesInLastSCC) { |
| 281 | assert(!N->isDead()); |
| 282 | EdgesOfLastSeenNodes += getLocalCalls(F&: N->getFunction()); |
| 283 | } |
| 284 | |
| 285 | // Check on nodes that may have got added to SCC |
| 286 | for (const auto &N : *CurSCC) { |
| 287 | assert(!N.isDead()); |
| 288 | auto I = NodesInLastSCC.insert(Ptr: &N); |
| 289 | if (I.second) |
| 290 | EdgesOfLastSeenNodes += getLocalCalls(F&: N.getFunction()); |
| 291 | } |
| 292 | assert(NodeCount >= NodesInLastSCC.size()); |
| 293 | assert(EdgeCount >= EdgesOfLastSeenNodes); |
| 294 | } |
| 295 | |
| 296 | int64_t MLInlineAdvisor::getLocalCalls(Function &F) { |
| 297 | return getCachedFPI(F).DirectCallsToDefinedFunctions; |
| 298 | } |
| 299 | |
| 300 | // Update the internal state of the advisor, and force invalidate feature |
| 301 | // analysis. Currently, we maintain minimal (and very simple) global state - the |
| 302 | // number of functions and the number of static calls. We also keep track of the |
| 303 | // total IR size in this module, to stop misbehaving policies at a certain bloat |
| 304 | // factor (-ml-advisor-size-increase-threshold) |
| 305 | void MLInlineAdvisor::onSuccessfulInlining(const MLInlineAdvice &Advice, |
| 306 | bool CalleeWasDeleted) { |
| 307 | assert(!ForceStop); |
| 308 | Function *Caller = Advice.getCaller(); |
| 309 | Function *Callee = Advice.getCallee(); |
| 310 | // The caller features aren't valid anymore. |
| 311 | { |
| 312 | PreservedAnalyses PA = PreservedAnalyses::all(); |
| 313 | PA.abandon<FunctionPropertiesAnalysis>(); |
| 314 | PA.abandon<LoopAnalysis>(); |
| 315 | FAM.invalidate(IR&: *Caller, PA); |
| 316 | } |
| 317 | Advice.updateCachedCallerFPI(FAM); |
| 318 | if (Caller == Callee) { |
| 319 | assert(!CalleeWasDeleted); |
| 320 | // We double-counted CallerAndCalleeEdges - since the caller and callee |
| 321 | // would be the same |
| 322 | assert(Advice.CallerAndCalleeEdges % 2 == 0); |
| 323 | CurrentIRSize += getIRSize(F&: *Caller) - Advice.CallerIRSize; |
| 324 | EdgeCount += getCachedFPI(*Caller).DirectCallsToDefinedFunctions - |
| 325 | Advice.CallerAndCalleeEdges / 2; |
| 326 | // The NodeCount would stay the same. |
| 327 | } else { |
| 328 | int64_t IRSizeAfter = |
| 329 | getIRSize(F&: *Caller) + (CalleeWasDeleted ? 0 : Advice.CalleeIRSize); |
| 330 | CurrentIRSize += IRSizeAfter - (Advice.CallerIRSize + Advice.CalleeIRSize); |
| 331 | |
| 332 | // We can delta-update module-wide features. We know the inlining only |
| 333 | // changed the caller, and maybe the callee (by deleting the latter). Nodes |
| 334 | // are simple to update. For edges, we 'forget' the edges that the caller |
| 335 | // and callee used to have before inlining, and add back what they currently |
| 336 | // have together. |
| 337 | int64_t NewCallerAndCalleeEdges = |
| 338 | getCachedFPI(*Caller).DirectCallsToDefinedFunctions; |
| 339 | |
| 340 | // A dead function's node is not actually removed from the call graph until |
| 341 | // the end of the call graph walk, but the node no longer belongs to any |
| 342 | // valid SCC. |
| 343 | if (CalleeWasDeleted) { |
| 344 | --NodeCount; |
| 345 | NodesInLastSCC.erase(Ptr: CG.lookup(F: *Callee)); |
| 346 | DeadFunctions.insert(V: Callee); |
| 347 | } else { |
| 348 | NewCallerAndCalleeEdges += |
| 349 | getCachedFPI(*Callee).DirectCallsToDefinedFunctions; |
| 350 | } |
| 351 | EdgeCount += (NewCallerAndCalleeEdges - Advice.CallerAndCalleeEdges); |
| 352 | } |
| 353 | if (CurrentIRSize > Opts.ml_advisor_size_increase_threshold * InitialIRSize) |
| 354 | ForceStop = true; |
| 355 | |
| 356 | assert(CurrentIRSize >= 0 && EdgeCount >= 0 && NodeCount >= 0); |
| 357 | } |
| 358 | |
| 359 | int64_t MLInlineAdvisor::getModuleIRSize() const { |
| 360 | int64_t Ret = 0; |
| 361 | for (auto &F : M) |
| 362 | if (!F.isDeclaration()) |
| 363 | Ret += getIRSize(F); |
| 364 | return Ret; |
| 365 | } |
| 366 | |
| 367 | FunctionPropertiesInfo &MLInlineAdvisor::getCachedFPI(Function &F) const { |
| 368 | auto InsertPair = FPICache.try_emplace(k: &F); |
| 369 | if (!InsertPair.second) |
| 370 | return InsertPair.first->second; |
| 371 | InsertPair.first->second = FAM.getResult<FunctionPropertiesAnalysis>(IR&: F); |
| 372 | return InsertPair.first->second; |
| 373 | } |
| 374 | |
| 375 | std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdviceImpl(CallBase &CB) { |
| 376 | if (auto Skip = getSkipAdviceIfUnreachableCallsite(CB)) |
| 377 | return Skip; |
| 378 | |
| 379 | auto &Caller = *CB.getCaller(); |
| 380 | auto &Callee = *CB.getCalledFunction(); |
| 381 | |
| 382 | auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { |
| 383 | return FAM.getResult<AssumptionAnalysis>(IR&: F); |
| 384 | }; |
| 385 | auto &TIR = FAM.getResult<TargetIRAnalysis>(IR&: Caller); |
| 386 | auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: Caller); |
| 387 | |
| 388 | if (Opts.ml_inliner_skip_policy == SkipMLPolicyCriteria::IfCallerIsNotCold) { |
| 389 | if (!PSI.isFunctionEntryCold(F: &Caller)) { |
| 390 | // Return a MLInlineAdvice, despite delegating to the default advice, |
| 391 | // because we need to keep track of the internal state. This is different |
| 392 | // from the other instances where we return a "default" InlineAdvice, |
| 393 | // which happen at points we won't come back to the MLAdvisor for |
| 394 | // decisions requiring that state. |
| 395 | return ForceStop ? std::make_unique<InlineAdvice>(args: this, args&: CB, args&: ORE, |
| 396 | args: GetDefaultAdvice(CB)) |
| 397 | : std::make_unique<MLInlineAdvice>(args: this, args&: CB, args&: ORE, |
| 398 | args: GetDefaultAdvice(CB)); |
| 399 | } |
| 400 | } |
| 401 | auto MandatoryKind = InlineAdvisor::getMandatoryKind(CB, FAM, ORE); |
| 402 | // If this is a "never inline" case, there won't be any changes to internal |
| 403 | // state we need to track, so we can just return the base InlineAdvice, which |
| 404 | // will do nothing interesting. |
| 405 | // Same thing if this is a recursive case. |
| 406 | if (MandatoryKind == InlineAdvisor::MandatoryInliningKind::Never || |
| 407 | &Caller == &Callee) |
| 408 | return getMandatoryAdvice(CB, Advice: false); |
| 409 | |
| 410 | bool Mandatory = |
| 411 | MandatoryKind == InlineAdvisor::MandatoryInliningKind::Always; |
| 412 | |
| 413 | // If we need to stop, we won't want to track anymore any state changes, so |
| 414 | // we just return the base InlineAdvice, which acts as a noop. |
| 415 | if (ForceStop) { |
| 416 | ORE.emit(RemarkBuilder: [&] { |
| 417 | return OptimizationRemarkMissed(DEBUG_TYPE, "ForceStop" , &CB) |
| 418 | << "Won't attempt inlining because module size grew too much." ; |
| 419 | }); |
| 420 | return std::make_unique<InlineAdvice>(args: this, args&: CB, args&: ORE, args&: Mandatory); |
| 421 | } |
| 422 | |
| 423 | int CostEstimate = 0; |
| 424 | if (!Mandatory) { |
| 425 | auto IsCallSiteInlinable = |
| 426 | llvm::getInliningCostEstimate(Call&: CB, CallerTTI&: TIR, GetAssumptionCache); |
| 427 | if (!IsCallSiteInlinable) { |
| 428 | // We can't inline this for correctness reasons, so return the base |
| 429 | // InlineAdvice, as we don't care about tracking any state changes (which |
| 430 | // won't happen). |
| 431 | return std::make_unique<InlineAdvice>(args: this, args&: CB, args&: ORE, args: false); |
| 432 | } |
| 433 | CostEstimate = *IsCallSiteInlinable; |
| 434 | } |
| 435 | |
| 436 | const auto CostFeatures = |
| 437 | llvm::getInliningCostFeatures(Call&: CB, CallerTTI&: TIR, GetAssumptionCache); |
| 438 | if (!CostFeatures) { |
| 439 | return std::make_unique<InlineAdvice>(args: this, args&: CB, args&: ORE, args: false); |
| 440 | } |
| 441 | |
| 442 | if (Mandatory) |
| 443 | return getMandatoryAdvice(CB, Advice: true); |
| 444 | |
| 445 | auto NumCtantParams = 0; |
| 446 | for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) { |
| 447 | NumCtantParams += (isa<Constant>(Val: *I)); |
| 448 | } |
| 449 | |
| 450 | auto &CallerBefore = getCachedFPI(F&: Caller); |
| 451 | auto &CalleeBefore = getCachedFPI(F&: Callee); |
| 452 | |
| 453 | *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::callee_basic_block_count) = |
| 454 | CalleeBefore.BasicBlockCount; |
| 455 | *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::callsite_height) = |
| 456 | getInitialFunctionLevel(F: Caller); |
| 457 | *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::node_count) = NodeCount; |
| 458 | *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::nr_ctant_params) = |
| 459 | NumCtantParams; |
| 460 | *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::edge_count) = EdgeCount; |
| 461 | *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::caller_users) = |
| 462 | CallerBefore.Uses; |
| 463 | *ModelRunner->getTensor<int64_t>( |
| 464 | FeatureID: FeatureIndex::caller_conditionally_executed_blocks) = |
| 465 | CallerBefore.BlocksReachedFromConditionalInstruction; |
| 466 | *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::caller_basic_block_count) = |
| 467 | CallerBefore.BasicBlockCount; |
| 468 | *ModelRunner->getTensor<int64_t>( |
| 469 | FeatureID: FeatureIndex::callee_conditionally_executed_blocks) = |
| 470 | CalleeBefore.BlocksReachedFromConditionalInstruction; |
| 471 | *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::callee_users) = |
| 472 | CalleeBefore.Uses; |
| 473 | *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::cost_estimate) = CostEstimate; |
| 474 | *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::is_callee_avail_external) = |
| 475 | Callee.hasAvailableExternallyLinkage(); |
| 476 | *ModelRunner->getTensor<int64_t>(FeatureID: FeatureIndex::is_caller_avail_external) = |
| 477 | Caller.hasAvailableExternallyLinkage(); |
| 478 | |
| 479 | if (UseIR2Vec) { |
| 480 | // Python side expects float embeddings. The IR2Vec embeddings are doubles |
| 481 | // as of now due to the restriction of fromJSON method used by the |
| 482 | // readVocabulary method in ir2vec::Embeddings. |
| 483 | auto setEmbedding = [&](const ir2vec::Embedding &Embedding, |
| 484 | FeatureIndex Index) { |
| 485 | llvm::transform(Range: Embedding, d_first: ModelRunner->getTensor<float>(FeatureID: Index), |
| 486 | F: [](double Val) { return static_cast<float>(Val); }); |
| 487 | }; |
| 488 | |
| 489 | setEmbedding(CalleeBefore.getFunctionEmbedding(), |
| 490 | FeatureIndex::callee_embedding); |
| 491 | setEmbedding(CallerBefore.getFunctionEmbedding(), |
| 492 | FeatureIndex::caller_embedding); |
| 493 | } |
| 494 | |
| 495 | // Add the cost features |
| 496 | for (size_t I = 0; |
| 497 | I < static_cast<size_t>(InlineCostFeatureIndex::NumberOfFeatures); ++I) { |
| 498 | *ModelRunner->getTensor<int64_t>(FeatureID: inlineCostFeatureToMlFeature( |
| 499 | Feature: static_cast<InlineCostFeatureIndex>(I))) = CostFeatures->at(n: I); |
| 500 | } |
| 501 | // This one would have been set up to be right at the end. |
| 502 | if (!Opts.inliner_interactive_channel_base.empty() && |
| 503 | Opts.inliner_interactive_include_default) |
| 504 | *ModelRunner->getTensor<int64_t>(FeatureID: getFeatureMap().size() - 1) = |
| 505 | GetDefaultAdvice(CB); |
| 506 | return getAdviceFromModel(CB, ORE); |
| 507 | } |
| 508 | |
| 509 | std::unique_ptr<MLInlineAdvice> |
| 510 | MLInlineAdvisor::(CallBase &CB, |
| 511 | OptimizationRemarkEmitter &ORE) { |
| 512 | return std::make_unique<MLInlineAdvice>( |
| 513 | args: this, args&: CB, args&: ORE, args: static_cast<bool>(ModelRunner->evaluate<int64_t>())); |
| 514 | } |
| 515 | |
| 516 | std::unique_ptr<InlineAdvice> |
| 517 | MLInlineAdvisor::getSkipAdviceIfUnreachableCallsite(CallBase &CB) { |
| 518 | if (!FAM.getResult<DominatorTreeAnalysis>(IR&: *CB.getCaller()) |
| 519 | .isReachableFromEntry(A: CB.getParent())) |
| 520 | return std::make_unique<InlineAdvice>(args: this, args&: CB, args&: getCallerORE(CB), args: false); |
| 521 | return nullptr; |
| 522 | } |
| 523 | |
| 524 | std::unique_ptr<InlineAdvice> MLInlineAdvisor::getMandatoryAdvice(CallBase &CB, |
| 525 | bool Advice) { |
| 526 | // Make sure we track inlinings in all cases - mandatory or not. |
| 527 | if (auto Skip = getSkipAdviceIfUnreachableCallsite(CB)) |
| 528 | return Skip; |
| 529 | if (Advice && !ForceStop) |
| 530 | return getMandatoryAdviceImpl(CB); |
| 531 | |
| 532 | // If this is a "never inline" case, there won't be any changes to internal |
| 533 | // state we need to track, so we can just return the base InlineAdvice, which |
| 534 | // will do nothing interesting. |
| 535 | // Same if we are forced to stop - we don't track anymore. |
| 536 | return std::make_unique<InlineAdvice>(args: this, args&: CB, args&: getCallerORE(CB), args&: Advice); |
| 537 | } |
| 538 | |
| 539 | std::unique_ptr<MLInlineAdvice> |
| 540 | MLInlineAdvisor::getMandatoryAdviceImpl(CallBase &CB) { |
| 541 | return std::make_unique<MLInlineAdvice>(args: this, args&: CB, args&: getCallerORE(CB), args: true); |
| 542 | } |
| 543 | |
| 544 | void MLInlineAdvisor::print(raw_ostream &OS) const { |
| 545 | OS << "[MLInlineAdvisor] Nodes: " << NodeCount << " Edges: " << EdgeCount |
| 546 | << " EdgesOfLastSeenNodes: " << EdgesOfLastSeenNodes << "\n" ; |
| 547 | OS << "[MLInlineAdvisor] FPI:\n" ; |
| 548 | for (auto I : FPICache) { |
| 549 | OS << I.first->getName() << ":\n" ; |
| 550 | I.second.print(OS); |
| 551 | OS << "\n" ; |
| 552 | } |
| 553 | OS << "\n" ; |
| 554 | OS << "[MLInlineAdvisor] FuncLevels:\n" ; |
| 555 | for (auto I : FunctionLevels) |
| 556 | OS << (DeadFunctions.contains(V: &I.first->getFunction()) |
| 557 | ? "<deleted>" |
| 558 | : I.first->getFunction().getName()) |
| 559 | << " : " << I.second << "\n" ; |
| 560 | |
| 561 | OS << "\n" ; |
| 562 | } |
| 563 | |
| 564 | MLInlineAdvice::(MLInlineAdvisor *Advisor, CallBase &CB, |
| 565 | OptimizationRemarkEmitter &ORE, |
| 566 | bool Recommendation) |
| 567 | : InlineAdvice(Advisor, CB, ORE, Recommendation), |
| 568 | CallerIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(F&: *Caller)), |
| 569 | CalleeIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(F&: *Callee)), |
| 570 | CallerAndCalleeEdges(Advisor->isForcedToStop() |
| 571 | ? 0 |
| 572 | : (Advisor->getLocalCalls(F&: *Caller) + |
| 573 | Advisor->getLocalCalls(F&: *Callee))), |
| 574 | PreInlineCallerFPI(Advisor->getCachedFPI(F&: *Caller)) { |
| 575 | if (Recommendation) |
| 576 | FPU.emplace(args&: Advisor->getCachedFPI(F&: *getCaller()), args&: CB); |
| 577 | } |
| 578 | |
| 579 | void MLInlineAdvice::( |
| 580 | DiagnosticInfoOptimizationBase &OR) { |
| 581 | using namespace ore; |
| 582 | OR << NV("Callee" , Callee->getName()); |
| 583 | for (size_t I = 0; I < getAdvisor()->getFeatureMap().size(); ++I) |
| 584 | OR << NV(getAdvisor()->getFeatureMap()[I].name(), |
| 585 | *getAdvisor()->getModelRunner().getTensor<int64_t>(FeatureID: I)); |
| 586 | OR << NV("ShouldInline" , isInliningRecommended()); |
| 587 | } |
| 588 | |
| 589 | void MLInlineAdvice::updateCachedCallerFPI(FunctionAnalysisManager &FAM) const { |
| 590 | FPU->finish(FAM); |
| 591 | } |
| 592 | |
| 593 | void MLInlineAdvice::recordInliningImpl() { |
| 594 | ORE.emit(RemarkBuilder: [&]() { |
| 595 | OptimizationRemark R(DEBUG_TYPE, "InliningSuccess" , DLoc, Block); |
| 596 | reportContextForRemark(OR&: R); |
| 597 | return R; |
| 598 | }); |
| 599 | getAdvisor()->onSuccessfulInlining(Advice: *this, /*CalleeWasDeleted*/ false); |
| 600 | } |
| 601 | |
| 602 | void MLInlineAdvice::recordInliningWithCalleeDeletedImpl() { |
| 603 | ORE.emit(RemarkBuilder: [&]() { |
| 604 | OptimizationRemark R(DEBUG_TYPE, "InliningSuccessWithCalleeDeleted" , DLoc, |
| 605 | Block); |
| 606 | reportContextForRemark(OR&: R); |
| 607 | return R; |
| 608 | }); |
| 609 | getAdvisor()->onSuccessfulInlining(Advice: *this, /*CalleeWasDeleted*/ true); |
| 610 | } |
| 611 | |
| 612 | void MLInlineAdvice::recordUnsuccessfulInliningImpl( |
| 613 | const InlineResult &Result) { |
| 614 | getAdvisor()->getCachedFPI(F&: *Caller) = PreInlineCallerFPI; |
| 615 | ORE.emit(RemarkBuilder: [&]() { |
| 616 | OptimizationRemarkMissed R(DEBUG_TYPE, "InliningAttemptedAndUnsuccessful" , |
| 617 | DLoc, Block); |
| 618 | reportContextForRemark(OR&: R); |
| 619 | return R; |
| 620 | }); |
| 621 | } |
| 622 | void MLInlineAdvice::recordUnattemptedInliningImpl() { |
| 623 | assert(!FPU); |
| 624 | ORE.emit(RemarkBuilder: [&]() { |
| 625 | OptimizationRemarkMissed R(DEBUG_TYPE, "IniningNotAttempted" , DLoc, Block); |
| 626 | reportContextForRemark(OR&: R); |
| 627 | return R; |
| 628 | }); |
| 629 | } |
| 630 | |