1//===- PlaceSafepoints.cpp - Place GC Safepoints --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Place garbage collection safepoints at appropriate locations in the IR. This
10// does not make relocation semantics or variable liveness explicit. That's
11// done by RewriteStatepointsForGC.
12//
13// Terminology:
14// - A call is said to be "parseable" if there is a stack map generated for the
15// return PC of the call. A runtime can determine where values listed in the
16// deopt arguments and (after RewriteStatepointsForGC) gc arguments are located
17// on the stack when the code is suspended inside such a call. Every parse
18// point is represented by a call wrapped in an gc.statepoint intrinsic.
19// - A "poll" is an explicit check in the generated code to determine if the
20// runtime needs the generated code to cooperate by calling a helper routine
21// and thus suspending its execution at a known state. The call to the helper
22// routine will be parseable. The (gc & runtime specific) logic of a poll is
23// assumed to be provided in a function of the name "gc.safepoint_poll".
24//
25// We aim to insert polls such that running code can quickly be brought to a
26// well defined state for inspection by the collector. In the current
27// implementation, this is done via the insertion of poll sites at method entry
28// and the backedge of most loops. We try to avoid inserting more polls than
29// are necessary to ensure a finite period between poll sites. This is not
30// because the poll itself is expensive in the generated code; it's not. Polls
31// do tend to impact the optimizer itself in negative ways; we'd like to avoid
32// perturbing the optimization of the method as much as we can.
33//
34// We also need to make most call sites parseable. The callee might execute a
35// poll (or otherwise be inspected by the GC). If so, the entire stack
36// (including the suspended frame of the current method) must be parseable.
37//
38// This pass will insert:
39// - Call parse points ("call safepoints") for any call which may need to
40// reach a safepoint during the execution of the callee function.
41// - Backedge safepoint polls and entry safepoint polls to ensure that
42// executing code reaches a safepoint poll in a finite amount of time.
43//
44// We do not currently support return statepoints, but adding them would not
45// be hard. They are not required for correctness - entry safepoints are an
46// alternative - but some GCs may prefer them. Patches welcome.
47//
48//===----------------------------------------------------------------------===//
49
50#include "llvm/Transforms/Scalar/PlaceSafepoints.h"
51#include "ScalarOptions.h"
52#include "llvm/InitializePasses.h"
53#include "llvm/Pass.h"
54
55#include "llvm/ADT/SetVector.h"
56#include "llvm/ADT/Statistic.h"
57#include "llvm/Analysis/CFG.h"
58#include "llvm/Analysis/LoopInfo.h"
59#include "llvm/Analysis/ScalarEvolution.h"
60#include "llvm/Analysis/TargetLibraryInfo.h"
61#include "llvm/IR/Dominators.h"
62#include "llvm/IR/IntrinsicInst.h"
63#include "llvm/IR/LegacyPassManager.h"
64#include "llvm/IR/Module.h"
65#include "llvm/IR/Statepoint.h"
66#include "llvm/Support/Debug.h"
67#include "llvm/Transforms/Scalar.h"
68#include "llvm/Transforms/Utils/BasicBlockUtils.h"
69#include "llvm/Transforms/Utils/Cloning.h"
70#include "llvm/Transforms/Utils/Local.h"
71
72using namespace llvm;
73
74#define DEBUG_TYPE "place-safepoints"
75
76STATISTIC(NumEntrySafepoints, "Number of entry safepoints inserted");
77STATISTIC(NumBackedgeSafepoints, "Number of backedge safepoints inserted");
78
79STATISTIC(CallInLoop,
80 "Number of loops without safepoints due to calls in loop");
81STATISTIC(FiniteExecution,
82 "Number of loops without safepoints finite execution");
83
84namespace {
85/// An analysis pass whose purpose is to identify each of the backedges in
86/// the function which require a safepoint poll to be inserted.
87class PlaceBackedgeSafepointsLegacyPass : public FunctionPass {
88public:
89 static char ID;
90
91 /// The output of the pass - gives a list of each backedge (described by
92 /// pointing at the branch) which need a poll inserted.
93 std::vector<Instruction *> PollLocations;
94
95 /// True unless we're running spp-no-calls in which case we need to disable
96 /// the call-dependent placement opts.
97 bool CallSafepointsEnabled;
98
99 PlaceBackedgeSafepointsLegacyPass(bool CallSafepoints = false)
100 : FunctionPass(ID), CallSafepointsEnabled(CallSafepoints),
101 Opts(ScalarOptions::Global) {
102 initializePlaceBackedgeSafepointsLegacyPassPass(
103 *PassRegistry::getPassRegistry());
104 }
105
106 bool runOnLoop(Loop *);
107
108 void runOnLoopAndSubLoops(Loop *L) {
109 // Visit all the subloops
110 for (Loop *I : *L)
111 runOnLoopAndSubLoops(L: I);
112 runOnLoop(L);
113 }
114
115 bool runOnFunction(Function &F) override {
116 SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
117 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
118 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
119 TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
120 for (Loop *I : *LI) {
121 runOnLoopAndSubLoops(L: I);
122 }
123 return false;
124 }
125
126 void getAnalysisUsage(AnalysisUsage &AU) const override {
127 AU.addRequired<DominatorTreeWrapperPass>();
128 AU.addRequired<ScalarEvolutionWrapperPass>();
129 AU.addRequired<LoopInfoWrapperPass>();
130 AU.addRequired<TargetLibraryInfoWrapperPass>();
131 // We no longer modify the IR at all in this pass. Thus all
132 // analysis are preserved.
133 AU.setPreservesAll();
134 }
135
136private:
137 const ScalarOptions &Opts;
138 ScalarEvolution *SE = nullptr;
139 DominatorTree *DT = nullptr;
140 LoopInfo *LI = nullptr;
141 TargetLibraryInfo *TLI = nullptr;
142};
143} // namespace
144
145char PlaceBackedgeSafepointsLegacyPass::ID = 0;
146
147INITIALIZE_PASS_BEGIN(PlaceBackedgeSafepointsLegacyPass,
148 "place-backedge-safepoints-impl",
149 "Place Backedge Safepoints", false, false)
150INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
151INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
152INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
153INITIALIZE_PASS_END(PlaceBackedgeSafepointsLegacyPass,
154 "place-backedge-safepoints-impl",
155 "Place Backedge Safepoints", false, false)
156
157static bool containsUnconditionalCallSafepoint(Loop *L, BasicBlock *Header,
158 BasicBlock *Pred,
159 DominatorTree &DT,
160 const TargetLibraryInfo &TLI);
161
162static bool mustBeFiniteCountedLoop(const ScalarOptions &Opts, Loop *L,
163 ScalarEvolution *SE, BasicBlock *Pred);
164
165static Instruction *findLocationForEntrySafepoint(Function &F,
166 DominatorTree &DT);
167
168static bool isGCSafepointPoll(Function &F);
169static bool shouldRewriteFunction(Function &F);
170static bool enableEntrySafepoints(const ScalarOptions &Opts, Function &F);
171static bool enableBackedgeSafepoints(const ScalarOptions &Opts, Function &F);
172static bool enableCallSafepoints(const ScalarOptions &Opts, Function &F);
173
174static void
175InsertSafepointPoll(BasicBlock::iterator InsertBefore,
176 std::vector<CallBase *> &ParsePointsNeeded /*rval*/,
177 const TargetLibraryInfo &TLI);
178
179bool PlaceBackedgeSafepointsLegacyPass::runOnLoop(Loop *L) {
180 // Loop through all loop latches (branches controlling backedges). We need
181 // to place a safepoint on every backedge (potentially).
182 // Note: In common usage, there will be only one edge due to LoopSimplify
183 // having run sometime earlier in the pipeline, but this code must be correct
184 // w.r.t. loops with multiple backedges.
185 BasicBlock *Header = L->getHeader();
186 SmallVector<BasicBlock *, 16> LoopLatches;
187 L->getLoopLatches(LoopLatches);
188 for (BasicBlock *Pred : LoopLatches) {
189 assert(L->contains(Pred));
190
191 // Make a policy decision about whether this loop needs a safepoint or
192 // not. Note that this is about unburdening the optimizer in loops, not
193 // avoiding the runtime cost of the actual safepoint.
194 if (!Opts.spp_all_backedges) {
195 if (mustBeFiniteCountedLoop(Opts, L, SE, Pred)) {
196 LLVM_DEBUG(dbgs() << "skipping safepoint placement in finite loop\n");
197 FiniteExecution++;
198 continue;
199 }
200 if (CallSafepointsEnabled &&
201 containsUnconditionalCallSafepoint(L, Header, Pred, DT&: *DT, TLI: *TLI)) {
202 // Note: This is only semantically legal since we won't do any further
203 // IPO or inlining before the actual call insertion.. If we hadn't, we
204 // might latter loose this call safepoint.
205 LLVM_DEBUG(
206 dbgs()
207 << "skipping safepoint placement due to unconditional call\n");
208 CallInLoop++;
209 continue;
210 }
211 }
212
213 // TODO: We can create an inner loop which runs a finite number of
214 // iterations with an outer loop which contains a safepoint. This would
215 // not help runtime performance that much, but it might help our ability to
216 // optimize the inner loop.
217
218 // Safepoint insertion would involve creating a new basic block (as the
219 // target of the current backedge) which does the safepoint (of all live
220 // variables) and branches to the true header
221 Instruction *Term = Pred->getTerminator();
222
223 LLVM_DEBUG(dbgs() << "[LSP] terminator instruction: " << *Term);
224
225 PollLocations.push_back(x: Term);
226 }
227
228 return false;
229}
230
231bool PlaceSafepointsPass::runImpl(Function &F, const TargetLibraryInfo &TLI) {
232 if (F.isDeclaration() || F.empty()) {
233 // This is a declaration, nothing to do. Must exit early to avoid crash in
234 // dom tree calculation
235 return false;
236 }
237
238 if (isGCSafepointPoll(F)) {
239 // Given we're inlining this inside of safepoint poll insertion, this
240 // doesn't make any sense. Note that we do make any contained calls
241 // parseable after we inline a poll.
242 return false;
243 }
244
245 if (!shouldRewriteFunction(F))
246 return false;
247
248 const ScalarOptions &Opts = ScalarOptions::Global;
249 bool Modified = false;
250
251 // In various bits below, we rely on the fact that uses are reachable from
252 // defs. When there are basic blocks unreachable from the entry, dominance
253 // and reachablity queries return non-sensical results. Thus, we preprocess
254 // the function to ensure these properties hold.
255 Modified |= removeUnreachableBlocks(F);
256
257 // STEP 1 - Insert the safepoint polling locations. We do not need to
258 // actually insert parse points yet. That will be done for all polls and
259 // calls in a single pass.
260
261 DominatorTree DT;
262 DT.recalculate(Func&: F);
263
264 SmallVector<Instruction *, 16> PollsNeeded;
265 std::vector<CallBase *> ParsePointNeeded;
266
267 if (enableBackedgeSafepoints(Opts, F)) {
268 // Construct a pass manager to run the LoopPass backedge logic. We
269 // need the pass manager to handle scheduling all the loop passes
270 // appropriately. Doing this by hand is painful and just not worth messing
271 // with for the moment.
272 legacy::FunctionPassManager FPM(F.getParent());
273 bool CanAssumeCallSafepoints = enableCallSafepoints(Opts, F);
274
275 FPM.add(P: new TargetLibraryInfoWrapperPass(TLI));
276 auto *PBS = new PlaceBackedgeSafepointsLegacyPass(CanAssumeCallSafepoints);
277 FPM.add(P: PBS);
278 FPM.run(F);
279
280 // We preserve dominance information when inserting the poll, otherwise
281 // we'd have to recalculate this on every insert
282 DT.recalculate(Func&: F);
283
284 auto &PollLocations = PBS->PollLocations;
285
286 auto OrderByBBName = [](Instruction *a, Instruction *b) {
287 return a->getParent()->getName() < b->getParent()->getName();
288 };
289 // We need the order of list to be stable so that naming ends up stable
290 // when we split edges. This makes test cases much easier to write.
291 llvm::sort(C&: PollLocations, Comp: OrderByBBName);
292
293 // We can sometimes end up with duplicate poll locations. This happens if
294 // a single loop is visited more than once. The fact this happens seems
295 // wrong, but it does happen for the split-backedge.ll test case.
296 PollLocations.erase(first: llvm::unique(R&: PollLocations), last: PollLocations.end());
297
298 // Insert a poll at each point the analysis pass identified
299 // The poll location must be the terminator of a loop latch block.
300 for (Instruction *Term : PollLocations) {
301 // We are inserting a poll, the function is modified
302 Modified = true;
303
304 if (Opts.spp_split_backedge) {
305 // Split the backedge of the loop and insert the poll within that new
306 // basic block. This creates a loop with two latches per original
307 // latch (which is non-ideal), but this appears to be easier to
308 // optimize in practice than inserting the poll immediately before the
309 // latch test.
310
311 // Since this is a latch, at least one of the successors must dominate
312 // it. Its possible that we have a) duplicate edges to the same header
313 // and b) edges to distinct loop headers. We need to insert pools on
314 // each.
315 SetVector<BasicBlock *> Headers;
316 for (BasicBlock *Succ : successors(BB: Term->getParent()))
317 if (DT.dominates(A: Succ, B: Term->getParent()))
318 Headers.insert(X: Succ);
319 assert(!Headers.empty() && "poll location is not a loop latch?");
320
321 // The split loop structure here is so that we only need to recalculate
322 // the dominator tree once. Alternatively, we could just keep it up to
323 // date and use a more natural merged loop.
324 for (BasicBlock *Header : Headers) {
325 BasicBlock *NewBB = SplitEdge(From: Term->getParent(), To: Header, DT: &DT);
326 PollsNeeded.push_back(Elt: NewBB->getTerminator());
327 NumBackedgeSafepoints++;
328 }
329 } else {
330 // Split the latch block itself, right before the terminator.
331 PollsNeeded.push_back(Elt: Term);
332 NumBackedgeSafepoints++;
333 }
334 }
335 }
336
337 if (enableEntrySafepoints(Opts, F)) {
338 if (Instruction *Location = findLocationForEntrySafepoint(F, DT)) {
339 PollsNeeded.push_back(Elt: Location);
340 Modified = true;
341 NumEntrySafepoints++;
342 }
343 // TODO: else we should assert that there was, in fact, a policy choice to
344 // not insert a entry safepoint poll.
345 }
346
347 // Now that we've identified all the needed safepoint poll locations, insert
348 // safepoint polls themselves.
349 for (Instruction *PollLocation : PollsNeeded) {
350 std::vector<CallBase *> RuntimeCalls;
351 InsertSafepointPoll(InsertBefore: PollLocation->getIterator(), ParsePointsNeeded&: RuntimeCalls, TLI);
352 llvm::append_range(C&: ParsePointNeeded, R&: RuntimeCalls);
353 }
354
355 return Modified;
356}
357
358PreservedAnalyses PlaceSafepointsPass::run(Function &F,
359 FunctionAnalysisManager &AM) {
360 auto &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
361
362 if (!runImpl(F, TLI))
363 return PreservedAnalyses::all();
364
365 // TODO: can we preserve more?
366 return PreservedAnalyses::none();
367}
368
369static bool needsStatepoint(CallBase *Call, const TargetLibraryInfo &TLI) {
370 if (callsGCLeafFunction(Call, TLI))
371 return false;
372 if (auto *CI = dyn_cast<CallInst>(Val: Call)) {
373 if (CI->isInlineAsm())
374 return false;
375 }
376
377 return !(isa<GCStatepointInst>(Val: Call) || isa<GCRelocateInst>(Val: Call) ||
378 isa<GCResultInst>(Val: Call));
379}
380
381/// Returns true if this loop is known to contain a call safepoint which
382/// must unconditionally execute on any iteration of the loop which returns
383/// to the loop header via an edge from Pred. Returns a conservative correct
384/// answer; i.e. false is always valid.
385static bool containsUnconditionalCallSafepoint(Loop *L, BasicBlock *Header,
386 BasicBlock *Pred,
387 DominatorTree &DT,
388 const TargetLibraryInfo &TLI) {
389 // In general, we're looking for any cut of the graph which ensures
390 // there's a call safepoint along every edge between Header and Pred.
391 // For the moment, we look only for the 'cuts' that consist of a single call
392 // instruction in a block which is dominated by the Header and dominates the
393 // loop latch (Pred) block. Somewhat surprisingly, walking the entire chain
394 // of such dominating blocks gets substantially more occurrences than just
395 // checking the Pred and Header blocks themselves. This may be due to the
396 // density of loop exit conditions caused by range and null checks.
397 // TODO: structure this as an analysis pass, cache the result for subloops,
398 // avoid dom tree recalculations
399 assert(DT.dominates(Header, Pred) && "loop latch not dominated by header?");
400
401 BasicBlock *Current = Pred;
402 while (true) {
403 for (Instruction &I : *Current) {
404 if (auto *Call = dyn_cast<CallBase>(Val: &I))
405 // Note: Technically, needing a safepoint isn't quite the right
406 // condition here. We should instead be checking if the target method
407 // has an
408 // unconditional poll. In practice, this is only a theoretical concern
409 // since we don't have any methods with conditional-only safepoint
410 // polls.
411 if (needsStatepoint(Call, TLI))
412 return true;
413 }
414
415 if (Current == Header)
416 break;
417 Current = DT.getNode(BB: Current)->getIDom()->getBlock();
418 }
419
420 return false;
421}
422
423/// Returns true if this loop is known to terminate in a finite number of
424/// iterations. Note that this function may return false for a loop which
425/// does actual terminate in a finite constant number of iterations due to
426/// conservatism in the analysis.
427static bool mustBeFiniteCountedLoop(const ScalarOptions &Opts, Loop *L,
428 ScalarEvolution *SE, BasicBlock *Pred) {
429 // A conservative bound on the loop as a whole.
430 const SCEV *MaxTrips = SE->getConstantMaxBackedgeTakenCount(L);
431 if (!isa<SCEVCouldNotCompute>(Val: MaxTrips) &&
432 SE->getUnsignedRange(S: MaxTrips).getUnsignedMax().isIntN(
433 N: Opts.spp_counted_loop_trip_width))
434 return true;
435
436 // If this is a conditional branch to the header with the alternate path
437 // being outside the loop, we can ask questions about the execution frequency
438 // of the exit block.
439 if (L->isLoopExiting(BB: Pred)) {
440 // This returns an exact expression only. TODO: We really only need an
441 // upper bound here, but SE doesn't expose that.
442 const SCEV *MaxExec = SE->getExitCount(L, ExitingBlock: Pred);
443 if (!isa<SCEVCouldNotCompute>(Val: MaxExec) &&
444 SE->getUnsignedRange(S: MaxExec).getUnsignedMax().isIntN(
445 N: Opts.spp_counted_loop_trip_width))
446 return true;
447 }
448
449 return /* not finite */ false;
450}
451
452static void scanOneBB(Instruction *Start, Instruction *End,
453 std::vector<CallInst *> &Calls,
454 DenseSet<BasicBlock *> &Seen,
455 std::vector<BasicBlock *> &Worklist) {
456 for (BasicBlock::iterator BBI(Start), BBE0 = Start->getParent()->end(),
457 BBE1 = BasicBlock::iterator(End);
458 BBI != BBE0 && BBI != BBE1; BBI++) {
459 if (CallInst *CI = dyn_cast<CallInst>(Val: &*BBI))
460 Calls.push_back(x: CI);
461
462 // FIXME: This code does not handle invokes
463 assert(!isa<InvokeInst>(&*BBI) &&
464 "support for invokes in poll code needed");
465
466 // Only add the successor blocks if we reach the terminator instruction
467 // without encountering end first
468 if (BBI->isTerminator()) {
469 BasicBlock *BB = BBI->getParent();
470 for (BasicBlock *Succ : successors(BB)) {
471 if (Seen.insert(V: Succ).second) {
472 Worklist.push_back(x: Succ);
473 }
474 }
475 }
476 }
477}
478
479static void scanInlinedCode(Instruction *Start, Instruction *End,
480 std::vector<CallInst *> &Calls,
481 DenseSet<BasicBlock *> &Seen) {
482 Calls.clear();
483 std::vector<BasicBlock *> Worklist;
484 Seen.insert(V: Start->getParent());
485 scanOneBB(Start, End, Calls, Seen, Worklist);
486 while (!Worklist.empty()) {
487 BasicBlock *BB = Worklist.back();
488 Worklist.pop_back();
489 scanOneBB(Start: &*BB->begin(), End, Calls, Seen, Worklist);
490 }
491}
492
493/// Returns true if an entry safepoint is not required before this callsite in
494/// the caller function.
495static bool doesNotRequireEntrySafepointBefore(CallBase *Call) {
496 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Call)) {
497 switch (II->getIntrinsicID()) {
498 case Intrinsic::experimental_gc_statepoint:
499 case Intrinsic::experimental_patchpoint_void:
500 case Intrinsic::experimental_patchpoint:
501 // The can wrap an actual call which may grow the stack by an unbounded
502 // amount or run forever.
503 return false;
504 default:
505 // Most LLVM intrinsics are things which do not expand to actual calls, or
506 // at least if they do, are leaf functions that cause only finite stack
507 // growth. In particular, the optimizer likes to form things like memsets
508 // out of stores in the original IR. Another important example is
509 // llvm.localescape which must occur in the entry block. Inserting a
510 // safepoint before it is not legal since it could push the localescape
511 // out of the entry block.
512 return true;
513 }
514 }
515 return false;
516}
517
518static Instruction *findLocationForEntrySafepoint(Function &F,
519 DominatorTree &DT) {
520
521 // Conceptually, this poll needs to be on method entry, but in
522 // practice, we place it as late in the entry block as possible. We
523 // can place it as late as we want as long as it dominates all calls
524 // that can grow the stack. This, combined with backedge polls,
525 // give us all the progress guarantees we need.
526
527 // hasNextInstruction and nextInstruction are used to iterate
528 // through a "straight line" execution sequence.
529
530 auto HasNextInstruction = [](Instruction *I) {
531 if (!I->isTerminator())
532 return true;
533
534 BasicBlock *nextBB = I->getParent()->getUniqueSuccessor();
535 return nextBB && (nextBB->getUniquePredecessor() != nullptr);
536 };
537
538 auto NextInstruction = [&](Instruction *I) {
539 assert(HasNextInstruction(I) &&
540 "first check if there is a next instruction!");
541
542 if (I->isTerminator())
543 return &I->getParent()->getUniqueSuccessor()->front();
544 return &*++I->getIterator();
545 };
546
547 Instruction *Cursor = nullptr;
548 for (Cursor = &F.getEntryBlock().front(); HasNextInstruction(Cursor);
549 Cursor = NextInstruction(Cursor)) {
550
551 // We need to ensure a safepoint poll occurs before any 'real' call. The
552 // easiest way to ensure finite execution between safepoints in the face of
553 // recursive and mutually recursive functions is to enforce that each take
554 // a safepoint. Additionally, we need to ensure a poll before any call
555 // which can grow the stack by an unbounded amount. This isn't required
556 // for GC semantics per se, but is a common requirement for languages
557 // which detect stack overflow via guard pages and then throw exceptions.
558 if (auto *Call = dyn_cast<CallBase>(Val: Cursor)) {
559 if (doesNotRequireEntrySafepointBefore(Call))
560 continue;
561 break;
562 }
563 }
564
565 assert((HasNextInstruction(Cursor) || Cursor->isTerminator()) &&
566 "either we stopped because of a call, or because of terminator");
567
568 return Cursor;
569}
570
571const char GCSafepointPollName[] = "gc.safepoint_poll";
572
573static bool isGCSafepointPoll(Function &F) {
574 return F.getName() == GCSafepointPollName;
575}
576
577/// Returns true if this function should be rewritten to include safepoint
578/// polls and parseable call sites. The main point of this function is to be
579/// an extension point for custom logic.
580static bool shouldRewriteFunction(Function &F) {
581 // TODO: This should check the GCStrategy
582 if (F.hasGC()) {
583 const auto &FunctionGCName = F.getGC();
584 const StringRef StatepointExampleName("statepoint-example");
585 const StringRef CoreCLRName("coreclr");
586 return (StatepointExampleName == FunctionGCName) ||
587 (CoreCLRName == FunctionGCName);
588 } else
589 return false;
590}
591
592// TODO: These should become properties of the GCStrategy, possibly with
593// command line overrides.
594static bool enableEntrySafepoints(const ScalarOptions &Opts, Function &F) {
595 return !Opts.spp_no_entry;
596}
597static bool enableBackedgeSafepoints(const ScalarOptions &Opts, Function &F) {
598 return !Opts.spp_no_backedge;
599}
600static bool enableCallSafepoints(const ScalarOptions &Opts, Function &F) {
601 return !Opts.spp_no_call;
602}
603
604// Insert a safepoint poll immediately before the given instruction. Does
605// not handle the parsability of state at the runtime call, that's the
606// callers job.
607static void
608InsertSafepointPoll(BasicBlock::iterator InsertBefore,
609 std::vector<CallBase *> &ParsePointsNeeded /*rval*/,
610 const TargetLibraryInfo &TLI) {
611 BasicBlock *OrigBB = InsertBefore->getParent();
612 Module *M = InsertBefore->getModule();
613 assert(M && "must be part of a module");
614
615 // Inline the safepoint poll implementation - this will get all the branch,
616 // control flow, etc.. Most importantly, it will introduce the actual slow
617 // path call - where we need to insert a safepoint (parsepoint).
618
619 auto *F = M->getFunction(Name: GCSafepointPollName);
620 assert(F && "gc.safepoint_poll function is missing");
621 assert(F->getFunctionType() ==
622 FunctionType::get(Type::getVoidTy(M->getContext()), false) &&
623 "gc.safepoint_poll declared with wrong type");
624 assert(!F->empty() && "gc.safepoint_poll must be a non-empty function");
625 CallInst *PollCall = CallInst::Create(Func: F, NameStr: "", InsertBefore);
626
627 // Record some information about the call site we're replacing
628 BasicBlock::iterator Before(PollCall), After(PollCall);
629 bool IsBegin = false;
630 if (Before == OrigBB->begin())
631 IsBegin = true;
632 else
633 Before--;
634
635 After++;
636 assert(After != OrigBB->end() && "must have successor");
637
638 // Do the actual inlining
639 InlineFunctionInfo IFI;
640 bool InlineStatus = InlineFunction(CB&: *PollCall, IFI).isSuccess();
641 assert(InlineStatus && "inline must succeed");
642 (void)InlineStatus; // suppress warning in release-asserts
643
644 // Check post-conditions
645 assert(IFI.StaticAllocas.empty() && "can't have allocs");
646
647 std::vector<CallInst *> Calls; // new calls
648 DenseSet<BasicBlock *> BBs; // new BBs + insertee
649
650 // Include only the newly inserted instructions, Note: begin may not be valid
651 // if we inserted to the beginning of the basic block
652 BasicBlock::iterator Start = IsBegin ? OrigBB->begin() : std::next(x: Before);
653
654 // If your poll function includes an unreachable at the end, that's not
655 // valid. Fuzzers/test case reducers can create this, so check for it.
656 assert(isPotentiallyReachable(&*Start, &*After) &&
657 "malformed poll function");
658
659 scanInlinedCode(Start: &*Start, End: &*After, Calls, Seen&: BBs);
660 assert(!Calls.empty() && "slow path not found for safepoint poll");
661
662 // Record the fact we need a parsable state at the runtime call contained in
663 // the poll function. This is required so that the runtime knows how to
664 // parse the last frame when we actually take the safepoint (i.e. execute
665 // the slow path)
666 assert(ParsePointsNeeded.empty());
667 for (auto *CI : Calls) {
668 // No safepoint needed or wanted
669 if (!needsStatepoint(Call: CI, TLI))
670 continue;
671
672 // These are likely runtime calls. Should we assert that via calling
673 // convention or something?
674 ParsePointsNeeded.push_back(x: CI);
675 }
676 assert(ParsePointsNeeded.size() <= Calls.size());
677}
678