1//===- PlaceSafepoints.cpp - Place GC Safepoints --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Place garbage collection safepoints at appropriate locations in the IR. This
10// does not make relocation semantics or variable liveness explicit. That's
11// done by RewriteStatepointsForGC.
12//
13// Terminology:
14// - A call is said to be "parseable" if there is a stack map generated for the
15// return PC of the call. A runtime can determine where values listed in the
16// deopt arguments and (after RewriteStatepointsForGC) gc arguments are located
17// on the stack when the code is suspended inside such a call. Every parse
18// point is represented by a call wrapped in an gc.statepoint intrinsic.
19// - A "poll" is an explicit check in the generated code to determine if the
20// runtime needs the generated code to cooperate by calling a helper routine
21// and thus suspending its execution at a known state. The call to the helper
22// routine will be parseable. The (gc & runtime specific) logic of a poll is
23// assumed to be provided in a function of the name "gc.safepoint_poll".
24//
25// We aim to insert polls such that running code can quickly be brought to a
26// well defined state for inspection by the collector. In the current
27// implementation, this is done via the insertion of poll sites at method entry
28// and the backedge of most loops. We try to avoid inserting more polls than
29// are necessary to ensure a finite period between poll sites. This is not
30// because the poll itself is expensive in the generated code; it's not. Polls
31// do tend to impact the optimizer itself in negative ways; we'd like to avoid
32// perturbing the optimization of the method as much as we can.
33//
34// We also need to make most call sites parseable. The callee might execute a
35// poll (or otherwise be inspected by the GC). If so, the entire stack
36// (including the suspended frame of the current method) must be parseable.
37//
38// This pass will insert:
39// - Call parse points ("call safepoints") for any call which may need to
40// reach a safepoint during the execution of the callee function.
41// - Backedge safepoint polls and entry safepoint polls to ensure that
42// executing code reaches a safepoint poll in a finite amount of time.
43//
44// We do not currently support return statepoints, but adding them would not
45// be hard. They are not required for correctness - entry safepoints are an
46// alternative - but some GCs may prefer them. Patches welcome.
47//
48//===----------------------------------------------------------------------===//
49
50#include "llvm/Transforms/Scalar/PlaceSafepoints.h"
51#include "ScalarOptions.h"
52#include "llvm/InitializePasses.h"
53#include "llvm/Pass.h"
54
55#include "llvm/ADT/SetVector.h"
56#include "llvm/ADT/Statistic.h"
57#include "llvm/Analysis/CFG.h"
58#include "llvm/Analysis/LoopInfo.h"
59#include "llvm/Analysis/ScalarEvolution.h"
60#include "llvm/Analysis/TargetLibraryInfo.h"
61#include "llvm/IR/Dominators.h"
62#include "llvm/IR/IntrinsicInst.h"
63#include "llvm/IR/LegacyPassManager.h"
64#include "llvm/IR/Module.h"
65#include "llvm/IR/Statepoint.h"
66#include "llvm/Support/Debug.h"
67#include "llvm/Transforms/Scalar.h"
68#include "llvm/Transforms/Utils/BasicBlockUtils.h"
69#include "llvm/Transforms/Utils/Cloning.h"
70#include "llvm/Transforms/Utils/Local.h"
71
72using namespace llvm;
73
74#define DEBUG_TYPE "place-safepoints"
75
76STATISTIC(NumEntrySafepoints, "Number of entry safepoints inserted");
77STATISTIC(NumBackedgeSafepoints, "Number of backedge safepoints inserted");
78
79STATISTIC(CallInLoop,
80 "Number of loops without safepoints due to calls in loop");
81STATISTIC(FiniteExecution,
82 "Number of loops without safepoints finite execution");
83
84namespace {
85/// An analysis pass whose purpose is to identify each of the backedges in
86/// the function which require a safepoint poll to be inserted.
87class PlaceBackedgeSafepointsLegacyPass : public FunctionPass {
88public:
89 static char ID;
90
91 /// The output of the pass - gives a list of each backedge (described by
92 /// pointing at the branch) which need a poll inserted.
93 std::vector<Instruction *> PollLocations;
94
95 /// True unless we're running spp-no-calls in which case we need to disable
96 /// the call-dependent placement opts.
97 bool CallSafepointsEnabled;
98
99 PlaceBackedgeSafepointsLegacyPass(bool CallSafepoints = false)
100 : FunctionPass(ID), CallSafepointsEnabled(CallSafepoints) {
101 initializePlaceBackedgeSafepointsLegacyPassPass(
102 *PassRegistry::getPassRegistry());
103 }
104
105 bool runOnLoop(Loop *);
106
107 void runOnLoopAndSubLoops(Loop *L) {
108 // Visit all the subloops
109 for (Loop *I : *L)
110 runOnLoopAndSubLoops(L: I);
111 runOnLoop(L);
112 }
113
114 bool runOnFunction(Function &F) override {
115 Opts = &F.getContext().getLibraryOptions().get<ScalarOptions>();
116 SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
117 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
118 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
119 TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
120 for (Loop *I : *LI) {
121 runOnLoopAndSubLoops(L: I);
122 }
123 return false;
124 }
125
126 void getAnalysisUsage(AnalysisUsage &AU) const override {
127 AU.addRequired<DominatorTreeWrapperPass>();
128 AU.addRequired<ScalarEvolutionWrapperPass>();
129 AU.addRequired<LoopInfoWrapperPass>();
130 AU.addRequired<TargetLibraryInfoWrapperPass>();
131 // We no longer modify the IR at all in this pass. Thus all
132 // analysis are preserved.
133 AU.setPreservesAll();
134 }
135
136private:
137 const ScalarOptions *Opts = nullptr;
138 ScalarEvolution *SE = nullptr;
139 DominatorTree *DT = nullptr;
140 LoopInfo *LI = nullptr;
141 TargetLibraryInfo *TLI = nullptr;
142};
143} // namespace
144
145char PlaceBackedgeSafepointsLegacyPass::ID = 0;
146
147INITIALIZE_PASS_BEGIN(PlaceBackedgeSafepointsLegacyPass,
148 "place-backedge-safepoints-impl",
149 "Place Backedge Safepoints", false, false)
150INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
151INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
152INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
153INITIALIZE_PASS_END(PlaceBackedgeSafepointsLegacyPass,
154 "place-backedge-safepoints-impl",
155 "Place Backedge Safepoints", false, false)
156
157static bool containsUnconditionalCallSafepoint(Loop *L, BasicBlock *Header,
158 BasicBlock *Pred,
159 DominatorTree &DT,
160 const TargetLibraryInfo &TLI);
161
162static bool mustBeFiniteCountedLoop(const ScalarOptions &Opts, Loop *L,
163 ScalarEvolution *SE, BasicBlock *Pred);
164
165static Instruction *findLocationForEntrySafepoint(Function &F,
166 DominatorTree &DT);
167
168static bool isGCSafepointPoll(Function &F);
169static bool shouldRewriteFunction(Function &F);
170static bool enableEntrySafepoints(const ScalarOptions &Opts, Function &F);
171static bool enableBackedgeSafepoints(const ScalarOptions &Opts, Function &F);
172static bool enableCallSafepoints(const ScalarOptions &Opts, Function &F);
173
174static void
175InsertSafepointPoll(BasicBlock::iterator InsertBefore,
176 std::vector<CallBase *> &ParsePointsNeeded /*rval*/,
177 const TargetLibraryInfo &TLI);
178
179bool PlaceBackedgeSafepointsLegacyPass::runOnLoop(Loop *L) {
180 // Loop through all loop latches (branches controlling backedges). We need
181 // to place a safepoint on every backedge (potentially).
182 // Note: In common usage, there will be only one edge due to LoopSimplify
183 // having run sometime earlier in the pipeline, but this code must be correct
184 // w.r.t. loops with multiple backedges.
185 BasicBlock *Header = L->getHeader();
186 SmallVector<BasicBlock *, 16> LoopLatches;
187 L->getLoopLatches(LoopLatches);
188 for (BasicBlock *Pred : LoopLatches) {
189 assert(L->contains(Pred));
190
191 // Make a policy decision about whether this loop needs a safepoint or
192 // not. Note that this is about unburdening the optimizer in loops, not
193 // avoiding the runtime cost of the actual safepoint.
194 if (!Opts->spp_all_backedges) {
195 if (mustBeFiniteCountedLoop(Opts: *Opts, L, SE, Pred)) {
196 LLVM_DEBUG(dbgs() << "skipping safepoint placement in finite loop\n");
197 FiniteExecution++;
198 continue;
199 }
200 if (CallSafepointsEnabled &&
201 containsUnconditionalCallSafepoint(L, Header, Pred, DT&: *DT, TLI: *TLI)) {
202 // Note: This is only semantically legal since we won't do any further
203 // IPO or inlining before the actual call insertion.. If we hadn't, we
204 // might latter loose this call safepoint.
205 LLVM_DEBUG(
206 dbgs()
207 << "skipping safepoint placement due to unconditional call\n");
208 CallInLoop++;
209 continue;
210 }
211 }
212
213 // TODO: We can create an inner loop which runs a finite number of
214 // iterations with an outer loop which contains a safepoint. This would
215 // not help runtime performance that much, but it might help our ability to
216 // optimize the inner loop.
217
218 // Safepoint insertion would involve creating a new basic block (as the
219 // target of the current backedge) which does the safepoint (of all live
220 // variables) and branches to the true header
221 Instruction *Term = Pred->getTerminator();
222
223 LLVM_DEBUG(dbgs() << "[LSP] terminator instruction: " << *Term);
224
225 PollLocations.push_back(x: Term);
226 }
227
228 return false;
229}
230
231bool PlaceSafepointsPass::runImpl(Function &F, const TargetLibraryInfo &TLI) {
232 if (F.isDeclaration() || F.empty()) {
233 // This is a declaration, nothing to do. Must exit early to avoid crash in
234 // dom tree calculation
235 return false;
236 }
237
238 if (isGCSafepointPoll(F)) {
239 // Given we're inlining this inside of safepoint poll insertion, this
240 // doesn't make any sense. Note that we do make any contained calls
241 // parseable after we inline a poll.
242 return false;
243 }
244
245 if (!shouldRewriteFunction(F))
246 return false;
247
248 const ScalarOptions &Opts =
249 F.getContext().getLibraryOptions().get<ScalarOptions>();
250 bool Modified = false;
251
252 // In various bits below, we rely on the fact that uses are reachable from
253 // defs. When there are basic blocks unreachable from the entry, dominance
254 // and reachablity queries return non-sensical results. Thus, we preprocess
255 // the function to ensure these properties hold.
256 Modified |= removeUnreachableBlocks(F);
257
258 // STEP 1 - Insert the safepoint polling locations. We do not need to
259 // actually insert parse points yet. That will be done for all polls and
260 // calls in a single pass.
261
262 DominatorTree DT;
263 DT.recalculate(Func&: F);
264
265 SmallVector<Instruction *, 16> PollsNeeded;
266 std::vector<CallBase *> ParsePointNeeded;
267
268 if (enableBackedgeSafepoints(Opts, F)) {
269 // Construct a pass manager to run the LoopPass backedge logic. We
270 // need the pass manager to handle scheduling all the loop passes
271 // appropriately. Doing this by hand is painful and just not worth messing
272 // with for the moment.
273 legacy::FunctionPassManager FPM(F.getParent());
274 bool CanAssumeCallSafepoints = enableCallSafepoints(Opts, F);
275
276 FPM.add(P: new TargetLibraryInfoWrapperPass(TLI));
277 auto *PBS = new PlaceBackedgeSafepointsLegacyPass(CanAssumeCallSafepoints);
278 FPM.add(P: PBS);
279 FPM.run(F);
280
281 // We preserve dominance information when inserting the poll, otherwise
282 // we'd have to recalculate this on every insert
283 DT.recalculate(Func&: F);
284
285 auto &PollLocations = PBS->PollLocations;
286
287 auto OrderByBBName = [](Instruction *a, Instruction *b) {
288 return a->getParent()->getName() < b->getParent()->getName();
289 };
290 // We need the order of list to be stable so that naming ends up stable
291 // when we split edges. This makes test cases much easier to write.
292 llvm::sort(C&: PollLocations, Comp: OrderByBBName);
293
294 // We can sometimes end up with duplicate poll locations. This happens if
295 // a single loop is visited more than once. The fact this happens seems
296 // wrong, but it does happen for the split-backedge.ll test case.
297 PollLocations.erase(first: llvm::unique(R&: PollLocations), last: PollLocations.end());
298
299 // Insert a poll at each point the analysis pass identified
300 // The poll location must be the terminator of a loop latch block.
301 for (Instruction *Term : PollLocations) {
302 // We are inserting a poll, the function is modified
303 Modified = true;
304
305 if (Opts.spp_split_backedge) {
306 // Split the backedge of the loop and insert the poll within that new
307 // basic block. This creates a loop with two latches per original
308 // latch (which is non-ideal), but this appears to be easier to
309 // optimize in practice than inserting the poll immediately before the
310 // latch test.
311
312 // Since this is a latch, at least one of the successors must dominate
313 // it. Its possible that we have a) duplicate edges to the same header
314 // and b) edges to distinct loop headers. We need to insert pools on
315 // each.
316 SetVector<BasicBlock *> Headers;
317 for (BasicBlock *Succ : successors(BB: Term->getParent()))
318 if (DT.dominates(A: Succ, B: Term->getParent()))
319 Headers.insert(X: Succ);
320 assert(!Headers.empty() && "poll location is not a loop latch?");
321
322 // The split loop structure here is so that we only need to recalculate
323 // the dominator tree once. Alternatively, we could just keep it up to
324 // date and use a more natural merged loop.
325 for (BasicBlock *Header : Headers) {
326 BasicBlock *NewBB = SplitEdge(From: Term->getParent(), To: Header, DT: &DT);
327 PollsNeeded.push_back(Elt: NewBB->getTerminator());
328 NumBackedgeSafepoints++;
329 }
330 } else {
331 // Split the latch block itself, right before the terminator.
332 PollsNeeded.push_back(Elt: Term);
333 NumBackedgeSafepoints++;
334 }
335 }
336 }
337
338 if (enableEntrySafepoints(Opts, F)) {
339 if (Instruction *Location = findLocationForEntrySafepoint(F, DT)) {
340 PollsNeeded.push_back(Elt: Location);
341 Modified = true;
342 NumEntrySafepoints++;
343 }
344 // TODO: else we should assert that there was, in fact, a policy choice to
345 // not insert a entry safepoint poll.
346 }
347
348 // Now that we've identified all the needed safepoint poll locations, insert
349 // safepoint polls themselves.
350 for (Instruction *PollLocation : PollsNeeded) {
351 std::vector<CallBase *> RuntimeCalls;
352 InsertSafepointPoll(InsertBefore: PollLocation->getIterator(), ParsePointsNeeded&: RuntimeCalls, TLI);
353 llvm::append_range(C&: ParsePointNeeded, R&: RuntimeCalls);
354 }
355
356 return Modified;
357}
358
359PreservedAnalyses PlaceSafepointsPass::run(Function &F,
360 FunctionAnalysisManager &AM) {
361 auto &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
362
363 if (!runImpl(F, TLI))
364 return PreservedAnalyses::all();
365
366 // TODO: can we preserve more?
367 return PreservedAnalyses::none();
368}
369
370static bool needsStatepoint(CallBase *Call, const TargetLibraryInfo &TLI) {
371 if (callsGCLeafFunction(Call, TLI))
372 return false;
373 if (auto *CI = dyn_cast<CallInst>(Val: Call)) {
374 if (CI->isInlineAsm())
375 return false;
376 }
377
378 return !(isa<GCStatepointInst>(Val: Call) || isa<GCRelocateInst>(Val: Call) ||
379 isa<GCResultInst>(Val: Call));
380}
381
382/// Returns true if this loop is known to contain a call safepoint which
383/// must unconditionally execute on any iteration of the loop which returns
384/// to the loop header via an edge from Pred. Returns a conservative correct
385/// answer; i.e. false is always valid.
386static bool containsUnconditionalCallSafepoint(Loop *L, BasicBlock *Header,
387 BasicBlock *Pred,
388 DominatorTree &DT,
389 const TargetLibraryInfo &TLI) {
390 // In general, we're looking for any cut of the graph which ensures
391 // there's a call safepoint along every edge between Header and Pred.
392 // For the moment, we look only for the 'cuts' that consist of a single call
393 // instruction in a block which is dominated by the Header and dominates the
394 // loop latch (Pred) block. Somewhat surprisingly, walking the entire chain
395 // of such dominating blocks gets substantially more occurrences than just
396 // checking the Pred and Header blocks themselves. This may be due to the
397 // density of loop exit conditions caused by range and null checks.
398 // TODO: structure this as an analysis pass, cache the result for subloops,
399 // avoid dom tree recalculations
400 assert(DT.dominates(Header, Pred) && "loop latch not dominated by header?");
401
402 BasicBlock *Current = Pred;
403 while (true) {
404 for (Instruction &I : *Current) {
405 if (auto *Call = dyn_cast<CallBase>(Val: &I))
406 // Note: Technically, needing a safepoint isn't quite the right
407 // condition here. We should instead be checking if the target method
408 // has an
409 // unconditional poll. In practice, this is only a theoretical concern
410 // since we don't have any methods with conditional-only safepoint
411 // polls.
412 if (needsStatepoint(Call, TLI))
413 return true;
414 }
415
416 if (Current == Header)
417 break;
418 Current = DT.getNode(BB: Current)->getIDom()->getBlock();
419 }
420
421 return false;
422}
423
424/// Returns true if this loop is known to terminate in a finite number of
425/// iterations. Note that this function may return false for a loop which
426/// does actual terminate in a finite constant number of iterations due to
427/// conservatism in the analysis.
428static bool mustBeFiniteCountedLoop(const ScalarOptions &Opts, Loop *L,
429 ScalarEvolution *SE, BasicBlock *Pred) {
430 // A conservative bound on the loop as a whole.
431 const SCEV *MaxTrips = SE->getConstantMaxBackedgeTakenCount(L);
432 if (!isa<SCEVCouldNotCompute>(Val: MaxTrips) &&
433 SE->getUnsignedRange(S: MaxTrips).getUnsignedMax().isIntN(
434 N: Opts.spp_counted_loop_trip_width))
435 return true;
436
437 // If this is a conditional branch to the header with the alternate path
438 // being outside the loop, we can ask questions about the execution frequency
439 // of the exit block.
440 if (L->isLoopExiting(BB: Pred)) {
441 // This returns an exact expression only. TODO: We really only need an
442 // upper bound here, but SE doesn't expose that.
443 const SCEV *MaxExec = SE->getExitCount(L, ExitingBlock: Pred);
444 if (!isa<SCEVCouldNotCompute>(Val: MaxExec) &&
445 SE->getUnsignedRange(S: MaxExec).getUnsignedMax().isIntN(
446 N: Opts.spp_counted_loop_trip_width))
447 return true;
448 }
449
450 return /* not finite */ false;
451}
452
453static void scanOneBB(Instruction *Start, Instruction *End,
454 std::vector<CallInst *> &Calls,
455 DenseSet<BasicBlock *> &Seen,
456 std::vector<BasicBlock *> &Worklist) {
457 for (BasicBlock::iterator BBI(Start), BBE0 = Start->getParent()->end(),
458 BBE1 = BasicBlock::iterator(End);
459 BBI != BBE0 && BBI != BBE1; BBI++) {
460 if (CallInst *CI = dyn_cast<CallInst>(Val: &*BBI))
461 Calls.push_back(x: CI);
462
463 // FIXME: This code does not handle invokes
464 assert(!isa<InvokeInst>(&*BBI) &&
465 "support for invokes in poll code needed");
466
467 // Only add the successor blocks if we reach the terminator instruction
468 // without encountering end first
469 if (BBI->isTerminator()) {
470 BasicBlock *BB = BBI->getParent();
471 for (BasicBlock *Succ : successors(BB)) {
472 if (Seen.insert(V: Succ).second) {
473 Worklist.push_back(x: Succ);
474 }
475 }
476 }
477 }
478}
479
480static void scanInlinedCode(Instruction *Start, Instruction *End,
481 std::vector<CallInst *> &Calls,
482 DenseSet<BasicBlock *> &Seen) {
483 Calls.clear();
484 std::vector<BasicBlock *> Worklist;
485 Seen.insert(V: Start->getParent());
486 scanOneBB(Start, End, Calls, Seen, Worklist);
487 while (!Worklist.empty()) {
488 BasicBlock *BB = Worklist.back();
489 Worklist.pop_back();
490 scanOneBB(Start: &*BB->begin(), End, Calls, Seen, Worklist);
491 }
492}
493
494/// Returns true if an entry safepoint is not required before this callsite in
495/// the caller function.
496static bool doesNotRequireEntrySafepointBefore(CallBase *Call) {
497 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Call)) {
498 switch (II->getIntrinsicID()) {
499 case Intrinsic::experimental_gc_statepoint:
500 case Intrinsic::experimental_patchpoint_void:
501 case Intrinsic::experimental_patchpoint:
502 // The can wrap an actual call which may grow the stack by an unbounded
503 // amount or run forever.
504 return false;
505 default:
506 // Most LLVM intrinsics are things which do not expand to actual calls, or
507 // at least if they do, are leaf functions that cause only finite stack
508 // growth. In particular, the optimizer likes to form things like memsets
509 // out of stores in the original IR. Another important example is
510 // llvm.localescape which must occur in the entry block. Inserting a
511 // safepoint before it is not legal since it could push the localescape
512 // out of the entry block.
513 return true;
514 }
515 }
516 return false;
517}
518
519static Instruction *findLocationForEntrySafepoint(Function &F,
520 DominatorTree &DT) {
521
522 // Conceptually, this poll needs to be on method entry, but in
523 // practice, we place it as late in the entry block as possible. We
524 // can place it as late as we want as long as it dominates all calls
525 // that can grow the stack. This, combined with backedge polls,
526 // give us all the progress guarantees we need.
527
528 // hasNextInstruction and nextInstruction are used to iterate
529 // through a "straight line" execution sequence.
530
531 auto HasNextInstruction = [](Instruction *I) {
532 if (!I->isTerminator())
533 return true;
534
535 BasicBlock *nextBB = I->getParent()->getUniqueSuccessor();
536 return nextBB && (nextBB->getUniquePredecessor() != nullptr);
537 };
538
539 auto NextInstruction = [&](Instruction *I) {
540 assert(HasNextInstruction(I) &&
541 "first check if there is a next instruction!");
542
543 if (I->isTerminator())
544 return &I->getParent()->getUniqueSuccessor()->front();
545 return &*++I->getIterator();
546 };
547
548 Instruction *Cursor = nullptr;
549 for (Cursor = &F.getEntryBlock().front(); HasNextInstruction(Cursor);
550 Cursor = NextInstruction(Cursor)) {
551
552 // We need to ensure a safepoint poll occurs before any 'real' call. The
553 // easiest way to ensure finite execution between safepoints in the face of
554 // recursive and mutually recursive functions is to enforce that each take
555 // a safepoint. Additionally, we need to ensure a poll before any call
556 // which can grow the stack by an unbounded amount. This isn't required
557 // for GC semantics per se, but is a common requirement for languages
558 // which detect stack overflow via guard pages and then throw exceptions.
559 if (auto *Call = dyn_cast<CallBase>(Val: Cursor)) {
560 if (doesNotRequireEntrySafepointBefore(Call))
561 continue;
562 break;
563 }
564 }
565
566 assert((HasNextInstruction(Cursor) || Cursor->isTerminator()) &&
567 "either we stopped because of a call, or because of terminator");
568
569 return Cursor;
570}
571
572const char GCSafepointPollName[] = "gc.safepoint_poll";
573
574static bool isGCSafepointPoll(Function &F) {
575 return F.getName() == GCSafepointPollName;
576}
577
578/// Returns true if this function should be rewritten to include safepoint
579/// polls and parseable call sites. The main point of this function is to be
580/// an extension point for custom logic.
581static bool shouldRewriteFunction(Function &F) {
582 // TODO: This should check the GCStrategy
583 if (F.hasGC()) {
584 const auto &FunctionGCName = F.getGC();
585 const StringRef StatepointExampleName("statepoint-example");
586 const StringRef CoreCLRName("coreclr");
587 return (StatepointExampleName == FunctionGCName) ||
588 (CoreCLRName == FunctionGCName);
589 } else
590 return false;
591}
592
593// TODO: These should become properties of the GCStrategy, possibly with
594// command line overrides.
595static bool enableEntrySafepoints(const ScalarOptions &Opts, Function &F) {
596 return !Opts.spp_no_entry;
597}
598static bool enableBackedgeSafepoints(const ScalarOptions &Opts, Function &F) {
599 return !Opts.spp_no_backedge;
600}
601static bool enableCallSafepoints(const ScalarOptions &Opts, Function &F) {
602 return !Opts.spp_no_call;
603}
604
605// Insert a safepoint poll immediately before the given instruction. Does
606// not handle the parsability of state at the runtime call, that's the
607// callers job.
608static void
609InsertSafepointPoll(BasicBlock::iterator InsertBefore,
610 std::vector<CallBase *> &ParsePointsNeeded /*rval*/,
611 const TargetLibraryInfo &TLI) {
612 BasicBlock *OrigBB = InsertBefore->getParent();
613 Module *M = InsertBefore->getModule();
614 assert(M && "must be part of a module");
615
616 // Inline the safepoint poll implementation - this will get all the branch,
617 // control flow, etc.. Most importantly, it will introduce the actual slow
618 // path call - where we need to insert a safepoint (parsepoint).
619
620 auto *F = M->getFunction(Name: GCSafepointPollName);
621 assert(F && "gc.safepoint_poll function is missing");
622 assert(F->getFunctionType() ==
623 FunctionType::get(Type::getVoidTy(M->getContext()), false) &&
624 "gc.safepoint_poll declared with wrong type");
625 assert(!F->empty() && "gc.safepoint_poll must be a non-empty function");
626 CallInst *PollCall = CallInst::Create(Func: F, NameStr: "", InsertBefore);
627
628 // Record some information about the call site we're replacing
629 BasicBlock::iterator Before(PollCall), After(PollCall);
630 bool IsBegin = false;
631 if (Before == OrigBB->begin())
632 IsBegin = true;
633 else
634 Before--;
635
636 After++;
637 assert(After != OrigBB->end() && "must have successor");
638
639 // Do the actual inlining
640 InlineFunctionInfo IFI;
641 bool InlineStatus = InlineFunction(CB&: *PollCall, IFI).isSuccess();
642 assert(InlineStatus && "inline must succeed");
643 (void)InlineStatus; // suppress warning in release-asserts
644
645 // Check post-conditions
646 assert(IFI.StaticAllocas.empty() && "can't have allocs");
647
648 std::vector<CallInst *> Calls; // new calls
649 DenseSet<BasicBlock *> BBs; // new BBs + insertee
650
651 // Include only the newly inserted instructions, Note: begin may not be valid
652 // if we inserted to the beginning of the basic block
653 BasicBlock::iterator Start = IsBegin ? OrigBB->begin() : std::next(x: Before);
654
655 // If your poll function includes an unreachable at the end, that's not
656 // valid. Fuzzers/test case reducers can create this, so check for it.
657 assert(isPotentiallyReachable(&*Start, &*After) &&
658 "malformed poll function");
659
660 scanInlinedCode(Start: &*Start, End: &*After, Calls, Seen&: BBs);
661 assert(!Calls.empty() && "slow path not found for safepoint poll");
662
663 // Record the fact we need a parsable state at the runtime call contained in
664 // the poll function. This is required so that the runtime knows how to
665 // parse the last frame when we actually take the safepoint (i.e. execute
666 // the slow path)
667 assert(ParsePointsNeeded.empty());
668 for (auto *CI : Calls) {
669 // No safepoint needed or wanted
670 if (!needsStatepoint(Call: CI, TLI))
671 continue;
672
673 // These are likely runtime calls. Should we assert that via calling
674 // convention or something?
675 ParsePointsNeeded.push_back(x: CI);
676 }
677 assert(ParsePointsNeeded.size() <= Calls.size());
678}
679