1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by an instrumentor.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/Twine.h"
21#include "llvm/Analysis/BlockFrequencyInfo.h"
22#include "llvm/Analysis/BranchProbabilityInfo.h"
23#include "llvm/Analysis/CFG.h"
24#include "llvm/Analysis/LoopInfo.h"
25#include "llvm/Analysis/TargetLibraryInfo.h"
26#include "llvm/IR/Attributes.h"
27#include "llvm/IR/BasicBlock.h"
28#include "llvm/IR/CFG.h"
29#include "llvm/IR/Constant.h"
30#include "llvm/IR/Constants.h"
31#include "llvm/IR/DIBuilder.h"
32#include "llvm/IR/DerivedTypes.h"
33#include "llvm/IR/DiagnosticInfo.h"
34#include "llvm/IR/Dominators.h"
35#include "llvm/IR/Function.h"
36#include "llvm/IR/GlobalValue.h"
37#include "llvm/IR/GlobalVariable.h"
38#include "llvm/IR/IRBuilder.h"
39#include "llvm/IR/Instruction.h"
40#include "llvm/IR/Instructions.h"
41#include "llvm/IR/IntrinsicInst.h"
42#include "llvm/IR/MDBuilder.h"
43#include "llvm/IR/Module.h"
44#include "llvm/IR/Type.h"
45#include "llvm/Pass.h"
46#include "llvm/ProfileData/InstrProf.h"
47#include "llvm/ProfileData/InstrProfCorrelator.h"
48#include "llvm/Support/Casting.h"
49#include "llvm/Support/CommandLine.h"
50#include "llvm/Support/Compiler.h"
51#include "llvm/Support/Error.h"
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/TargetParser/Triple.h"
54#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
55#include "llvm/Transforms/Utils/BasicBlockUtils.h"
56#include "llvm/Transforms/Utils/Instrumentation.h"
57#include "llvm/Transforms/Utils/ModuleUtils.h"
58#include "llvm/Transforms/Utils/SSAUpdater.h"
59#include <algorithm>
60#include <cassert>
61#include <cstdint>
62#include <string>
63
64using namespace llvm;
65
66#define DEBUG_TYPE "instrprof"
67
68namespace llvm {
69// Command line option to enable vtable value profiling. Defined in
70// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
71extern cl::opt<bool> EnableVTableValueProfiling;
72LLVM_ABI cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate(
73 "profile-correlate",
74 cl::desc("Use debug info or binary file to correlate profiles."),
75 cl::init(Val: InstrProfCorrelator::NONE),
76 cl::values(clEnumValN(InstrProfCorrelator::NONE, "",
77 "No profile correlation"),
78 clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info",
79 "Use debug info to correlate"),
80 clEnumValN(InstrProfCorrelator::BINARY, "binary",
81 "Use binary to correlate")));
82} // namespace llvm
83
84namespace {
85
86cl::opt<bool> DoHashBasedCounterSplit(
87 "hash-based-counter-split",
88 cl::desc("Rename counter variable of a comdat function based on cfg hash"),
89 cl::init(Val: true));
90
91cl::opt<bool>
92 RuntimeCounterRelocation("runtime-counter-relocation",
93 cl::desc("Enable relocating counters at runtime."),
94 cl::init(Val: false));
95
96cl::opt<bool> ValueProfileStaticAlloc(
97 "vp-static-alloc",
98 cl::desc("Do static counter allocation for value profiler"),
99 cl::init(Val: true));
100
101cl::opt<double> NumCountersPerValueSite(
102 "vp-counters-per-site",
103 cl::desc("The average number of profile counters allocated "
104 "per value profiling site."),
105 // This is set to a very small value because in real programs, only
106 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
107 // For those sites with non-zero profile, the average number of targets
108 // is usually smaller than 2.
109 cl::init(Val: 1.0));
110
111cl::opt<bool> AtomicCounterUpdateAll(
112 "instrprof-atomic-counter-update-all",
113 cl::desc("Make all profile counter updates atomic (for testing only)"),
114 cl::init(Val: false));
115
116cl::opt<bool> AtomicCounterUpdatePromoted(
117 "atomic-counter-update-promoted",
118 cl::desc("Do counter update using atomic fetch add "
119 " for promoted counters only"),
120 cl::init(Val: false));
121
122cl::opt<bool> AtomicFirstCounter(
123 "atomic-first-counter",
124 cl::desc("Use atomic fetch add for first counter in a function (usually "
125 "the entry counter)"),
126 cl::init(Val: false));
127
128cl::opt<bool> ConditionalCounterUpdate(
129 "conditional-counter-update",
130 cl::desc("Do conditional counter updates in single byte counters mode)"),
131 cl::init(Val: false));
132
133// If the option is not specified, the default behavior about whether
134// counter promotion is done depends on how instrumentation lowering
135// pipeline is setup, i.e., the default value of true of this option
136// does not mean the promotion will be done by default. Explicitly
137// setting this option can override the default behavior.
138cl::opt<bool> DoCounterPromotion("do-counter-promotion",
139 cl::desc("Do counter register promotion"),
140 cl::init(Val: false));
141cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
142 "max-counter-promotions-per-loop", cl::init(Val: 20),
143 cl::desc("Max number counter promotions per loop to avoid"
144 " increasing register pressure too much"));
145
146// A debug option
147cl::opt<int>
148 MaxNumOfPromotions("max-counter-promotions", cl::init(Val: -1),
149 cl::desc("Max number of allowed counter promotions"));
150
151cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
152 "speculative-counter-promotion-max-exiting", cl::init(Val: 3),
153 cl::desc("The max number of exiting blocks of a loop to allow "
154 " speculative counter promotion"));
155
156cl::opt<bool> SpeculativeCounterPromotionToLoop(
157 "speculative-counter-promotion-to-loop",
158 cl::desc("When the option is false, if the target block is in a loop, "
159 "the promotion will be disallowed unless the promoted counter "
160 " update can be further/iteratively promoted into an acyclic "
161 " region."));
162
163cl::opt<bool> IterativeCounterPromotion(
164 "iterative-counter-promotion", cl::init(Val: true),
165 cl::desc("Allow counter promotion across the whole loop nest."));
166
167cl::opt<bool> SkipRetExitBlock(
168 "skip-ret-exit-block", cl::init(Val: true),
169 cl::desc("Suppress counter promotion if exit blocks contain ret."));
170
171static cl::opt<bool> SampledInstr("sampled-instrumentation", cl::ZeroOrMore,
172 cl::init(Val: false),
173 cl::desc("Do PGO instrumentation sampling"));
174
175static cl::opt<unsigned> SampledInstrPeriod(
176 "sampled-instr-period",
177 cl::desc("Set the profile instrumentation sample period. A sample period "
178 "of 0 is invalid. For each sample period, a fixed number of "
179 "consecutive samples will be recorded. The number is controlled "
180 "by 'sampled-instr-burst-duration' flag. The default sample "
181 "period of 65536 is optimized for generating efficient code that "
182 "leverages unsigned short integer wrapping in overflow, but this "
183 "is disabled under simple sampling (burst duration = 1)."),
184 cl::init(USHRT_MAX + 1));
185
186static cl::opt<unsigned> SampledInstrBurstDuration(
187 "sampled-instr-burst-duration",
188 cl::desc("Set the profile instrumentation burst duration, which can range "
189 "from 1 to the value of 'sampled-instr-period' (0 is invalid). "
190 "This number of samples will be recorded for each "
191 "'sampled-instr-period' count update. Setting to 1 enables simple "
192 "sampling, in which case it is recommended to set "
193 "'sampled-instr-period' to a prime number."),
194 cl::init(Val: 200));
195
196struct SampledInstrumentationConfig {
197 unsigned BurstDuration;
198 unsigned Period;
199 bool UseShort;
200 bool IsSimpleSampling;
201 bool IsFastSampling;
202};
203
204static SampledInstrumentationConfig getSampledInstrumentationConfig() {
205 SampledInstrumentationConfig config;
206 config.BurstDuration = SampledInstrBurstDuration.getValue();
207 config.Period = SampledInstrPeriod.getValue();
208 if (config.BurstDuration > config.Period)
209 report_fatal_error(
210 reason: "SampledBurstDuration must be less than or equal to SampledPeriod");
211 if (config.Period == 0 || config.BurstDuration == 0)
212 report_fatal_error(
213 reason: "SampledPeriod and SampledBurstDuration must be greater than 0");
214 config.IsSimpleSampling = (config.BurstDuration == 1);
215 // If (BurstDuration == 1 && Period == 65536), generate the simple sampling
216 // style code.
217 config.IsFastSampling =
218 (!config.IsSimpleSampling && config.Period == USHRT_MAX + 1);
219 config.UseShort = (config.Period <= USHRT_MAX) || config.IsFastSampling;
220 return config;
221}
222
223using LoadStorePair = std::pair<Instruction *, Instruction *>;
224
225static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {
226 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(Val: M.getModuleFlag(Key: Flag));
227 if (!MD)
228 return 0;
229
230 // If the flag is a ConstantAsMetadata, it should be an integer representable
231 // in 64-bits.
232 return cast<ConstantInt>(Val: MD->getValue())->getZExtValue();
233}
234
235static bool enablesValueProfiling(const Module &M) {
236 return isIRPGOFlagSet(M: &M) ||
237 getIntModuleFlagOrZero(M, Flag: "EnableValueProfiling") != 0;
238}
239
240// Conservatively returns true if value profiling is enabled.
241static bool profDataReferencedByCode(const Module &M) {
242 return enablesValueProfiling(M);
243}
244
245class InstrLowerer final {
246public:
247 InstrLowerer(Module &M, const InstrProfOptions &Options,
248 std::function<const TargetLibraryInfo &(Function &F)> GetTLI,
249 bool IsCS)
250 : M(M), Options(Options), TT(M.getTargetTriple()), IsCS(IsCS),
251 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {}
252
253 bool lower();
254
255private:
256 Module &M;
257 const InstrProfOptions Options;
258 const Triple TT;
259 // Is this lowering for the context-sensitive instrumentation.
260 const bool IsCS;
261
262 std::function<const TargetLibraryInfo &(Function &F)> GetTLI;
263
264 const bool DataReferencedByCode;
265
266 struct PerFunctionProfileData {
267 uint32_t NumValueSites[IPVK_Last + 1] = {};
268 GlobalVariable *RegionCounters = nullptr;
269 GlobalVariable *DataVar = nullptr;
270 GlobalVariable *RegionBitmaps = nullptr;
271 uint32_t NumBitmapBytes = 0;
272
273 PerFunctionProfileData() = default;
274 };
275 DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
276 // Key is virtual table variable, value is 'VTableProfData' in the form of
277 // GlobalVariable.
278 DenseMap<GlobalVariable *, GlobalVariable *> VTableDataMap;
279 /// If runtime relocation is enabled, this maps functions to the load
280 /// instruction that produces the profile relocation bias.
281 DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;
282 std::vector<GlobalValue *> CompilerUsedVars;
283 std::vector<GlobalValue *> UsedVars;
284 std::vector<GlobalVariable *> ReferencedNames;
285 // The list of virtual table variables of which the VTableProfData is
286 // collected.
287 std::vector<GlobalVariable *> ReferencedVTables;
288 GlobalVariable *NamesVar = nullptr;
289 size_t NamesSize = 0;
290
291 // vector of counter load/store pairs to be register promoted.
292 std::vector<LoadStorePair> PromotionCandidates;
293
294 int64_t TotalCountersPromoted = 0;
295
296 /// Lower instrumentation intrinsics in the function. Returns true if there
297 /// any lowering.
298 bool lowerIntrinsics(Function *F);
299
300 /// Register-promote counter loads and stores in loops.
301 void promoteCounterLoadStores(Function *F);
302
303 /// Returns true if relocating counters at runtime is enabled.
304 bool isRuntimeCounterRelocationEnabled() const;
305
306 /// Returns true if profile counter update register promotion is enabled.
307 bool isCounterPromotionEnabled() const;
308
309 /// Return true if profile sampling is enabled.
310 bool isSamplingEnabled() const;
311
312 /// Count the number of instrumented value sites for the function.
313 void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
314
315 /// Replace instrprof.value.profile with a call to runtime library.
316 void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
317
318 /// Replace instrprof.cover with a store instruction to the coverage byte.
319 void lowerCover(InstrProfCoverInst *Inc);
320
321 /// Replace instrprof.timestamp with a call to
322 /// INSTR_PROF_PROFILE_SET_TIMESTAMP.
323 void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction);
324
325 /// Replace instrprof.increment with an increment of the appropriate value.
326 void lowerIncrement(InstrProfIncrementInst *Inc);
327
328 /// Force emitting of name vars for unused functions.
329 void lowerCoverageData(GlobalVariable *CoverageNamesVar);
330
331 /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction
332 /// using the index represented by the a temp value into a bitmap.
333 void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins);
334
335 /// Get the Bias value for data to access mmap-ed area.
336 /// Create it if it hasn't been seen.
337 GlobalVariable *getOrCreateBiasVar(StringRef VarName);
338
339 /// Compute the address of the counter value that this profiling instruction
340 /// acts on.
341 Value *getCounterAddress(InstrProfCntrInstBase *I);
342
343 /// Lower the incremental instructions under profile sampling predicates.
344 void doSampling(Instruction *I);
345
346 /// Get the region counters for an increment, creating them if necessary.
347 ///
348 /// If the counter array doesn't yet exist, the profile data variables
349 /// referring to them will also be created.
350 GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc);
351
352 /// Create the region counters.
353 GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc,
354 StringRef Name,
355 GlobalValue::LinkageTypes Linkage);
356
357 /// Compute the address of the test vector bitmap that this profiling
358 /// instruction acts on.
359 Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I);
360
361 /// Get the region bitmaps for an increment, creating them if necessary.
362 ///
363 /// If the bitmap array doesn't yet exist, the profile data variables
364 /// referring to them will also be created.
365 GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc);
366
367 /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with
368 /// an MC/DC Decision region. The number of bytes required is indicated by
369 /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called
370 /// as part of setupProfileSection() and is conceptually very similar to
371 /// what is done for profile data counters in createRegionCounters().
372 GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
373 StringRef Name,
374 GlobalValue::LinkageTypes Linkage);
375
376 /// Set Comdat property of GV, if required.
377 void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName);
378
379 /// Setup the sections into which counters and bitmaps are allocated.
380 GlobalVariable *setupProfileSection(InstrProfInstBase *Inc,
381 InstrProfSectKind IPSK);
382
383 /// Create INSTR_PROF_DATA variable for counters and bitmaps.
384 void createDataVariable(InstrProfCntrInstBase *Inc);
385
386 /// Get the counters for virtual table values, creating them if necessary.
387 void getOrCreateVTableProfData(GlobalVariable *GV);
388
389 /// Emit the section with compressed function names.
390 void emitNameData();
391
392 /// Emit the section with compressed vtable names.
393 void emitVTableNames();
394
395 /// Emit value nodes section for value profiling.
396 void emitVNodes();
397
398 /// Emit runtime registration functions for each profile data variable.
399 void emitRegistration();
400
401 /// Emit the necessary plumbing to pull in the runtime initialization.
402 /// Returns true if a change was made.
403 bool emitRuntimeHook();
404
405 /// Add uses of our data variables and runtime hook.
406 void emitUses();
407
408 /// Create a static initializer for our data, on platforms that need it,
409 /// and for any profile output file that was specified.
410 void emitInitialization();
411};
412
413///
414/// A helper class to promote one counter RMW operation in the loop
415/// into register update.
416///
417/// RWM update for the counter will be sinked out of the loop after
418/// the transformation.
419///
420class PGOCounterPromoterHelper : public LoadAndStorePromoter {
421public:
422 PGOCounterPromoterHelper(
423 Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,
424 BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
425 ArrayRef<Instruction *> InsertPts,
426 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
427 LoopInfo &LI)
428 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
429 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
430 assert(isa<LoadInst>(L));
431 assert(isa<StoreInst>(S));
432 SSA.AddAvailableValue(BB: PH, V: Init);
433 }
434
435 void doExtraRewritesBeforeFinalDeletion() override {
436 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
437 BasicBlock *ExitBlock = ExitBlocks[i];
438 Instruction *InsertPos = InsertPts[i];
439 // Get LiveIn value into the ExitBlock. If there are multiple
440 // predecessors, the value is defined by a PHI node in this
441 // block.
442 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(BB: ExitBlock);
443 Value *Addr = cast<StoreInst>(Val: Store)->getPointerOperand();
444 Type *Ty = LiveInValue->getType();
445 IRBuilder<> Builder(InsertPos);
446 if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Val: Addr)) {
447 // If isRuntimeCounterRelocationEnabled() is true then the address of
448 // the store instruction is computed with two instructions in
449 // InstrProfiling::getCounterAddress(). We need to copy those
450 // instructions to this block to compute Addr correctly.
451 // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
452 // %Addr = inttoptr i64 %BiasAdd to i64*
453 auto *OrigBiasInst = dyn_cast<BinaryOperator>(Val: AddrInst->getOperand(i_nocapture: 0));
454 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
455 Value *BiasInst = Builder.Insert(I: OrigBiasInst->clone());
456 Addr = Builder.CreateIntToPtr(V: BiasInst,
457 DestTy: PointerType::getUnqual(C&: Ty->getContext()));
458 }
459 if (AtomicCounterUpdatePromoted)
460 // automic update currently can only be promoted across the current
461 // loop, not the whole loop nest.
462 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: LiveInValue,
463 Align: MaybeAlign(),
464 Ordering: AtomicOrdering::SequentiallyConsistent);
465 else {
466 LoadInst *OldVal = Builder.CreateLoad(Ty, Ptr: Addr, Name: "pgocount.promoted");
467 auto *NewVal = Builder.CreateAdd(LHS: OldVal, RHS: LiveInValue);
468 auto *NewStore = Builder.CreateStore(Val: NewVal, Ptr: Addr);
469
470 // Now update the parent loop's candidate list:
471 if (IterativeCounterPromotion) {
472 auto *TargetLoop = LI.getLoopFor(BB: ExitBlock);
473 if (TargetLoop)
474 LoopToCandidates[TargetLoop].emplace_back(Args&: OldVal, Args&: NewStore);
475 }
476 }
477 }
478 }
479
480private:
481 Instruction *Store;
482 ArrayRef<BasicBlock *> ExitBlocks;
483 ArrayRef<Instruction *> InsertPts;
484 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
485 LoopInfo &LI;
486};
487
488/// A helper class to do register promotion for all profile counter
489/// updates in a loop.
490///
491class PGOCounterPromoter {
492public:
493 PGOCounterPromoter(
494 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
495 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
496 : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {
497
498 // Skip collection of ExitBlocks and InsertPts for loops that will not be
499 // able to have counters promoted.
500 SmallVector<BasicBlock *, 8> LoopExitBlocks;
501 SmallPtrSet<BasicBlock *, 8> BlockSet;
502
503 L.getExitBlocks(ExitBlocks&: LoopExitBlocks);
504 if (!isPromotionPossible(LP: &L, LoopExitBlocks))
505 return;
506
507 for (BasicBlock *ExitBlock : LoopExitBlocks) {
508 if (BlockSet.insert(Ptr: ExitBlock).second &&
509 llvm::none_of(Range: predecessors(BB: ExitBlock), P: [&](const BasicBlock *Pred) {
510 return llvm::isPresplitCoroSuspendExitEdge(Src: *Pred, Dest: *ExitBlock);
511 })) {
512 ExitBlocks.push_back(Elt: ExitBlock);
513 InsertPts.push_back(Elt: &*ExitBlock->getFirstInsertionPt());
514 }
515 }
516 }
517
518 bool run(int64_t *NumPromoted) {
519 // Skip 'infinite' loops:
520 if (ExitBlocks.size() == 0)
521 return false;
522
523 // Skip if any of the ExitBlocks contains a ret instruction.
524 // This is to prevent dumping of incomplete profile -- if the
525 // the loop is a long running loop and dump is called in the middle
526 // of the loop, the result profile is incomplete.
527 // FIXME: add other heuristics to detect long running loops.
528 if (SkipRetExitBlock) {
529 for (auto *BB : ExitBlocks)
530 if (isa<ReturnInst>(Val: BB->getTerminator()))
531 return false;
532 }
533
534 unsigned MaxProm = getMaxNumOfPromotionsInLoop(LP: &L);
535 if (MaxProm == 0)
536 return false;
537
538 unsigned Promoted = 0;
539 for (auto &Cand : LoopToCandidates[&L]) {
540
541 SmallVector<PHINode *, 4> NewPHIs;
542 SSAUpdater SSA(&NewPHIs);
543 Value *InitVal = ConstantInt::get(Ty: Cand.first->getType(), V: 0);
544
545 // If BFI is set, we will use it to guide the promotions.
546 if (BFI) {
547 auto *BB = Cand.first->getParent();
548 auto InstrCount = BFI->getBlockProfileCount(BB);
549 if (!InstrCount)
550 continue;
551 auto PreheaderCount = BFI->getBlockProfileCount(BB: L.getLoopPreheader());
552 // If the average loop trip count is not greater than 1.5, we skip
553 // promotion.
554 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
555 continue;
556 }
557
558 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
559 L.getLoopPreheader(), ExitBlocks,
560 InsertPts, LoopToCandidates, LI);
561 Promoter.run(Insts: SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
562 Promoted++;
563 if (Promoted >= MaxProm)
564 break;
565
566 (*NumPromoted)++;
567 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
568 break;
569 }
570
571 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
572 << L.getLoopDepth() << ")\n");
573 return Promoted != 0;
574 }
575
576private:
577 bool allowSpeculativeCounterPromotion(Loop *LP) {
578 SmallVector<BasicBlock *, 8> ExitingBlocks;
579 L.getExitingBlocks(ExitingBlocks);
580 // Not considierered speculative.
581 if (ExitingBlocks.size() == 1)
582 return true;
583 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
584 return false;
585 return true;
586 }
587
588 // Check whether the loop satisfies the basic conditions needed to perform
589 // Counter Promotions.
590 bool
591 isPromotionPossible(Loop *LP,
592 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
593 // We can't insert into a catchswitch.
594 if (llvm::any_of(Range: LoopExitBlocks, P: [](BasicBlock *Exit) {
595 return isa<CatchSwitchInst>(Val: Exit->getTerminator());
596 }))
597 return false;
598
599 if (!LP->hasDedicatedExits())
600 return false;
601
602 BasicBlock *PH = LP->getLoopPreheader();
603 if (!PH)
604 return false;
605
606 return true;
607 }
608
609 // Returns the max number of Counter Promotions for LP.
610 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
611 SmallVector<BasicBlock *, 8> LoopExitBlocks;
612 LP->getExitBlocks(ExitBlocks&: LoopExitBlocks);
613 if (!isPromotionPossible(LP, LoopExitBlocks))
614 return 0;
615
616 SmallVector<BasicBlock *, 8> ExitingBlocks;
617 LP->getExitingBlocks(ExitingBlocks);
618
619 // If BFI is set, we do more aggressive promotions based on BFI.
620 if (BFI)
621 return (unsigned)-1;
622
623 // Not considierered speculative.
624 if (ExitingBlocks.size() == 1)
625 return MaxNumOfPromotionsPerLoop;
626
627 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
628 return 0;
629
630 // Whether the target block is in a loop does not matter:
631 if (SpeculativeCounterPromotionToLoop)
632 return MaxNumOfPromotionsPerLoop;
633
634 // Now check the target block:
635 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
636 for (auto *TargetBlock : LoopExitBlocks) {
637 auto *TargetLoop = LI.getLoopFor(BB: TargetBlock);
638 if (!TargetLoop)
639 continue;
640 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(LP: TargetLoop);
641 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
642 MaxProm =
643 std::min(a: MaxProm, b: std::max(a: MaxPromForTarget, b: PendingCandsInTarget) -
644 PendingCandsInTarget);
645 }
646 return MaxProm;
647 }
648
649 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
650 SmallVector<BasicBlock *, 8> ExitBlocks;
651 SmallVector<Instruction *, 8> InsertPts;
652 Loop &L;
653 LoopInfo &LI;
654 BlockFrequencyInfo *BFI;
655};
656
657enum class ValueProfilingCallType {
658 // Individual values are tracked. Currently used for indiret call target
659 // profiling.
660 Default,
661
662 // MemOp: the memop size value profiling.
663 MemOp
664};
665
666} // end anonymous namespace
667
668PreservedAnalyses InstrProfilingLoweringPass::run(Module &M,
669 ModuleAnalysisManager &AM) {
670 FunctionAnalysisManager &FAM =
671 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
672 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
673 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
674 };
675 InstrLowerer Lowerer(M, Options, GetTLI, IsCS);
676 if (!Lowerer.lower())
677 return PreservedAnalyses::all();
678
679 return PreservedAnalyses::none();
680}
681
682//
683// Perform instrumentation sampling.
684//
685// There are 3 favors of sampling:
686// (1) Full burst sampling: We transform:
687// Increment_Instruction;
688// to:
689// if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
690// Increment_Instruction;
691// }
692// __llvm_profile_sampling__ += 1;
693// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
694// __llvm_profile_sampling__ = 0;
695// }
696//
697// "__llvm_profile_sampling__" is a thread-local global shared by all PGO
698// counters (value-instrumentation and edge instrumentation).
699//
700// (2) Fast burst sampling:
701// "__llvm_profile_sampling__" variable is an unsigned type, meaning it will
702// wrap around to zero when overflows. In this case, the second check is
703// unnecessary, so we won't generate check2 when the SampledInstrPeriod is
704// set to 65536 (64K). The code after:
705// if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
706// Increment_Instruction;
707// }
708// __llvm_profile_sampling__ += 1;
709//
710// (3) Simple sampling:
711// When SampledInstrBurstDuration is set to 1, we do a simple sampling:
712// __llvm_profile_sampling__ += 1;
713// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
714// __llvm_profile_sampling__ = 0;
715// Increment_Instruction;
716// }
717//
718// Note that, the code snippet after the transformation can still be counter
719// promoted. However, with sampling enabled, counter updates are expected to
720// be infrequent, making the benefits of counter promotion negligible.
721// Moreover, counter promotion can potentially cause issues in server
722// applications, particularly when the counters are dumped without a clean
723// exit. To mitigate this risk, counter promotion is disabled by default when
724// sampling is enabled. This behavior can be overridden using the internal
725// option.
726void InstrLowerer::doSampling(Instruction *I) {
727 if (!isSamplingEnabled())
728 return;
729
730 SampledInstrumentationConfig config = getSampledInstrumentationConfig();
731 auto GetConstant = [&config](IRBuilder<> &Builder, uint32_t C) {
732 if (config.UseShort)
733 return Builder.getInt16(C);
734 else
735 return Builder.getInt32(C);
736 };
737
738 IntegerType *SamplingVarTy;
739 if (config.UseShort)
740 SamplingVarTy = Type::getInt16Ty(C&: M.getContext());
741 else
742 SamplingVarTy = Type::getInt32Ty(C&: M.getContext());
743 auto *SamplingVar =
744 M.getGlobalVariable(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
745 assert(SamplingVar && "SamplingVar not set properly");
746
747 // Create the condition for checking the burst duration.
748 Instruction *SamplingVarIncr;
749 Value *NewSamplingVarVal;
750 MDBuilder MDB(I->getContext());
751 MDNode *BranchWeight;
752 IRBuilder<> CondBuilder(I);
753 auto *LoadSamplingVar = CondBuilder.CreateLoad(Ty: SamplingVarTy, Ptr: SamplingVar);
754 if (config.IsSimpleSampling) {
755 // For the simple sampling, just create the load and increments.
756 IRBuilder<> IncBuilder(I);
757 NewSamplingVarVal =
758 IncBuilder.CreateAdd(LHS: LoadSamplingVar, RHS: GetConstant(IncBuilder, 1));
759 SamplingVarIncr = IncBuilder.CreateStore(Val: NewSamplingVarVal, Ptr: SamplingVar);
760 } else {
761 // For the burst-sampling, create the conditional update.
762 auto *DurationCond = CondBuilder.CreateICmpULE(
763 LHS: LoadSamplingVar, RHS: GetConstant(CondBuilder, config.BurstDuration - 1));
764 BranchWeight = MDB.createBranchWeights(
765 TrueWeight: config.BurstDuration, FalseWeight: config.Period - config.BurstDuration);
766 Instruction *ThenTerm = SplitBlockAndInsertIfThen(
767 Cond: DurationCond, SplitBefore: I, /* Unreachable */ false, BranchWeights: BranchWeight);
768 IRBuilder<> IncBuilder(I);
769 NewSamplingVarVal =
770 IncBuilder.CreateAdd(LHS: LoadSamplingVar, RHS: GetConstant(IncBuilder, 1));
771 SamplingVarIncr = IncBuilder.CreateStore(Val: NewSamplingVarVal, Ptr: SamplingVar);
772 I->moveBefore(InsertPos: ThenTerm->getIterator());
773 }
774
775 if (config.IsFastSampling)
776 return;
777
778 // Create the condition for checking the period.
779 Instruction *ThenTerm, *ElseTerm;
780 IRBuilder<> PeriodCondBuilder(SamplingVarIncr);
781 auto *PeriodCond = PeriodCondBuilder.CreateICmpUGE(
782 LHS: NewSamplingVarVal, RHS: GetConstant(PeriodCondBuilder, config.Period));
783 BranchWeight = MDB.createBranchWeights(TrueWeight: 1, FalseWeight: config.Period - 1);
784 SplitBlockAndInsertIfThenElse(Cond: PeriodCond, SplitBefore: SamplingVarIncr, ThenTerm: &ThenTerm,
785 ElseTerm: &ElseTerm, BranchWeights: BranchWeight);
786
787 // For the simple sampling, the counter update happens in sampling var reset.
788 if (config.IsSimpleSampling)
789 I->moveBefore(InsertPos: ThenTerm->getIterator());
790
791 IRBuilder<> ResetBuilder(ThenTerm);
792 ResetBuilder.CreateStore(Val: GetConstant(ResetBuilder, 0), Ptr: SamplingVar);
793 SamplingVarIncr->moveBefore(InsertPos: ElseTerm->getIterator());
794}
795
796bool InstrLowerer::lowerIntrinsics(Function *F) {
797 bool MadeChange = false;
798 PromotionCandidates.clear();
799 SmallVector<InstrProfInstBase *, 8> InstrProfInsts;
800
801 // To ensure compatibility with sampling, we save the intrinsics into
802 // a buffer to prevent potential breakage of the iterator (as the
803 // intrinsics will be moved to a different BB).
804 for (BasicBlock &BB : *F) {
805 for (Instruction &Instr : llvm::make_early_inc_range(Range&: BB)) {
806 if (auto *IP = dyn_cast<InstrProfInstBase>(Val: &Instr))
807 InstrProfInsts.push_back(Elt: IP);
808 }
809 }
810
811 for (auto *Instr : InstrProfInsts) {
812 doSampling(I: Instr);
813 if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(Val: Instr)) {
814 lowerIncrement(Inc: IPIS);
815 MadeChange = true;
816 } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(Val: Instr)) {
817 lowerIncrement(Inc: IPI);
818 MadeChange = true;
819 } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(Val: Instr)) {
820 lowerTimestamp(TimestampInstruction: IPC);
821 MadeChange = true;
822 } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(Val: Instr)) {
823 lowerCover(Inc: IPC);
824 MadeChange = true;
825 } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(Val: Instr)) {
826 lowerValueProfileInst(Ins: IPVP);
827 MadeChange = true;
828 } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(Val: Instr)) {
829 IPMP->eraseFromParent();
830 MadeChange = true;
831 } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(Val: Instr)) {
832 lowerMCDCTestVectorBitmapUpdate(Ins: IPBU);
833 MadeChange = true;
834 }
835 }
836
837 if (!MadeChange)
838 return false;
839
840 promoteCounterLoadStores(F);
841 return true;
842}
843
844bool InstrLowerer::isRuntimeCounterRelocationEnabled() const {
845 // Mach-O don't support weak external references.
846 if (TT.isOSBinFormatMachO())
847 return false;
848
849 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
850 return RuntimeCounterRelocation;
851
852 // Fuchsia uses runtime counter relocation by default.
853 return TT.isOSFuchsia();
854}
855
856bool InstrLowerer::isSamplingEnabled() const {
857 if (SampledInstr.getNumOccurrences() > 0)
858 return SampledInstr;
859 return Options.Sampling;
860}
861
862bool InstrLowerer::isCounterPromotionEnabled() const {
863 if (DoCounterPromotion.getNumOccurrences() > 0)
864 return DoCounterPromotion;
865
866 return Options.DoCounterPromotion;
867}
868
869void InstrLowerer::promoteCounterLoadStores(Function *F) {
870 if (!isCounterPromotionEnabled())
871 return;
872
873 DominatorTree DT(*F);
874 LoopInfo LI(DT);
875 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
876
877 std::unique_ptr<BlockFrequencyInfo> BFI;
878 if (Options.UseBFIInPromotion) {
879 std::unique_ptr<BranchProbabilityInfo> BPI;
880 BPI.reset(p: new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
881 BFI.reset(p: new BlockFrequencyInfo(*F, *BPI, LI));
882 }
883
884 for (const auto &LoadStore : PromotionCandidates) {
885 auto *CounterLoad = LoadStore.first;
886 auto *CounterStore = LoadStore.second;
887 BasicBlock *BB = CounterLoad->getParent();
888 Loop *ParentLoop = LI.getLoopFor(BB);
889 if (!ParentLoop)
890 continue;
891 LoopPromotionCandidates[ParentLoop].emplace_back(Args&: CounterLoad, Args&: CounterStore);
892 }
893
894 SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
895
896 // Do a post-order traversal of the loops so that counter updates can be
897 // iteratively hoisted outside the loop nest.
898 for (auto *Loop : llvm::reverse(C&: Loops)) {
899 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
900 Promoter.run(NumPromoted: &TotalCountersPromoted);
901 }
902}
903
904static bool needsRuntimeHookUnconditionally(const Triple &TT) {
905 // On Fuchsia, we only need runtime hook if any counters are present.
906 if (TT.isOSFuchsia())
907 return false;
908
909 return true;
910}
911
912/// Check if the module contains uses of any profiling intrinsics.
913static bool containsProfilingIntrinsics(Module &M) {
914 auto containsIntrinsic = [&](int ID) {
915 if (auto *F = Intrinsic::getDeclarationIfExists(M: &M, id: ID))
916 return !F->use_empty();
917 return false;
918 };
919 return containsIntrinsic(Intrinsic::instrprof_cover) ||
920 containsIntrinsic(Intrinsic::instrprof_increment) ||
921 containsIntrinsic(Intrinsic::instrprof_increment_step) ||
922 containsIntrinsic(Intrinsic::instrprof_timestamp) ||
923 containsIntrinsic(Intrinsic::instrprof_value_profile);
924}
925
926bool InstrLowerer::lower() {
927 bool MadeChange = false;
928 bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
929 if (NeedsRuntimeHook)
930 MadeChange = emitRuntimeHook();
931
932 if (!IsCS && isSamplingEnabled())
933 createProfileSamplingVar(M);
934
935 bool ContainsProfiling = containsProfilingIntrinsics(M);
936 GlobalVariable *CoverageNamesVar =
937 M.getNamedGlobal(Name: getCoverageUnusedNamesVarName());
938 // Improve compile time by avoiding linear scans when there is no work.
939 if (!ContainsProfiling && !CoverageNamesVar)
940 return MadeChange;
941
942 // We did not know how many value sites there would be inside
943 // the instrumented function. This is counting the number of instrumented
944 // target value sites to enter it as field in the profile data variable.
945 for (Function &F : M) {
946 InstrProfCntrInstBase *FirstProfInst = nullptr;
947 for (BasicBlock &BB : F) {
948 for (auto I = BB.begin(), E = BB.end(); I != E; I++) {
949 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Val&: I))
950 computeNumValueSiteCounts(Ins: Ind);
951 else {
952 if (FirstProfInst == nullptr &&
953 (isa<InstrProfIncrementInst>(Val: I) || isa<InstrProfCoverInst>(Val: I)))
954 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(Val&: I);
955 // If the MCDCBitmapParameters intrinsic seen, create the bitmaps.
956 if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(Val&: I))
957 static_cast<void>(getOrCreateRegionBitmaps(Inc: Params));
958 }
959 }
960 }
961
962 // Use a profile intrinsic to create the region counters and data variable.
963 // Also create the data variable based on the MCDCParams.
964 if (FirstProfInst != nullptr) {
965 static_cast<void>(getOrCreateRegionCounters(Inc: FirstProfInst));
966 }
967 }
968
969 if (EnableVTableValueProfiling)
970 for (GlobalVariable &GV : M.globals())
971 // Global variables with type metadata are virtual table variables.
972 if (GV.hasMetadata(KindID: LLVMContext::MD_type))
973 getOrCreateVTableProfData(GV: &GV);
974
975 for (Function &F : M)
976 MadeChange |= lowerIntrinsics(F: &F);
977
978 if (CoverageNamesVar) {
979 lowerCoverageData(CoverageNamesVar);
980 MadeChange = true;
981 }
982
983 if (!MadeChange)
984 return false;
985
986 emitVNodes();
987 emitNameData();
988 emitVTableNames();
989
990 // Emit runtime hook for the cases where the target does not unconditionally
991 // require pulling in profile runtime, and coverage is enabled on code that is
992 // not eliminated by the front-end, e.g. unused functions with internal
993 // linkage.
994 if (!NeedsRuntimeHook && ContainsProfiling)
995 emitRuntimeHook();
996
997 emitRegistration();
998 emitUses();
999 emitInitialization();
1000 return true;
1001}
1002
1003static FunctionCallee getOrInsertValueProfilingCall(
1004 Module &M, const TargetLibraryInfo &TLI,
1005 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
1006 LLVMContext &Ctx = M.getContext();
1007 auto *ReturnTy = Type::getVoidTy(C&: M.getContext());
1008
1009 AttributeList AL;
1010 if (auto AK = TLI.getExtAttrForI32Param(Signed: false))
1011 AL = AL.addParamAttribute(C&: M.getContext(), ArgNo: 2, Kind: AK);
1012
1013 assert((CallType == ValueProfilingCallType::Default ||
1014 CallType == ValueProfilingCallType::MemOp) &&
1015 "Must be Default or MemOp");
1016 Type *ParamTypes[] = {
1017#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
1018#include "llvm/ProfileData/InstrProfData.inc"
1019 };
1020 auto *ValueProfilingCallTy =
1021 FunctionType::get(Result: ReturnTy, Params: ArrayRef(ParamTypes), isVarArg: false);
1022 StringRef FuncName = CallType == ValueProfilingCallType::Default
1023 ? getInstrProfValueProfFuncName()
1024 : getInstrProfValueProfMemOpFuncName();
1025 return M.getOrInsertFunction(Name: FuncName, T: ValueProfilingCallTy, AttributeList: AL);
1026}
1027
1028void InstrLowerer::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
1029 GlobalVariable *Name = Ind->getName();
1030 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1031 uint64_t Index = Ind->getIndex()->getZExtValue();
1032 auto &PD = ProfileDataMap[Name];
1033 PD.NumValueSites[ValueKind] =
1034 std::max(a: PD.NumValueSites[ValueKind], b: (uint32_t)(Index + 1));
1035}
1036
1037void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
1038 // TODO: Value profiling heavily depends on the data section which is omitted
1039 // in lightweight mode. We need to move the value profile pointer to the
1040 // Counter struct to get this working.
1041 assert(
1042 ProfileCorrelate == InstrProfCorrelator::NONE &&
1043 "Value profiling is not yet supported with lightweight instrumentation");
1044 GlobalVariable *Name = Ind->getName();
1045 auto It = ProfileDataMap.find(Val: Name);
1046 assert(It != ProfileDataMap.end() && It->second.DataVar &&
1047 "value profiling detected in function with no counter increment");
1048
1049 GlobalVariable *DataVar = It->second.DataVar;
1050 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1051 uint64_t Index = Ind->getIndex()->getZExtValue();
1052 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
1053 Index += It->second.NumValueSites[Kind];
1054
1055 IRBuilder<> Builder(Ind);
1056 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
1057 llvm::InstrProfValueKind::IPVK_MemOPSize);
1058 CallInst *Call = nullptr;
1059 auto *TLI = &GetTLI(*Ind->getFunction());
1060 auto *NormalizedDataVarPtr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1061 C: DataVar, Ty: PointerType::get(C&: M.getContext(), AddressSpace: 0));
1062
1063 // To support value profiling calls within Windows exception handlers, funclet
1064 // information contained within operand bundles needs to be copied over to
1065 // the library call. This is required for the IR to be processed by the
1066 // WinEHPrepare pass.
1067 SmallVector<OperandBundleDef, 1> OpBundles;
1068 Ind->getOperandBundlesAsDefs(Defs&: OpBundles);
1069 if (!IsMemOpSize) {
1070 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1071 Builder.getInt32(C: Index)};
1072 Call = Builder.CreateCall(Callee: getOrInsertValueProfilingCall(M, TLI: *TLI), Args,
1073 OpBundles);
1074 } else {
1075 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1076 Builder.getInt32(C: Index)};
1077 Call = Builder.CreateCall(
1078 Callee: getOrInsertValueProfilingCall(M, TLI: *TLI, CallType: ValueProfilingCallType::MemOp),
1079 Args, OpBundles);
1080 }
1081 if (auto AK = TLI->getExtAttrForI32Param(Signed: false))
1082 Call->addParamAttr(ArgNo: 2, Kind: AK);
1083 Ind->replaceAllUsesWith(V: Call);
1084 Ind->eraseFromParent();
1085}
1086
1087GlobalVariable *InstrLowerer::getOrCreateBiasVar(StringRef VarName) {
1088 GlobalVariable *Bias = M.getGlobalVariable(Name: VarName);
1089 if (Bias)
1090 return Bias;
1091
1092 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1093
1094 // Compiler must define this variable when runtime counter relocation
1095 // is being used. Runtime has a weak external reference that is used
1096 // to check whether that's the case or not.
1097 Bias = new GlobalVariable(M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
1098 Constant::getNullValue(Ty: Int64Ty), VarName);
1099 Bias->setVisibility(GlobalVariable::HiddenVisibility);
1100 // A definition that's weak (linkonce_odr) without being in a COMDAT
1101 // section wouldn't lead to link errors, but it would lead to a dead
1102 // data word from every TU but one. Putting it in COMDAT ensures there
1103 // will be exactly one data slot in the link.
1104 if (TT.supportsCOMDAT())
1105 Bias->setComdat(M.getOrInsertComdat(Name: VarName));
1106
1107 return Bias;
1108}
1109
1110Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
1111 auto *Counters = getOrCreateRegionCounters(Inc: I);
1112 IRBuilder<> Builder(I);
1113
1114 if (isa<InstrProfTimestampInst>(Val: I))
1115 Counters->setAlignment(Align(8));
1116
1117 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
1118 Ty: Counters->getValueType(), Ptr: Counters, Idx0: 0, Idx1: I->getIndex()->getZExtValue());
1119
1120 if (!isRuntimeCounterRelocationEnabled())
1121 return Addr;
1122
1123 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1124 Function *Fn = I->getParent()->getParent();
1125 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
1126 if (!BiasLI) {
1127 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1128 auto *Bias = getOrCreateBiasVar(VarName: getInstrProfCounterBiasVarName());
1129 BiasLI = EntryBuilder.CreateLoad(Ty: Int64Ty, Ptr: Bias, Name: "profc_bias");
1130 // Bias doesn't change after startup.
1131 BiasLI->setMetadata(KindID: LLVMContext::MD_invariant_load,
1132 Node: MDNode::get(Context&: M.getContext(), MDs: {}));
1133 }
1134 auto *Add = Builder.CreateAdd(LHS: Builder.CreatePtrToInt(V: Addr, DestTy: Int64Ty), RHS: BiasLI);
1135 return Builder.CreateIntToPtr(V: Add, DestTy: Addr->getType());
1136}
1137
1138Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
1139 auto *Bitmaps = getOrCreateRegionBitmaps(Inc: I);
1140 if (!isRuntimeCounterRelocationEnabled())
1141 return Bitmaps;
1142
1143 // Put BiasLI onto the entry block.
1144 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1145 Function *Fn = I->getFunction();
1146 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1147 auto *Bias = getOrCreateBiasVar(VarName: getInstrProfBitmapBiasVarName());
1148 auto *BiasLI = EntryBuilder.CreateLoad(Ty: Int64Ty, Ptr: Bias, Name: "profbm_bias");
1149 // Assume BiasLI invariant (in the function at least)
1150 BiasLI->setMetadata(KindID: LLVMContext::MD_invariant_load,
1151 Node: MDNode::get(Context&: M.getContext(), MDs: {}));
1152
1153 // Add Bias to Bitmaps and put it before the intrinsic.
1154 IRBuilder<> Builder(I);
1155 return Builder.CreatePtrAdd(Ptr: Bitmaps, Offset: BiasLI, Name: "profbm_addr");
1156}
1157
1158void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
1159 auto *Addr = getCounterAddress(I: CoverInstruction);
1160 IRBuilder<> Builder(CoverInstruction);
1161 if (ConditionalCounterUpdate) {
1162 Instruction *SplitBefore = CoverInstruction->getNextNode();
1163 auto &Ctx = CoverInstruction->getParent()->getContext();
1164 auto *Int8Ty = llvm::Type::getInt8Ty(C&: Ctx);
1165 Value *Load = Builder.CreateLoad(Ty: Int8Ty, Ptr: Addr, Name: "pgocount");
1166 Value *Cmp = Builder.CreateIsNotNull(Arg: Load, Name: "pgocount.ifnonzero");
1167 Instruction *ThenBranch =
1168 SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore, Unreachable: false);
1169 Builder.SetInsertPoint(ThenBranch);
1170 }
1171
1172 // We store zero to represent that this block is covered.
1173 Builder.CreateStore(Val: Builder.getInt8(C: 0), Ptr: Addr);
1174 CoverInstruction->eraseFromParent();
1175}
1176
1177void InstrLowerer::lowerTimestamp(
1178 InstrProfTimestampInst *TimestampInstruction) {
1179 assert(TimestampInstruction->getIndex()->isZeroValue() &&
1180 "timestamp probes are always the first probe for a function");
1181 auto &Ctx = M.getContext();
1182 auto *TimestampAddr = getCounterAddress(I: TimestampInstruction);
1183 IRBuilder<> Builder(TimestampInstruction);
1184 auto *CalleeTy =
1185 FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: TimestampAddr->getType(), isVarArg: false);
1186 auto Callee = M.getOrInsertFunction(
1187 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), T: CalleeTy);
1188 Builder.CreateCall(Callee, Args: {TimestampAddr});
1189 TimestampInstruction->eraseFromParent();
1190}
1191
1192void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
1193 auto *Addr = getCounterAddress(I: Inc);
1194
1195 IRBuilder<> Builder(Inc);
1196 if (Options.Atomic || AtomicCounterUpdateAll ||
1197 (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {
1198 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: Inc->getStep(),
1199 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
1200 } else {
1201 Value *IncStep = Inc->getStep();
1202 Value *Load = Builder.CreateLoad(Ty: IncStep->getType(), Ptr: Addr, Name: "pgocount");
1203 auto *Count = Builder.CreateAdd(LHS: Load, RHS: Inc->getStep());
1204 auto *Store = Builder.CreateStore(Val: Count, Ptr: Addr);
1205 if (isCounterPromotionEnabled())
1206 PromotionCandidates.emplace_back(args: cast<Instruction>(Val: Load), args&: Store);
1207 }
1208 Inc->eraseFromParent();
1209}
1210
1211void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
1212 ConstantArray *Names =
1213 cast<ConstantArray>(Val: CoverageNamesVar->getInitializer());
1214 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
1215 Constant *NC = Names->getOperand(i_nocapture: I);
1216 Value *V = NC->stripPointerCasts();
1217 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
1218 GlobalVariable *Name = cast<GlobalVariable>(Val: V);
1219
1220 Name->setLinkage(GlobalValue::PrivateLinkage);
1221 ReferencedNames.push_back(x: Name);
1222 if (isa<ConstantExpr>(Val: NC))
1223 NC->dropAllReferences();
1224 }
1225 CoverageNamesVar->eraseFromParent();
1226}
1227
1228void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
1229 InstrProfMCDCTVBitmapUpdate *Update) {
1230 auto &Ctx = M.getContext();
1231 IRBuilder<> Builder(Update);
1232 auto *Int8Ty = Type::getInt8Ty(C&: Ctx);
1233 auto *Int32Ty = Type::getInt32Ty(C&: Ctx);
1234 auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
1235 auto *BitmapAddr = getBitmapAddress(I: Update);
1236
1237 // Load Temp Val + BitmapIdx.
1238 // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
1239 auto *Temp = Builder.CreateAdd(
1240 LHS: Builder.CreateLoad(Ty: Int32Ty, Ptr: MCDCCondBitmapAddr, Name: "mcdc.temp"),
1241 RHS: Update->getBitmapIndex());
1242
1243 // Calculate byte offset using div8.
1244 // %1 = lshr i32 %mcdc.temp, 3
1245 auto *BitmapByteOffset = Builder.CreateLShr(LHS: Temp, RHS: 0x3);
1246
1247 // Add byte offset to section base byte address.
1248 // %4 = getelementptr inbounds i8, ptr @__profbm_test, i32 %1
1249 auto *BitmapByteAddr =
1250 Builder.CreateInBoundsPtrAdd(Ptr: BitmapAddr, Offset: BitmapByteOffset);
1251
1252 // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8)
1253 // %5 = and i32 %mcdc.temp, 7
1254 // %6 = trunc i32 %5 to i8
1255 auto *BitToSet = Builder.CreateTrunc(V: Builder.CreateAnd(LHS: Temp, RHS: 0x7), DestTy: Int8Ty);
1256
1257 // Shift bit offset left to form a bitmap.
1258 // %7 = shl i8 1, %6
1259 auto *ShiftedVal = Builder.CreateShl(LHS: Builder.getInt8(C: 0x1), RHS: BitToSet);
1260
1261 // Load profile bitmap byte.
1262 // %mcdc.bits = load i8, ptr %4, align 1
1263 auto *Bitmap = Builder.CreateLoad(Ty: Int8Ty, Ptr: BitmapByteAddr, Name: "mcdc.bits");
1264
1265 if (Options.Atomic || AtomicCounterUpdateAll) {
1266 // If ((Bitmap & Val) != Val), then execute atomic (Bitmap |= Val).
1267 // Note, just-loaded Bitmap might not be up-to-date. Use it just for
1268 // early testing.
1269 auto *Masked = Builder.CreateAnd(LHS: Bitmap, RHS: ShiftedVal);
1270 auto *ShouldStore = Builder.CreateICmpNE(LHS: Masked, RHS: ShiftedVal);
1271
1272 // Assume updating will be rare.
1273 auto *Unlikely = MDBuilder(Ctx).createUnlikelyBranchWeights();
1274 Instruction *ThenBranch =
1275 SplitBlockAndInsertIfThen(Cond: ShouldStore, SplitBefore: Update, Unreachable: false, BranchWeights: Unlikely);
1276
1277 // Execute if (unlikely(ShouldStore)).
1278 Builder.SetInsertPoint(ThenBranch);
1279 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: BitmapByteAddr, Val: ShiftedVal,
1280 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
1281 } else {
1282 // Perform logical OR of profile bitmap byte and shifted bit offset.
1283 // %8 = or i8 %mcdc.bits, %7
1284 auto *Result = Builder.CreateOr(LHS: Bitmap, RHS: ShiftedVal);
1285
1286 // Store the updated profile bitmap byte.
1287 // store i8 %8, ptr %3, align 1
1288 Builder.CreateStore(Val: Result, Ptr: BitmapByteAddr);
1289 }
1290
1291 Update->eraseFromParent();
1292}
1293
1294/// Get the name of a profiling variable for a particular function.
1295static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
1296 bool &Renamed) {
1297 StringRef NamePrefix = getInstrProfNameVarPrefix();
1298 StringRef Name = Inc->getName()->getName().substr(Start: NamePrefix.size());
1299 Function *F = Inc->getParent()->getParent();
1300 Module *M = F->getParent();
1301 if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
1302 !canRenameComdatFunc(F: *F)) {
1303 Renamed = false;
1304 return (Prefix + Name).str();
1305 }
1306 Renamed = true;
1307 uint64_t FuncHash = Inc->getHash()->getZExtValue();
1308 SmallVector<char, 24> HashPostfix;
1309 if (Name.ends_with(Suffix: (Twine(".") + Twine(FuncHash)).toStringRef(Out&: HashPostfix)))
1310 return (Prefix + Name).str();
1311 return (Prefix + Name + "." + Twine(FuncHash)).str();
1312}
1313
1314static inline bool shouldRecordFunctionAddr(Function *F) {
1315 // Only record function addresses if IR PGO is enabled or if clang value
1316 // profiling is enabled. Recording function addresses greatly increases object
1317 // file size, because it prevents the inliner from deleting functions that
1318 // have been inlined everywhere.
1319 if (!profDataReferencedByCode(M: *F->getParent()))
1320 return false;
1321
1322 // Check the linkage
1323 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
1324 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
1325 !HasAvailableExternallyLinkage)
1326 return true;
1327
1328 // A function marked 'alwaysinline' with available_externally linkage can't
1329 // have its address taken. Doing so would create an undefined external ref to
1330 // the function, which would fail to link.
1331 if (HasAvailableExternallyLinkage &&
1332 F->hasFnAttribute(Kind: Attribute::AlwaysInline))
1333 return false;
1334
1335 // Prohibit function address recording if the function is both internal and
1336 // COMDAT. This avoids the profile data variable referencing internal symbols
1337 // in COMDAT.
1338 if (F->hasLocalLinkage() && F->hasComdat())
1339 return false;
1340
1341 // Check uses of this function for other than direct calls or invokes to it.
1342 // Inline virtual functions have linkeOnceODR linkage. When a key method
1343 // exists, the vtable will only be emitted in the TU where the key method
1344 // is defined. In a TU where vtable is not available, the function won't
1345 // be 'addresstaken'. If its address is not recorded here, the profile data
1346 // with missing address may be picked by the linker leading to missing
1347 // indirect call target info.
1348 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
1349}
1350
1351static inline bool shouldUsePublicSymbol(Function *Fn) {
1352 // It isn't legal to make an alias of this function at all
1353 if (Fn->isDeclarationForLinker())
1354 return true;
1355
1356 // Symbols with local linkage can just use the symbol directly without
1357 // introducing relocations
1358 if (Fn->hasLocalLinkage())
1359 return true;
1360
1361 // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some
1362 // unfavorable interaction between the new alias and the alias renaming done
1363 // in LowerTypeTests under ThinLTO. For comdat functions that would normally
1364 // be deduplicated, but the renaming scheme ends up preventing renaming, since
1365 // it creates unique names for each alias, resulting in duplicated symbols. In
1366 // the future, we should update the CFI related passes to migrate these
1367 // aliases to the same module as the jump-table they refer to will be defined.
1368 if (Fn->hasMetadata(KindID: LLVMContext::MD_type))
1369 return true;
1370
1371 // For comdat functions, an alias would need the same linkage as the original
1372 // function and hidden visibility. There is no point in adding an alias with
1373 // identical linkage an visibility to avoid introducing symbolic relocations.
1374 if (Fn->hasComdat() &&
1375 (Fn->getVisibility() == GlobalValue::VisibilityTypes::HiddenVisibility))
1376 return true;
1377
1378 // its OK to use an alias
1379 return false;
1380}
1381
1382static inline Constant *getFuncAddrForProfData(Function *Fn) {
1383 auto *Int8PtrTy = PointerType::getUnqual(C&: Fn->getContext());
1384 // Store a nullptr in __llvm_profd, if we shouldn't use a real address
1385 if (!shouldRecordFunctionAddr(F: Fn))
1386 return ConstantPointerNull::get(T: Int8PtrTy);
1387
1388 // If we can't use an alias, we must use the public symbol, even though this
1389 // may require a symbolic relocation.
1390 if (shouldUsePublicSymbol(Fn))
1391 return Fn;
1392
1393 // When possible use a private alias to avoid symbolic relocations.
1394 auto *GA = GlobalAlias::create(Linkage: GlobalValue::LinkageTypes::PrivateLinkage,
1395 Name: Fn->getName() + ".local", Aliasee: Fn);
1396
1397 // When the instrumented function is a COMDAT function, we cannot use a
1398 // private alias. If we did, we would create reference to a local label in
1399 // this function's section. If this version of the function isn't selected by
1400 // the linker, then the metadata would introduce a reference to a discarded
1401 // section. So, for COMDAT functions, we need to adjust the linkage of the
1402 // alias. Using hidden visibility avoids a dynamic relocation and an entry in
1403 // the dynamic symbol table.
1404 //
1405 // Note that this handles COMDAT functions with visibility other than Hidden,
1406 // since that case is covered in shouldUsePublicSymbol()
1407 if (Fn->hasComdat()) {
1408 GA->setLinkage(Fn->getLinkage());
1409 GA->setVisibility(GlobalValue::VisibilityTypes::HiddenVisibility);
1410 }
1411
1412 // appendToCompilerUsed(*Fn->getParent(), {GA});
1413
1414 return GA;
1415}
1416
1417static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
1418 // compiler-rt uses linker support to get data/counters/name start/end for
1419 // ELF, COFF, Mach-O, XCOFF, and Wasm.
1420 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
1421 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF() ||
1422 TT.isOSBinFormatWasm())
1423 return false;
1424
1425 return true;
1426}
1427
1428void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO,
1429 StringRef CounterGroupName) {
1430 // Place lowered global variables in a comdat group if the associated function
1431 // or global variable is a COMDAT. This will make sure that only one copy of
1432 // global variable (e.g. function counters) of the COMDAT function will be
1433 // emitted after linking.
1434 bool NeedComdat = needsComdatForCounter(GV: *GO, M);
1435 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
1436
1437 if (!UseComdat)
1438 return;
1439
1440 // Keep in mind that this pass may run before the inliner, so we need to
1441 // create a new comdat group (for counters, profiling data, etc). If we use
1442 // the comdat of the parent function, that will result in relocations against
1443 // discarded sections.
1444 //
1445 // If the data variable is referenced by code, non-counter variables (notably
1446 // profiling data) and counters have to be in different comdats for COFF
1447 // because the Visual C++ linker will report duplicate symbol errors if there
1448 // are multiple external symbols with the same name marked
1449 // IMAGE_COMDAT_SELECT_ASSOCIATIVE.
1450 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
1451 ? GV->getName()
1452 : CounterGroupName;
1453 Comdat *C = M.getOrInsertComdat(Name: GroupName);
1454
1455 if (!NeedComdat) {
1456 // Object file format must be ELF since `UseComdat && !NeedComdat` is true.
1457 //
1458 // For ELF, when not using COMDAT, put counters, data and values into a
1459 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
1460 // allows -z start-stop-gc to discard the entire group when the function is
1461 // discarded.
1462 C->setSelectionKind(Comdat::NoDeduplicate);
1463 }
1464 GV->setComdat(C);
1465 // COFF doesn't allow the comdat group leader to have private linkage, so
1466 // upgrade private linkage to internal linkage to produce a symbol table
1467 // entry.
1468 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
1469 GV->setLinkage(GlobalValue::InternalLinkage);
1470}
1471
1472static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {
1473 if (!profDataReferencedByCode(M: *GV->getParent()))
1474 return false;
1475
1476 if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&
1477 !GV->hasAvailableExternallyLinkage())
1478 return true;
1479
1480 // This avoids the profile data from referencing internal symbols in
1481 // COMDAT.
1482 if (GV->hasLocalLinkage() && GV->hasComdat())
1483 return false;
1484
1485 return true;
1486}
1487
1488// FIXME: Introduce an internal alias like what's done for functions to reduce
1489// the number of relocation entries.
1490static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
1491 // Store a nullptr in __profvt_ if a real address shouldn't be used.
1492 if (!shouldRecordVTableAddr(GV))
1493 return ConstantPointerNull::get(T: PointerType::getUnqual(C&: GV->getContext()));
1494
1495 return GV;
1496}
1497
1498void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) {
1499 assert(ProfileCorrelate != InstrProfCorrelator::DEBUG_INFO &&
1500 "Value profiling is not supported with lightweight instrumentation");
1501 if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
1502 return;
1503
1504 // Skip llvm internal global variable or __prof variables.
1505 if (GV->getName().starts_with(Prefix: "llvm.") ||
1506 GV->getName().starts_with(Prefix: "__llvm") ||
1507 GV->getName().starts_with(Prefix: "__prof"))
1508 return;
1509
1510 // VTableProfData already created
1511 auto It = VTableDataMap.find(Val: GV);
1512 if (It != VTableDataMap.end() && It->second)
1513 return;
1514
1515 GlobalValue::LinkageTypes Linkage = GV->getLinkage();
1516 GlobalValue::VisibilityTypes Visibility = GV->getVisibility();
1517
1518 // This is to keep consistent with per-function profile data
1519 // for correctness.
1520 if (TT.isOSBinFormatXCOFF()) {
1521 Linkage = GlobalValue::InternalLinkage;
1522 Visibility = GlobalValue::DefaultVisibility;
1523 }
1524
1525 LLVMContext &Ctx = M.getContext();
1526 Type *DataTypes[] = {
1527#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
1528#include "llvm/ProfileData/InstrProfData.inc"
1529#undef INSTR_PROF_VTABLE_DATA
1530 };
1531
1532 auto *DataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes));
1533
1534 // Used by INSTR_PROF_VTABLE_DATA MACRO
1535 Constant *VTableAddr = getVTableAddrForProfData(GV);
1536 const std::string PGOVTableName = getPGOName(V: *GV);
1537 // Record the length of the vtable. This is needed since vtable pointers
1538 // loaded from C++ objects might be from the middle of a vtable definition.
1539 uint32_t VTableSizeVal = GV->getGlobalSize(DL: M.getDataLayout());
1540
1541 Constant *DataVals[] = {
1542#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
1543#include "llvm/ProfileData/InstrProfData.inc"
1544#undef INSTR_PROF_VTABLE_DATA
1545 };
1546
1547 auto *Data =
1548 new GlobalVariable(M, DataTy, /*constant=*/false, Linkage,
1549 ConstantStruct::get(T: DataTy, V: DataVals),
1550 getInstrProfVTableVarPrefix() + PGOVTableName);
1551
1552 Data->setVisibility(Visibility);
1553 Data->setSection(getInstrProfSectionName(IPSK: IPSK_vtab, OF: TT.getObjectFormat()));
1554 Data->setAlignment(Align(8));
1555
1556 maybeSetComdat(GV: Data, GO: GV, CounterGroupName: Data->getName());
1557
1558 VTableDataMap[GV] = Data;
1559
1560 ReferencedVTables.push_back(x: GV);
1561
1562 // VTable <Hash, Addr> is used by runtime but not referenced by other
1563 // sections. Conservatively mark it linker retained.
1564 UsedVars.push_back(x: Data);
1565}
1566
1567GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc,
1568 InstrProfSectKind IPSK) {
1569 GlobalVariable *NamePtr = Inc->getName();
1570
1571 // Match the linkage and visibility of the name global.
1572 Function *Fn = Inc->getParent()->getParent();
1573 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
1574 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1575
1576 // Use internal rather than private linkage so the counter variable shows up
1577 // in the symbol table when using debug info for correlation.
1578 if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO &&
1579 TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage)
1580 Linkage = GlobalValue::InternalLinkage;
1581
1582 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1583 // symbols in the same csect won't be discarded. When there are duplicate weak
1584 // symbols, we can NOT guarantee that the relocations get resolved to the
1585 // intended weak symbol, so we can not ensure the correctness of the relative
1586 // CounterPtr, so we have to use private linkage for counter and data symbols.
1587 if (TT.isOSBinFormatXCOFF()) {
1588 Linkage = GlobalValue::PrivateLinkage;
1589 Visibility = GlobalValue::DefaultVisibility;
1590 }
1591 // Move the name variable to the right section.
1592 bool Renamed;
1593 GlobalVariable *Ptr;
1594 StringRef VarPrefix;
1595 std::string VarName;
1596 if (IPSK == IPSK_cnts) {
1597 VarPrefix = getInstrProfCountersVarPrefix();
1598 VarName = getVarName(Inc, Prefix: VarPrefix, Renamed);
1599 InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Val: Inc);
1600 Ptr = createRegionCounters(Inc: CntrIncrement, Name: VarName, Linkage);
1601 } else if (IPSK == IPSK_bitmap) {
1602 VarPrefix = getInstrProfBitmapVarPrefix();
1603 VarName = getVarName(Inc, Prefix: VarPrefix, Renamed);
1604 InstrProfMCDCBitmapInstBase *BitmapUpdate =
1605 dyn_cast<InstrProfMCDCBitmapInstBase>(Val: Inc);
1606 Ptr = createRegionBitmaps(Inc: BitmapUpdate, Name: VarName, Linkage);
1607 } else {
1608 llvm_unreachable("Profile Section must be for Counters or Bitmaps");
1609 }
1610
1611 Ptr->setVisibility(Visibility);
1612 // Put the counters and bitmaps in their own sections so linkers can
1613 // remove unneeded sections.
1614 Ptr->setSection(getInstrProfSectionName(IPSK, OF: TT.getObjectFormat()));
1615 Ptr->setLinkage(Linkage);
1616 maybeSetComdat(GV: Ptr, GO: Fn, CounterGroupName: VarName);
1617 return Ptr;
1618}
1619
1620GlobalVariable *
1621InstrLowerer::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
1622 StringRef Name,
1623 GlobalValue::LinkageTypes Linkage) {
1624 uint64_t NumBytes = Inc->getNumBitmapBytes();
1625 auto *BitmapTy = ArrayType::get(ElementType: Type::getInt8Ty(C&: M.getContext()), NumElements: NumBytes);
1626 auto GV = new GlobalVariable(M, BitmapTy, false, Linkage,
1627 Constant::getNullValue(Ty: BitmapTy), Name);
1628 GV->setAlignment(Align(1));
1629 return GV;
1630}
1631
1632GlobalVariable *
1633InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
1634 GlobalVariable *NamePtr = Inc->getName();
1635 auto &PD = ProfileDataMap[NamePtr];
1636 if (PD.RegionBitmaps)
1637 return PD.RegionBitmaps;
1638
1639 // If RegionBitmaps doesn't already exist, create it by first setting up
1640 // the corresponding profile section.
1641 auto *BitmapPtr = setupProfileSection(Inc, IPSK: IPSK_bitmap);
1642 PD.RegionBitmaps = BitmapPtr;
1643 PD.NumBitmapBytes = Inc->getNumBitmapBytes();
1644 return PD.RegionBitmaps;
1645}
1646
1647GlobalVariable *
1648InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
1649 GlobalValue::LinkageTypes Linkage) {
1650 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1651 auto &Ctx = M.getContext();
1652 GlobalVariable *GV;
1653 if (isa<InstrProfCoverInst>(Val: Inc)) {
1654 auto *CounterTy = Type::getInt8Ty(C&: Ctx);
1655 auto *CounterArrTy = ArrayType::get(ElementType: CounterTy, NumElements: NumCounters);
1656 // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
1657 std::vector<Constant *> InitialValues(NumCounters,
1658 Constant::getAllOnesValue(Ty: CounterTy));
1659 GV = new GlobalVariable(M, CounterArrTy, false, Linkage,
1660 ConstantArray::get(T: CounterArrTy, V: InitialValues),
1661 Name);
1662 GV->setAlignment(Align(1));
1663 } else {
1664 auto *CounterTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NumCounters);
1665 GV = new GlobalVariable(M, CounterTy, false, Linkage,
1666 Constant::getNullValue(Ty: CounterTy), Name);
1667 GV->setAlignment(Align(8));
1668 }
1669 return GV;
1670}
1671
1672GlobalVariable *
1673InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
1674 GlobalVariable *NamePtr = Inc->getName();
1675 auto &PD = ProfileDataMap[NamePtr];
1676 if (PD.RegionCounters)
1677 return PD.RegionCounters;
1678
1679 // If RegionCounters doesn't already exist, create it by first setting up
1680 // the corresponding profile section.
1681 auto *CounterPtr = setupProfileSection(Inc, IPSK: IPSK_cnts);
1682 PD.RegionCounters = CounterPtr;
1683
1684 if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) {
1685 LLVMContext &Ctx = M.getContext();
1686 Function *Fn = Inc->getParent()->getParent();
1687 if (auto *SP = Fn->getSubprogram()) {
1688 DIBuilder DB(M, true, SP->getUnit());
1689 Metadata *FunctionNameAnnotation[] = {
1690 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::FunctionNameAttributeName),
1691 MDString::get(Context&: Ctx, Str: getPGOFuncNameVarInitializer(NameVar: NamePtr)),
1692 };
1693 Metadata *CFGHashAnnotation[] = {
1694 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::CFGHashAttributeName),
1695 ConstantAsMetadata::get(C: Inc->getHash()),
1696 };
1697 Metadata *NumCountersAnnotation[] = {
1698 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::NumCountersAttributeName),
1699 ConstantAsMetadata::get(C: Inc->getNumCounters()),
1700 };
1701 auto Annotations = DB.getOrCreateArray(Elements: {
1702 MDNode::get(Context&: Ctx, MDs: FunctionNameAnnotation),
1703 MDNode::get(Context&: Ctx, MDs: CFGHashAnnotation),
1704 MDNode::get(Context&: Ctx, MDs: NumCountersAnnotation),
1705 });
1706 auto *DICounter = DB.createGlobalVariableExpression(
1707 Context: SP, Name: CounterPtr->getName(), /*LinkageName=*/StringRef(), File: SP->getFile(),
1708 /*LineNo=*/0, Ty: DB.createUnspecifiedType(Name: "Profile Data Type"),
1709 IsLocalToUnit: CounterPtr->hasLocalLinkage(), /*IsDefined=*/isDefined: true, /*Expr=*/nullptr,
1710 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1711 Annotations);
1712 CounterPtr->addDebugInfo(GV: DICounter);
1713 DB.finalize();
1714 }
1715
1716 // Mark the counter variable as used so that it isn't optimized out.
1717 CompilerUsedVars.push_back(x: PD.RegionCounters);
1718 }
1719
1720 // Create the data variable (if it doesn't already exist).
1721 createDataVariable(Inc);
1722
1723 return PD.RegionCounters;
1724}
1725
1726void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
1727 // When debug information is correlated to profile data, a data variable
1728 // is not needed.
1729 if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)
1730 return;
1731
1732 GlobalVariable *NamePtr = Inc->getName();
1733 auto &PD = ProfileDataMap[NamePtr];
1734
1735 // Return if data variable was already created.
1736 if (PD.DataVar)
1737 return;
1738
1739 LLVMContext &Ctx = M.getContext();
1740
1741 Function *Fn = Inc->getParent()->getParent();
1742 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
1743 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1744
1745 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1746 // symbols in the same csect won't be discarded. When there are duplicate weak
1747 // symbols, we can NOT guarantee that the relocations get resolved to the
1748 // intended weak symbol, so we can not ensure the correctness of the relative
1749 // CounterPtr, so we have to use private linkage for counter and data symbols.
1750 if (TT.isOSBinFormatXCOFF()) {
1751 Linkage = GlobalValue::PrivateLinkage;
1752 Visibility = GlobalValue::DefaultVisibility;
1753 }
1754
1755 bool NeedComdat = needsComdatForCounter(GV: *Fn, M);
1756 bool Renamed;
1757
1758 // The Data Variable section is anchored to profile counters.
1759 std::string CntsVarName =
1760 getVarName(Inc, Prefix: getInstrProfCountersVarPrefix(), Renamed);
1761 std::string DataVarName =
1762 getVarName(Inc, Prefix: getInstrProfDataVarPrefix(), Renamed);
1763
1764 auto *Int8PtrTy = PointerType::getUnqual(C&: Ctx);
1765 // Allocate statically the array of pointers to value profile nodes for
1766 // the current function.
1767 Constant *ValuesPtrExpr = ConstantPointerNull::get(T: Int8PtrTy);
1768 uint64_t NS = 0;
1769 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1770 NS += PD.NumValueSites[Kind];
1771 if (NS > 0 && ValueProfileStaticAlloc &&
1772 !needsRuntimeRegistrationOfSectionRange(TT)) {
1773 ArrayType *ValuesTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NS);
1774 auto *ValuesVar = new GlobalVariable(
1775 M, ValuesTy, false, Linkage, Constant::getNullValue(Ty: ValuesTy),
1776 getVarName(Inc, Prefix: getInstrProfValuesVarPrefix(), Renamed));
1777 ValuesVar->setVisibility(Visibility);
1778 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *ValuesVar);
1779 ValuesVar->setSection(
1780 getInstrProfSectionName(IPSK: IPSK_vals, OF: TT.getObjectFormat()));
1781 ValuesVar->setAlignment(Align(8));
1782 maybeSetComdat(GV: ValuesVar, GO: Fn, CounterGroupName: CntsVarName);
1783 ValuesPtrExpr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1784 C: ValuesVar, Ty: PointerType::get(C&: Fn->getContext(), AddressSpace: 0));
1785 }
1786
1787 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1788 auto *CounterPtr = PD.RegionCounters;
1789
1790 uint64_t NumBitmapBytes = PD.NumBitmapBytes;
1791
1792 // Create data variable.
1793 auto *IntPtrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
1794 auto *Int16Ty = Type::getInt16Ty(C&: Ctx);
1795 auto *Int16ArrayTy = ArrayType::get(ElementType: Int16Ty, NumElements: IPVK_Last + 1);
1796 Type *DataTypes[] = {
1797#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1798#include "llvm/ProfileData/InstrProfData.inc"
1799 };
1800 auto *DataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes));
1801
1802 Constant *FunctionAddr = getFuncAddrForProfData(Fn);
1803
1804 Constant *Int16ArrayVals[IPVK_Last + 1];
1805 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1806 Int16ArrayVals[Kind] = ConstantInt::get(Ty: Int16Ty, V: PD.NumValueSites[Kind]);
1807
1808 if (isGPUProfTarget(M)) {
1809 Linkage = GlobalValue::ExternalLinkage;
1810 Visibility = GlobalValue::ProtectedVisibility;
1811 }
1812 // If the data variable is not referenced by code (if we don't emit
1813 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
1814 // data variable live under linker GC, the data variable can be private. This
1815 // optimization applies to ELF.
1816 //
1817 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
1818 // to be false.
1819 //
1820 // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
1821 // that other copies must have the same CFG and cannot have value profiling.
1822 // If no hash suffix, other profd copies may be referenced by code.
1823 else if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1824 (TT.isOSBinFormatELF() ||
1825 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
1826 Linkage = GlobalValue::PrivateLinkage;
1827 Visibility = GlobalValue::DefaultVisibility;
1828 }
1829 auto *Data =
1830 new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName);
1831 Constant *RelativeCounterPtr;
1832 GlobalVariable *BitmapPtr = PD.RegionBitmaps;
1833 Constant *RelativeBitmapPtr = ConstantInt::get(Ty: IntPtrTy, V: 0);
1834 InstrProfSectKind DataSectionKind;
1835 // With binary profile correlation, profile data is not loaded into memory.
1836 // profile data must reference profile counter with an absolute relocation.
1837 if (ProfileCorrelate == InstrProfCorrelator::BINARY) {
1838 DataSectionKind = IPSK_covdata;
1839 RelativeCounterPtr = ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy);
1840 if (BitmapPtr != nullptr)
1841 RelativeBitmapPtr = ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy);
1842 } else {
1843 // Reference the counter variable with a label difference (link-time
1844 // constant).
1845 DataSectionKind = IPSK_data;
1846 RelativeCounterPtr =
1847 ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy),
1848 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
1849 if (BitmapPtr != nullptr)
1850 RelativeBitmapPtr =
1851 ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy),
1852 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
1853 }
1854
1855 Constant *DataVals[] = {
1856#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1857#include "llvm/ProfileData/InstrProfData.inc"
1858 };
1859 Data->setInitializer(ConstantStruct::get(T: DataTy, V: DataVals));
1860
1861 Data->setVisibility(Visibility);
1862 Data->setSection(
1863 getInstrProfSectionName(IPSK: DataSectionKind, OF: TT.getObjectFormat()));
1864 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
1865 maybeSetComdat(GV: Data, GO: Fn, CounterGroupName: CntsVarName);
1866
1867 PD.DataVar = Data;
1868
1869 // Mark the data variable as used so that it isn't stripped out.
1870 CompilerUsedVars.push_back(x: Data);
1871 // Now that the linkage set by the FE has been passed to the data and counter
1872 // variables, reset Name variable's linkage and visibility to private so that
1873 // it can be removed later by the compiler.
1874 NamePtr->setLinkage(GlobalValue::PrivateLinkage);
1875 // Collect the referenced names to be used by emitNameData.
1876 ReferencedNames.push_back(x: NamePtr);
1877}
1878
1879void InstrLowerer::emitVNodes() {
1880 if (!ValueProfileStaticAlloc)
1881 return;
1882
1883 // For now only support this on platforms that do
1884 // not require runtime registration to discover
1885 // named section start/end.
1886 if (needsRuntimeRegistrationOfSectionRange(TT))
1887 return;
1888
1889 size_t TotalNS = 0;
1890 for (auto &PD : ProfileDataMap) {
1891 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1892 TotalNS += PD.second.NumValueSites[Kind];
1893 }
1894
1895 if (!TotalNS)
1896 return;
1897
1898 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1899// Heuristic for small programs with very few total value sites.
1900// The default value of vp-counters-per-site is chosen based on
1901// the observation that large apps usually have a low percentage
1902// of value sites that actually have any profile data, and thus
1903// the average number of counters per site is low. For small
1904// apps with very few sites, this may not be true. Bump up the
1905// number of counters in this case.
1906#define INSTR_PROF_MIN_VAL_COUNTS 10
1907 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
1908 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, b: (int)NumCounters * 2);
1909
1910 auto &Ctx = M.getContext();
1911 Type *VNodeTypes[] = {
1912#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1913#include "llvm/ProfileData/InstrProfData.inc"
1914 };
1915 auto *VNodeTy = StructType::get(Context&: Ctx, Elements: ArrayRef(VNodeTypes));
1916
1917 ArrayType *VNodesTy = ArrayType::get(ElementType: VNodeTy, NumElements: NumCounters);
1918 auto *VNodesVar = new GlobalVariable(
1919 M, VNodesTy, false, GlobalValue::PrivateLinkage,
1920 Constant::getNullValue(Ty: VNodesTy), getInstrProfVNodesVarName());
1921 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *VNodesVar);
1922 VNodesVar->setSection(
1923 getInstrProfSectionName(IPSK: IPSK_vnodes, OF: TT.getObjectFormat()));
1924 VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(Ty: VNodesTy));
1925 // VNodesVar is used by runtime but not referenced via relocation by other
1926 // sections. Conservatively make it linker retained.
1927 UsedVars.push_back(x: VNodesVar);
1928}
1929
1930void InstrLowerer::emitNameData() {
1931 if (ReferencedNames.empty())
1932 return;
1933
1934 std::string CompressedNameStr;
1935 if (Error E = collectPGOFuncNameStrings(NameVars: ReferencedNames, Result&: CompressedNameStr,
1936 doCompression: DoInstrProfNameCompression)) {
1937 report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false);
1938 }
1939
1940 auto &Ctx = M.getContext();
1941 auto *NamesVal =
1942 ConstantDataArray::getString(Context&: Ctx, Initializer: StringRef(CompressedNameStr), AddNull: false);
1943 NamesVar = new GlobalVariable(M, NamesVal->getType(), true,
1944 GlobalValue::PrivateLinkage, NamesVal,
1945 getInstrProfNamesVarName());
1946 if (isGPUProfTarget(M)) {
1947 NamesVar->setLinkage(GlobalValue::ExternalLinkage);
1948 NamesVar->setVisibility(GlobalValue::ProtectedVisibility);
1949 }
1950
1951 NamesSize = CompressedNameStr.size();
1952 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *NamesVar);
1953 NamesVar->setSection(
1954 ProfileCorrelate == InstrProfCorrelator::BINARY
1955 ? getInstrProfSectionName(IPSK: IPSK_covname, OF: TT.getObjectFormat())
1956 : getInstrProfSectionName(IPSK: IPSK_name, OF: TT.getObjectFormat()));
1957 // On COFF, it's important to reduce the alignment down to 1 to prevent the
1958 // linker from inserting padding before the start of the names section or
1959 // between names entries.
1960 NamesVar->setAlignment(Align(1));
1961 // NamesVar is used by runtime but not referenced via relocation by other
1962 // sections. Conservatively make it linker retained.
1963 UsedVars.push_back(x: NamesVar);
1964
1965 for (auto *NamePtr : ReferencedNames)
1966 NamePtr->eraseFromParent();
1967}
1968
1969void InstrLowerer::emitVTableNames() {
1970 if (!EnableVTableValueProfiling || ReferencedVTables.empty())
1971 return;
1972
1973 // Collect the PGO names of referenced vtables and compress them.
1974 std::string CompressedVTableNames;
1975 if (Error E = collectVTableStrings(VTables: ReferencedVTables, Result&: CompressedVTableNames,
1976 doCompression: DoInstrProfNameCompression)) {
1977 report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false);
1978 }
1979
1980 auto &Ctx = M.getContext();
1981 auto *VTableNamesVal = ConstantDataArray::getString(
1982 Context&: Ctx, Initializer: StringRef(CompressedVTableNames), AddNull: false /* AddNull */);
1983 GlobalVariable *VTableNamesVar =
1984 new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */,
1985 GlobalValue::PrivateLinkage, VTableNamesVal,
1986 getInstrProfVTableNamesVarName());
1987 VTableNamesVar->setSection(
1988 getInstrProfSectionName(IPSK: IPSK_vname, OF: TT.getObjectFormat()));
1989 VTableNamesVar->setAlignment(Align(1));
1990 // Make VTableNames linker retained.
1991 UsedVars.push_back(x: VTableNamesVar);
1992}
1993
1994void InstrLowerer::emitRegistration() {
1995 if (!needsRuntimeRegistrationOfSectionRange(TT))
1996 return;
1997
1998 // Construct the function.
1999 auto *VoidTy = Type::getVoidTy(C&: M.getContext());
2000 auto *VoidPtrTy = PointerType::getUnqual(C&: M.getContext());
2001 auto *Int64Ty = Type::getInt64Ty(C&: M.getContext());
2002 auto *RegisterFTy = FunctionType::get(Result: VoidTy, isVarArg: false);
2003 auto *RegisterF = Function::Create(Ty: RegisterFTy, Linkage: GlobalValue::InternalLinkage,
2004 N: getInstrProfRegFuncsName(), M);
2005 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2006 if (Options.NoRedZone)
2007 RegisterF->addFnAttr(Kind: Attribute::NoRedZone);
2008
2009 auto *RuntimeRegisterTy = FunctionType::get(Result: VoidTy, Params: VoidPtrTy, isVarArg: false);
2010 auto *RuntimeRegisterF =
2011 Function::Create(Ty: RuntimeRegisterTy, Linkage: GlobalVariable::ExternalLinkage,
2012 N: getInstrProfRegFuncName(), M);
2013
2014 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: RegisterF));
2015 for (Value *Data : CompilerUsedVars)
2016 if (!isa<Function>(Val: Data))
2017 // Check for addrspace cast when profiling GPU
2018 IRB.CreateCall(Callee: RuntimeRegisterF,
2019 Args: IRB.CreatePointerBitCastOrAddrSpaceCast(V: Data, DestTy: VoidPtrTy));
2020 for (Value *Data : UsedVars)
2021 if (Data != NamesVar && !isa<Function>(Val: Data))
2022 IRB.CreateCall(Callee: RuntimeRegisterF,
2023 Args: IRB.CreatePointerBitCastOrAddrSpaceCast(V: Data, DestTy: VoidPtrTy));
2024
2025 if (NamesVar) {
2026 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
2027 auto *NamesRegisterTy =
2028 FunctionType::get(Result: VoidTy, Params: ArrayRef(ParamTypes), isVarArg: false);
2029 auto *NamesRegisterF =
2030 Function::Create(Ty: NamesRegisterTy, Linkage: GlobalVariable::ExternalLinkage,
2031 N: getInstrProfNamesRegFuncName(), M);
2032 IRB.CreateCall(Callee: NamesRegisterF, Args: {IRB.CreatePointerBitCastOrAddrSpaceCast(
2033 V: NamesVar, DestTy: VoidPtrTy),
2034 IRB.getInt64(C: NamesSize)});
2035 }
2036
2037 IRB.CreateRetVoid();
2038}
2039
2040bool InstrLowerer::emitRuntimeHook() {
2041 // We expect the linker to be invoked with -u<hook_var> flag for Linux
2042 // in which case there is no need to emit the external variable.
2043 if (TT.isOSLinux() || TT.isOSAIX())
2044 return false;
2045
2046 // If the module's provided its own runtime, we don't need to do anything.
2047 if (M.getGlobalVariable(Name: getInstrProfRuntimeHookVarName()))
2048 return false;
2049
2050 // Declare an external variable that will pull in the runtime initialization.
2051 auto *Int32Ty = Type::getInt32Ty(C&: M.getContext());
2052 auto *Var =
2053 new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
2054 nullptr, getInstrProfRuntimeHookVarName());
2055 if (isGPUProfTarget(M))
2056 Var->setVisibility(GlobalValue::ProtectedVisibility);
2057 else
2058 Var->setVisibility(GlobalValue::HiddenVisibility);
2059
2060 if (TT.isOSBinFormatELF() && !TT.isPS()) {
2061 // Mark the user variable as used so that it isn't stripped out.
2062 CompilerUsedVars.push_back(x: Var);
2063 } else {
2064 // Make a function that uses it.
2065 auto *User = Function::Create(Ty: FunctionType::get(Result: Int32Ty, isVarArg: false),
2066 Linkage: GlobalValue::LinkOnceODRLinkage,
2067 N: getInstrProfRuntimeHookVarUseFuncName(), M);
2068 User->addFnAttr(Kind: Attribute::NoInline);
2069 if (Options.NoRedZone)
2070 User->addFnAttr(Kind: Attribute::NoRedZone);
2071 User->setVisibility(GlobalValue::HiddenVisibility);
2072 if (TT.supportsCOMDAT())
2073 User->setComdat(M.getOrInsertComdat(Name: User->getName()));
2074 // Explicitly mark this function as cold since it is never called.
2075 User->setEntryCount(Count: 0);
2076
2077 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: User));
2078 auto *Load = IRB.CreateLoad(Ty: Int32Ty, Ptr: Var);
2079 IRB.CreateRet(V: Load);
2080
2081 // Mark the function as used so that it isn't stripped out.
2082 CompilerUsedVars.push_back(x: User);
2083 }
2084 return true;
2085}
2086
2087void InstrLowerer::emitUses() {
2088 // The metadata sections are parallel arrays. Optimizers (e.g.
2089 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
2090 // we conservatively retain all unconditionally in the compiler.
2091 //
2092 // On ELF and Mach-O, the linker can guarantee the associated sections will be
2093 // retained or discarded as a unit, so llvm.compiler.used is sufficient.
2094 // Similarly on COFF, if prof data is not referenced by code we use one comdat
2095 // and ensure this GC property as well. Otherwise, we have to conservatively
2096 // make all of the sections retained by the linker.
2097 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
2098 (TT.isOSBinFormatCOFF() && !DataReferencedByCode))
2099 appendToCompilerUsed(M, Values: CompilerUsedVars);
2100 else
2101 appendToUsed(M, Values: CompilerUsedVars);
2102
2103 // We do not add proper references from used metadata sections to NamesVar and
2104 // VNodesVar, so we have to be conservative and place them in llvm.used
2105 // regardless of the target,
2106 appendToUsed(M, Values: UsedVars);
2107}
2108
2109void InstrLowerer::emitInitialization() {
2110 // Create ProfileFileName variable. Don't don't this for the
2111 // context-sensitive instrumentation lowering: This lowering is after
2112 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
2113 // have already create the variable before LTO/ThinLTO linking.
2114 if (!IsCS)
2115 createProfileFileNameVar(M, InstrProfileOutput: Options.InstrProfileOutput);
2116 Function *RegisterF = M.getFunction(Name: getInstrProfRegFuncsName());
2117 if (!RegisterF)
2118 return;
2119
2120 // Create the initialization function.
2121 auto *VoidTy = Type::getVoidTy(C&: M.getContext());
2122 auto *F = Function::Create(Ty: FunctionType::get(Result: VoidTy, isVarArg: false),
2123 Linkage: GlobalValue::InternalLinkage,
2124 N: getInstrProfInitFuncName(), M);
2125 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2126 F->addFnAttr(Kind: Attribute::NoInline);
2127 if (Options.NoRedZone)
2128 F->addFnAttr(Kind: Attribute::NoRedZone);
2129
2130 // Add the basic block and the necessary calls.
2131 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: F));
2132 IRB.CreateCall(Callee: RegisterF, Args: {});
2133 IRB.CreateRetVoid();
2134
2135 appendToGlobalCtors(M, F, Priority: 0);
2136}
2137
2138namespace llvm {
2139// Create the variable for profile sampling.
2140void createProfileSamplingVar(Module &M) {
2141 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
2142 IntegerType *SamplingVarTy;
2143 Constant *ValueZero;
2144 if (getSampledInstrumentationConfig().UseShort) {
2145 SamplingVarTy = Type::getInt16Ty(C&: M.getContext());
2146 ValueZero = Constant::getIntegerValue(Ty: SamplingVarTy, V: APInt(16, 0));
2147 } else {
2148 SamplingVarTy = Type::getInt32Ty(C&: M.getContext());
2149 ValueZero = Constant::getIntegerValue(Ty: SamplingVarTy, V: APInt(32, 0));
2150 }
2151 auto SamplingVar = new GlobalVariable(
2152 M, SamplingVarTy, false, GlobalValue::WeakAnyLinkage, ValueZero, VarName);
2153 SamplingVar->setVisibility(GlobalValue::DefaultVisibility);
2154 SamplingVar->setThreadLocal(true);
2155 Triple TT(M.getTargetTriple());
2156 if (TT.supportsCOMDAT()) {
2157 SamplingVar->setLinkage(GlobalValue::ExternalLinkage);
2158 SamplingVar->setComdat(M.getOrInsertComdat(Name: VarName));
2159 }
2160 appendToCompilerUsed(M, Values: SamplingVar);
2161}
2162} // namespace llvm
2163