1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by an instrumentor.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/Twine.h"
21#include "llvm/Analysis/BlockFrequencyInfo.h"
22#include "llvm/Analysis/BranchProbabilityInfo.h"
23#include "llvm/Analysis/CFG.h"
24#include "llvm/Analysis/LoopInfo.h"
25#include "llvm/Analysis/TargetLibraryInfo.h"
26#include "llvm/IR/Attributes.h"
27#include "llvm/IR/BasicBlock.h"
28#include "llvm/IR/CFG.h"
29#include "llvm/IR/Constant.h"
30#include "llvm/IR/Constants.h"
31#include "llvm/IR/CycleInfo.h"
32#include "llvm/IR/DIBuilder.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/DiagnosticInfo.h"
35#include "llvm/IR/Function.h"
36#include "llvm/IR/GlobalAlias.h"
37#include "llvm/IR/GlobalValue.h"
38#include "llvm/IR/GlobalVariable.h"
39#include "llvm/IR/IRBuilder.h"
40#include "llvm/IR/InstIterator.h"
41#include "llvm/IR/Instruction.h"
42#include "llvm/IR/Instructions.h"
43#include "llvm/IR/IntrinsicInst.h"
44#include "llvm/IR/Intrinsics.h"
45#include "llvm/IR/MDBuilder.h"
46#include "llvm/IR/Module.h"
47#include "llvm/IR/RuntimeLibcalls.h"
48#include "llvm/IR/Type.h"
49#include "llvm/Pass.h"
50#include "llvm/ProfileData/InstrProf.h"
51#include "llvm/ProfileData/InstrProfCorrelator.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CommandLine.h"
54#include "llvm/Support/Compiler.h"
55#include "llvm/Support/Error.h"
56#include "llvm/Support/ErrorHandling.h"
57#include "llvm/TargetParser/Triple.h"
58#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
59#include "llvm/Transforms/Utils/BasicBlockUtils.h"
60#include "llvm/Transforms/Utils/Instrumentation.h"
61#include "llvm/Transforms/Utils/ModuleUtils.h"
62#include "llvm/Transforms/Utils/SSAUpdater.h"
63#include <algorithm>
64#include <cassert>
65#include <cstdint>
66#include <string>
67
68using namespace llvm;
69
70#define DEBUG_TYPE "instrprof"
71
72namespace llvm {
73// Command line option to enable vtable value profiling. Defined in
74// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
75extern cl::opt<bool> EnableVTableValueProfiling;
76LLVM_ABI cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate(
77 "profile-correlate",
78 cl::desc("Use debug info or binary file to correlate profiles."),
79 cl::init(Val: InstrProfCorrelator::NONE),
80 cl::values(clEnumValN(InstrProfCorrelator::NONE, "",
81 "No profile correlation"),
82 clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info",
83 "Use debug info to correlate"),
84 clEnumValN(InstrProfCorrelator::BINARY, "binary",
85 "Use binary to correlate")));
86} // namespace llvm
87
88namespace {
89
90cl::opt<bool> DoHashBasedCounterSplit(
91 "hash-based-counter-split",
92 cl::desc("Rename counter variable of a comdat function based on cfg hash"),
93 cl::init(Val: true));
94
95cl::opt<bool>
96 RuntimeCounterRelocation("runtime-counter-relocation",
97 cl::desc("Enable relocating counters at runtime."),
98 cl::init(Val: false));
99
100cl::opt<bool> ValueProfileStaticAlloc(
101 "vp-static-alloc",
102 cl::desc("Do static counter allocation for value profiler"),
103 cl::init(Val: true));
104
105cl::opt<double> NumCountersPerValueSite(
106 "vp-counters-per-site",
107 cl::desc("The average number of profile counters allocated "
108 "per value profiling site."),
109 // This is set to a very small value because in real programs, only
110 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
111 // For those sites with non-zero profile, the average number of targets
112 // is usually smaller than 2.
113 cl::init(Val: 1.0));
114
115cl::opt<bool> AtomicCounterUpdateAll(
116 "instrprof-atomic-counter-update-all",
117 cl::desc("Make all profile counter updates atomic (for testing only)"),
118 cl::init(Val: false));
119
120cl::opt<bool> VerifyAtomicPromotion(
121 "verify-atomic-counter-promoted",
122 cl::desc("Check that all profile counter updates were made atomic; no-op "
123 "if atomic updates are not requested (-fprofile-update=atomic)"),
124 cl::init(Val: false));
125
126cl::opt<bool> AtomicCounterUpdatePromoted(
127 "atomic-counter-update-promoted",
128 cl::desc("Do counter update using atomic fetch add "
129 " for promoted counters only"),
130 cl::init(Val: false));
131
132cl::opt<bool> AtomicFirstCounter(
133 "atomic-first-counter",
134 cl::desc("Use atomic fetch add for first counter in a function (usually "
135 "the entry counter)"),
136 cl::init(Val: false));
137
138cl::opt<bool> ConditionalCounterUpdate(
139 "conditional-counter-update",
140 cl::desc("Do conditional counter updates in single byte counters mode)"),
141 cl::init(Val: false));
142
143// If the option is not specified, the default behavior about whether
144// counter promotion is done depends on how instrumentation lowering
145// pipeline is setup, i.e., the default value of true of this option
146// does not mean the promotion will be done by default. Explicitly
147// setting this option can override the default behavior.
148cl::opt<bool> DoCounterPromotion("do-counter-promotion",
149 cl::desc("Do counter register promotion"),
150 cl::init(Val: false));
151cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
152 "max-counter-promotions-per-loop", cl::init(Val: 20),
153 cl::desc("Max number counter promotions per loop to avoid"
154 " increasing register pressure too much"));
155
156// A debug option
157cl::opt<int>
158 MaxNumOfPromotions("max-counter-promotions", cl::init(Val: -1),
159 cl::desc("Max number of allowed counter promotions"));
160
161cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
162 "speculative-counter-promotion-max-exiting", cl::init(Val: 3),
163 cl::desc("The max number of exiting blocks of a loop to allow "
164 " speculative counter promotion"));
165
166cl::opt<bool> SpeculativeCounterPromotionToLoop(
167 "speculative-counter-promotion-to-loop",
168 cl::desc("When the option is false, if the target block is in a loop, "
169 "the promotion will be disallowed unless the promoted counter "
170 " update can be further/iteratively promoted into an acyclic "
171 " region."));
172
173static cl::opt<unsigned> OffloadPGOSampling(
174 "offload-pgo-sampling",
175 cl::desc("Log2 of the sampling period for offload PGO instrumentation. "
176 "Only 1 in every 2^N blocks is instrumented. "
177 "0 = all blocks, 1 = 50%, 2 = 25%, 3 = 12.5% (default). "
178 "Higher values reduce overhead at the cost of sparser profiles."),
179 cl::init(Val: 3));
180
181cl::opt<bool> IterativeCounterPromotion(
182 "iterative-counter-promotion", cl::init(Val: true),
183 cl::desc("Allow counter promotion across the whole loop nest."));
184
185cl::opt<bool> SkipRetExitBlock(
186 "skip-ret-exit-block", cl::init(Val: true),
187 cl::desc("Suppress counter promotion if exit blocks contain ret."));
188
189static cl::opt<bool> SampledInstr("sampled-instrumentation",
190 cl::desc("Do PGO instrumentation sampling"));
191
192static cl::opt<unsigned> SampledInstrPeriod(
193 "sampled-instr-period",
194 cl::desc("Set the profile instrumentation sample period. A sample period "
195 "of 0 is invalid. For each sample period, a fixed number of "
196 "consecutive samples will be recorded. The number is controlled "
197 "by 'sampled-instr-burst-duration' flag. The default sample "
198 "period of 65536 is optimized for generating efficient code that "
199 "leverages unsigned short integer wrapping in overflow, but this "
200 "is disabled under simple sampling (burst duration = 1)."),
201 cl::init(USHRT_MAX + 1));
202
203static cl::opt<unsigned> SampledInstrBurstDuration(
204 "sampled-instr-burst-duration",
205 cl::desc("Set the profile instrumentation burst duration, which can range "
206 "from 1 to the value of 'sampled-instr-period' (0 is invalid). "
207 "This number of samples will be recorded for each "
208 "'sampled-instr-period' count update. Setting to 1 enables simple "
209 "sampling, in which case it is recommended to set "
210 "'sampled-instr-period' to a prime number."),
211 cl::init(Val: 200));
212
213struct SampledInstrumentationConfig {
214 unsigned BurstDuration;
215 unsigned Period;
216 bool UseShort;
217 bool IsSimpleSampling;
218 bool IsFastSampling;
219};
220
221static SampledInstrumentationConfig getSampledInstrumentationConfig() {
222 SampledInstrumentationConfig config;
223 config.BurstDuration = SampledInstrBurstDuration.getValue();
224 config.Period = SampledInstrPeriod.getValue();
225 if (config.BurstDuration > config.Period)
226 report_fatal_error(
227 reason: "SampledBurstDuration must be less than or equal to SampledPeriod");
228 if (config.Period == 0 || config.BurstDuration == 0)
229 report_fatal_error(
230 reason: "SampledPeriod and SampledBurstDuration must be greater than 0");
231 config.IsSimpleSampling = (config.BurstDuration == 1);
232 // If (BurstDuration == 1 && Period == 65536), generate the simple sampling
233 // style code.
234 config.IsFastSampling =
235 (!config.IsSimpleSampling && config.Period == USHRT_MAX + 1);
236 config.UseShort = (config.Period <= USHRT_MAX) || config.IsFastSampling;
237 return config;
238}
239
240using LoadStorePair = std::pair<Instruction *, Instruction *>;
241
242static void makeAtomic(Instruction *Load, Instruction *Store) {
243 auto *Addition = dyn_cast<BinaryOperator>(Val: Store->getOperand(i: 0));
244 assert(Addition && Addition->getOpcode() == Instruction::BinaryOps::Add);
245 auto *Addend = Addition->getOperand(i_nocapture: 1);
246
247 IRBuilder<> Builder(Load);
248 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Store->getOperand(i: 1), Val: Addend,
249 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
250 Store->eraseFromParent();
251 Addition->eraseFromParent();
252 Load->eraseFromParent();
253}
254
255static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {
256 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(Val: M.getModuleFlag(Key: Flag));
257 if (!MD)
258 return 0;
259
260 // If the flag is a ConstantAsMetadata, it should be an integer representable
261 // in 64-bits.
262 return cast<ConstantInt>(Val: MD->getValue())->getZExtValue();
263}
264
265static bool enablesValueProfiling(const Module &M) {
266 return isIRPGOFlagSet(M: &M) ||
267 getIntModuleFlagOrZero(M, Flag: "EnableValueProfiling") != 0;
268}
269
270// Conservatively returns true if value profiling is enabled.
271static bool profDataReferencedByCode(const Module &M) {
272 return enablesValueProfiling(M);
273}
274
275class InstrLowerer final {
276public:
277 InstrLowerer(Module &M, const InstrProfOptions &Options,
278 std::function<const TargetLibraryInfo &(Function &F)> GetTLI,
279 bool IsCS)
280 : M(M), Options(Options), TT(M.getTargetTriple()), IsCS(IsCS),
281 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {}
282
283 bool lower();
284
285private:
286 Module &M;
287 const InstrProfOptions Options;
288 const Triple TT;
289 // Is this lowering for the context-sensitive instrumentation.
290 const bool IsCS;
291
292 std::function<const TargetLibraryInfo &(Function &F)> GetTLI;
293
294 const bool DataReferencedByCode;
295
296 struct PerFunctionProfileData {
297 uint32_t NumValueSites[IPVK_Last + 1] = {};
298 GlobalVariable *RegionCounters = nullptr;
299 GlobalVariable *UniformCounters =
300 nullptr; // Per-block uniform-entry counters
301 GlobalVariable *DataVar = nullptr;
302 GlobalVariable *RegionBitmaps = nullptr;
303 uint32_t NumBitmapBytes = 0;
304
305 PerFunctionProfileData() = default;
306 };
307 DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
308 // Key is virtual table variable, value is 'VTableProfData' in the form of
309 // GlobalVariable.
310 DenseMap<GlobalVariable *, GlobalVariable *> VTableDataMap;
311 /// If runtime relocation is enabled, this maps functions to the load
312 /// instruction that produces the profile relocation bias.
313 DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;
314 std::vector<GlobalValue *> CompilerUsedVars;
315 std::vector<GlobalValue *> UsedVars;
316 std::vector<GlobalVariable *> ReferencedNames;
317 // The list of virtual table variables of which the VTableProfData is
318 // collected.
319 std::vector<GlobalVariable *> ReferencedVTables;
320 GlobalVariable *NamesVar = nullptr;
321 size_t NamesSize = 0;
322
323 StructType *ProfileDataTy = nullptr;
324
325 // vector of counter load/store pairs to be register promoted.
326 std::vector<LoadStorePair> PromotionCandidates;
327
328 int64_t TotalCountersPromoted = 0;
329
330 // Per-function cache of invariant values for GPU PGO instrumentation.
331 // Computed once at the function entry and reused across all instrumentation
332 // points to avoid redundant IR and help the optimizer.
333 struct GPUPGOInvariants {
334 Value *Matched = nullptr;
335 bool WaveSizeStored = false;
336 };
337 DenseMap<Function *, GPUPGOInvariants> GPUInvariantsCache;
338
339 /// Emit invariant PGO values at the function entry block and cache them.
340 GPUPGOInvariants &getOrCreateGPUInvariants(Function *F);
341
342 /// Lower instrumentation intrinsics in the function. Returns true if there
343 /// any lowering.
344 bool lowerIntrinsics(Function *F);
345
346 /// Register-promote counter loads and stores in loops.
347 void promoteCounterLoadStores(Function *F);
348
349 /// Returns true if relocating counters at runtime is enabled.
350 bool isRuntimeCounterRelocationEnabled() const;
351
352 /// Returns true if profile counter update register promotion is enabled.
353 bool isCounterPromotionEnabled() const;
354
355 /// Returns true if profile counter updates should be atomic.
356 bool isAtomic() const;
357
358 /// Return true if profile sampling is enabled.
359 bool isSamplingEnabled() const;
360
361 /// Count the number of instrumented value sites for the function.
362 void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
363
364 /// Replace instrprof.value.profile with a call to runtime library.
365 void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
366
367 /// Replace instrprof.cover with a store instruction to the coverage byte.
368 void lowerCover(InstrProfCoverInst *Inc);
369
370 /// Replace instrprof.timestamp with a call to
371 /// INSTR_PROF_PROFILE_SET_TIMESTAMP.
372 void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction);
373
374 /// Replace instrprof.increment with an increment of the appropriate value.
375 void lowerIncrement(InstrProfIncrementInst *Inc);
376
377 /// Force emitting of name vars for unused functions.
378 void lowerCoverageData(GlobalVariable *CoverageNamesVar);
379
380 /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction
381 /// using the index represented by the a temp value into a bitmap.
382 void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins);
383
384 /// Get the Bias value for data to access mmap-ed area.
385 /// Create it if it hasn't been seen.
386 GlobalVariable *getOrCreateBiasVar(StringRef VarName);
387
388 /// Compute the address of the counter value that this profiling instruction
389 /// acts on.
390 Value *getCounterAddress(InstrProfCntrInstBase *I);
391
392 /// Lower the incremental instructions under profile sampling predicates.
393 void doSampling(Instruction *I);
394
395 /// Get the region counters for an increment, creating them if necessary.
396 ///
397 /// If the counter array doesn't yet exist, the profile data variables
398 /// referring to them will also be created.
399 GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc);
400
401 /// Get the uniform entry counters for GPU divergence tracking.
402 /// These counters track how often blocks are entered with all lanes active.
403 GlobalVariable *getOrCreateUniformCounters(InstrProfCntrInstBase *Inc);
404
405 /// Create the region counters.
406 GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc,
407 StringRef Name,
408 GlobalValue::LinkageTypes Linkage);
409
410 /// Compute the address of the test vector bitmap that this profiling
411 /// instruction acts on.
412 Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I);
413
414 /// Get the region bitmaps for an increment, creating them if necessary.
415 ///
416 /// If the bitmap array doesn't yet exist, the profile data variables
417 /// referring to them will also be created.
418 GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc);
419
420 /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with
421 /// an MC/DC Decision region. The number of bytes required is indicated by
422 /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called
423 /// as part of setupProfileSection() and is conceptually very similar to
424 /// what is done for profile data counters in createRegionCounters().
425 GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
426 StringRef Name,
427 GlobalValue::LinkageTypes Linkage);
428
429 /// Set Comdat property of GV, if required.
430 void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName);
431
432 /// Setup the sections into which counters and bitmaps are allocated.
433 GlobalVariable *setupProfileSection(InstrProfInstBase *Inc,
434 InstrProfSectKind IPSK);
435
436 /// Create INSTR_PROF_DATA variable for counters and bitmaps.
437 void createDataVariable(InstrProfCntrInstBase *Inc);
438
439 /// Get the counters for virtual table values, creating them if necessary.
440 void getOrCreateVTableProfData(GlobalVariable *GV);
441
442 /// Emit the section with compressed function names.
443 void emitNameData();
444
445 /// Emit the section with compressed vtable names.
446 void emitVTableNames();
447
448 /// Emit value nodes section for value profiling.
449 void emitVNodes();
450
451 /// Emit runtime registration functions for each profile data variable.
452 void emitRegistration();
453
454 /// Emit the necessary plumbing to pull in the runtime initialization.
455 /// Returns true if a change was made.
456 bool emitRuntimeHook();
457
458 /// Add uses of our data variables and runtime hook.
459 void emitUses();
460
461 /// Create a static initializer for our data, on platforms that need it,
462 /// and for any profile output file that was specified.
463 void emitInitialization();
464
465 /// Return the __llvm_profile_data struct type.
466 StructType *getProfileDataTy();
467};
468
469///
470/// A helper class to promote one counter RMW operation in the loop
471/// into register update.
472///
473/// RWM update for the counter will be sinked out of the loop after
474/// the transformation.
475///
476class PGOCounterPromoterHelper : public LoadAndStorePromoter {
477public:
478 PGOCounterPromoterHelper(
479 Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,
480 BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
481 ArrayRef<Instruction *> InsertPts,
482 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
483 LoopInfo &LI, bool IsAtomic)
484 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
485 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI),
486 IsAtomic(IsAtomic) {
487 assert(isa<LoadInst>(L));
488 assert(isa<StoreInst>(S));
489 SSA.AddAvailableValue(BB: PH, V: Init);
490 }
491
492 void doExtraRewritesBeforeFinalDeletion() override {
493 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
494 BasicBlock *ExitBlock = ExitBlocks[i];
495 Instruction *InsertPos = InsertPts[i];
496 // Get LiveIn value into the ExitBlock. If there are multiple
497 // predecessors, the value is defined by a PHI node in this
498 // block.
499 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(BB: ExitBlock);
500 Value *Addr = cast<StoreInst>(Val: Store)->getPointerOperand();
501 Type *Ty = LiveInValue->getType();
502 IRBuilder<> Builder(InsertPos);
503 if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Val: Addr)) {
504 // If isRuntimeCounterRelocationEnabled() is true then the address of
505 // the store instruction is computed with two instructions in
506 // InstrProfiling::getCounterAddress(). We need to copy those
507 // instructions to this block to compute Addr correctly.
508 // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
509 // %Addr = inttoptr i64 %BiasAdd to i64*
510 auto *OrigBiasInst = dyn_cast<BinaryOperator>(Val: AddrInst->getOperand(i_nocapture: 0));
511 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
512 Value *BiasInst = Builder.Insert(I: OrigBiasInst->clone());
513 Addr = Builder.CreateIntToPtr(V: BiasInst,
514 DestTy: PointerType::getUnqual(C&: Ty->getContext()));
515 }
516 auto *TargetLoop =
517 IterativeCounterPromotion ? LI.getLoopFor(BB: ExitBlock) : nullptr;
518 // Generate the relaxed atomic RMW if we've asked for it and no more
519 // promotion is possible.
520 if ((IsAtomic && !TargetLoop) || AtomicCounterUpdatePromoted)
521 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: LiveInValue,
522 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
523 else {
524 LoadInst *OldVal = Builder.CreateLoad(Ty, Ptr: Addr, Name: "pgocount.promoted");
525 auto *NewVal = Builder.CreateAdd(LHS: OldVal, RHS: LiveInValue);
526 auto *NewStore = Builder.CreateStore(Val: NewVal, Ptr: Addr);
527
528 // Now update the parent loop's candidate list:
529 if (TargetLoop)
530 LoopToCandidates[TargetLoop].emplace_back(Args&: OldVal, Args&: NewStore);
531 }
532 }
533 }
534
535private:
536 Instruction *Store;
537 ArrayRef<BasicBlock *> ExitBlocks;
538 ArrayRef<Instruction *> InsertPts;
539 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
540 LoopInfo &LI;
541 const bool IsAtomic;
542};
543
544/// A helper class to do register promotion for all profile counter
545/// updates in a loop.
546///
547class PGOCounterPromoter {
548public:
549 PGOCounterPromoter(
550 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
551 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI, bool IsAtomic)
552 : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI),
553 IsAtomic(IsAtomic) {
554
555 // Skip collection of ExitBlocks and InsertPts for loops that will not be
556 // able to have counters promoted.
557 SmallVector<BasicBlock *, 8> LoopExitBlocks;
558 SmallPtrSet<BasicBlock *, 8> BlockSet;
559
560 L.getExitBlocks(ExitBlocks&: LoopExitBlocks);
561 if (!isPromotionPossible(LP: &L, LoopExitBlocks))
562 return;
563
564 for (BasicBlock *ExitBlock : LoopExitBlocks) {
565 if (BlockSet.insert(Ptr: ExitBlock).second &&
566 llvm::none_of(Range: predecessors(BB: ExitBlock), P: [&](const BasicBlock *Pred) {
567 return llvm::isPresplitCoroSuspendExitEdge(Src: *Pred, Dest: *ExitBlock);
568 })) {
569 ExitBlocks.push_back(Elt: ExitBlock);
570 InsertPts.push_back(Elt: &*ExitBlock->getFirstInsertionPt());
571 }
572 }
573 }
574
575 bool run(int64_t *NumPromoted) {
576 bool RC = promoteCandidates(NumPromoted);
577 // In certain case, e.g. with -fprofile-update=atomic, we want to generate
578 // atomic updates of the PGO counters, but also perform promotion of these
579 // updates out of loops to reduce train time. The strategy is:
580 // 1) generate non-atomic load-increment-store sequence of instructions
581 // during lowerIntrinsics phase,
582 // 2) perform the promotion (in promoteCandidates function), then
583 // 3) convert all (promoted and unpromotable) updates to atomicRMW.
584 // This requires that promoted candidates are set to nullptr in the
585 // LoopToCandidates[&L] array by the promoteCandidates() function.
586 if (IsAtomic)
587 for (auto &Cand : LoopToCandidates[&L])
588 if (Cand.first != nullptr && Cand.second != nullptr)
589 makeAtomic(Load: Cand.first, Store: Cand.second);
590 return RC;
591 }
592
593private:
594 bool promoteCandidates(int64_t *NumPromoted) {
595 // Skip 'infinite' loops:
596 if (ExitBlocks.size() == 0)
597 return false;
598
599 // Skip if any of the ExitBlocks contains a ret instruction.
600 // This is to prevent dumping of incomplete profile -- if the
601 // the loop is a long running loop and dump is called in the middle
602 // of the loop, the result profile is incomplete.
603 // FIXME: add other heuristics to detect long running loops.
604 if (SkipRetExitBlock) {
605 for (auto *BB : ExitBlocks)
606 if (isa<ReturnInst>(Val: BB->getTerminator()))
607 return false;
608 }
609
610 unsigned MaxProm = getMaxNumOfPromotionsInLoop(LP: &L);
611 if (MaxProm == 0)
612 return false;
613
614 [[maybe_unused]] auto *Ptr = LoopToCandidates.getPointerIntoBucketsArray();
615 unsigned Promoted = 0;
616 for (auto &Cand : LoopToCandidates[&L]) {
617 SmallVector<PHINode *, 4> NewPHIs;
618 SSAUpdater SSA(&NewPHIs);
619 Value *InitVal = ConstantInt::get(Ty: Cand.first->getType(), V: 0);
620
621 // If BFI is set, we will use it to guide the promotions.
622 if (BFI) {
623 auto *BB = Cand.first->getParent();
624 auto InstrCount = BFI->getBlockProfileCount(BB);
625 if (!InstrCount)
626 continue;
627 auto PreheaderCount = BFI->getBlockProfileCount(BB: L.getLoopPreheader());
628 // If the average loop trip count is not greater than 1.5, we skip
629 // promotion.
630 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
631 continue;
632 }
633
634 PGOCounterPromoterHelper Promoter(
635 Cand.first, Cand.second, SSA, InitVal, L.getLoopPreheader(),
636 ExitBlocks, InsertPts, LoopToCandidates, LI, IsAtomic);
637 Promoter.run(Insts: SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
638
639 assert(LoopToCandidates.isPointerIntoBucketsArray(Ptr) &&
640 "References into LoopToCandidates might be invalid");
641 Cand = {nullptr, nullptr};
642
643 Promoted++;
644 if (Promoted >= MaxProm)
645 break;
646
647 (*NumPromoted)++;
648 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
649 break;
650 }
651
652 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
653 << L.getLoopDepth() << ")\n");
654 return Promoted != 0;
655 }
656
657private:
658 bool allowSpeculativeCounterPromotion(Loop *LP) {
659 SmallVector<BasicBlock *, 8> ExitingBlocks;
660 L.getExitingBlocks(ExitingBlocks);
661 // Not considierered speculative.
662 if (ExitingBlocks.size() == 1)
663 return true;
664 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
665 return false;
666 return true;
667 }
668
669 // Check whether the loop satisfies the basic conditions needed to perform
670 // Counter Promotions.
671 bool
672 isPromotionPossible(Loop *LP,
673 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
674 // We can't insert into a catchswitch.
675 if (llvm::any_of(Range: LoopExitBlocks, P: [](BasicBlock *Exit) {
676 return isa<CatchSwitchInst>(Val: Exit->getTerminator());
677 }))
678 return false;
679
680 if (!LP->hasDedicatedExits())
681 return false;
682
683 BasicBlock *PH = LP->getLoopPreheader();
684 if (!PH)
685 return false;
686
687 return true;
688 }
689
690 // Returns the max number of Counter Promotions for LP.
691 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
692 SmallVector<BasicBlock *, 8> LoopExitBlocks;
693 LP->getExitBlocks(ExitBlocks&: LoopExitBlocks);
694 if (!isPromotionPossible(LP, LoopExitBlocks))
695 return 0;
696
697 SmallVector<BasicBlock *, 8> ExitingBlocks;
698 LP->getExitingBlocks(ExitingBlocks);
699
700 // If BFI is set, we do more aggressive promotions based on BFI.
701 if (BFI)
702 return (unsigned)-1;
703
704 // Not considierered speculative.
705 if (ExitingBlocks.size() == 1)
706 return MaxNumOfPromotionsPerLoop;
707
708 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
709 return 0;
710
711 // Whether the target block is in a loop does not matter:
712 if (SpeculativeCounterPromotionToLoop)
713 return MaxNumOfPromotionsPerLoop;
714
715 // Now check the target block:
716 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
717 for (auto *TargetBlock : LoopExitBlocks) {
718 auto *TargetLoop = LI.getLoopFor(BB: TargetBlock);
719 if (!TargetLoop)
720 continue;
721 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(LP: TargetLoop);
722 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
723 MaxProm =
724 std::min(a: MaxProm, b: std::max(a: MaxPromForTarget, b: PendingCandsInTarget) -
725 PendingCandsInTarget);
726 }
727 return MaxProm;
728 }
729
730 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
731 SmallVector<BasicBlock *, 8> ExitBlocks;
732 SmallVector<Instruction *, 8> InsertPts;
733 Loop &L;
734 LoopInfo &LI;
735 BlockFrequencyInfo *BFI;
736 const bool IsAtomic; // Whether to convert counter updates to atomics.
737};
738
739enum class ValueProfilingCallType {
740 // Individual values are tracked. Currently used for indiret call target
741 // profiling.
742 Default,
743
744 // MemOp: the memop size value profiling.
745 MemOp
746};
747
748} // end anonymous namespace
749
750PreservedAnalyses InstrProfilingLoweringPass::run(Module &M,
751 ModuleAnalysisManager &AM) {
752 FunctionAnalysisManager &FAM =
753 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
754 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
755 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
756 };
757 InstrLowerer Lowerer(M, Options, GetTLI, IsCS);
758 if (!Lowerer.lower())
759 return PreservedAnalyses::all();
760
761 return PreservedAnalyses::none();
762}
763
764//
765// Perform instrumentation sampling.
766//
767// There are 3 favors of sampling:
768// (1) Full burst sampling: We transform:
769// Increment_Instruction;
770// to:
771// if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
772// Increment_Instruction;
773// }
774// __llvm_profile_sampling__ += 1;
775// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
776// __llvm_profile_sampling__ = 0;
777// }
778//
779// "__llvm_profile_sampling__" is a thread-local global shared by all PGO
780// counters (value-instrumentation and edge instrumentation).
781//
782// (2) Fast burst sampling:
783// "__llvm_profile_sampling__" variable is an unsigned type, meaning it will
784// wrap around to zero when overflows. In this case, the second check is
785// unnecessary, so we won't generate check2 when the SampledInstrPeriod is
786// set to 65536 (64K). The code after:
787// if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
788// Increment_Instruction;
789// }
790// __llvm_profile_sampling__ += 1;
791//
792// (3) Simple sampling:
793// When SampledInstrBurstDuration is set to 1, we do a simple sampling:
794// __llvm_profile_sampling__ += 1;
795// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
796// __llvm_profile_sampling__ = 0;
797// Increment_Instruction;
798// }
799//
800// Note that, the code snippet after the transformation can still be counter
801// promoted. However, with sampling enabled, counter updates are expected to
802// be infrequent, making the benefits of counter promotion negligible.
803// Moreover, counter promotion can potentially cause issues in server
804// applications, particularly when the counters are dumped without a clean
805// exit. To mitigate this risk, counter promotion is disabled by default when
806// sampling is enabled. This behavior can be overridden using the internal
807// option.
808void InstrLowerer::doSampling(Instruction *I) {
809 if (!isSamplingEnabled())
810 return;
811
812 SampledInstrumentationConfig config = getSampledInstrumentationConfig();
813 auto GetConstant = [&config](IRBuilder<> &Builder, uint32_t C) {
814 if (config.UseShort)
815 return Builder.getInt16(C);
816 else
817 return Builder.getInt32(C);
818 };
819
820 IntegerType *SamplingVarTy;
821 if (config.UseShort)
822 SamplingVarTy = Type::getInt16Ty(C&: M.getContext());
823 else
824 SamplingVarTy = Type::getInt32Ty(C&: M.getContext());
825 auto *SamplingVar =
826 M.getGlobalVariable(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
827 assert(SamplingVar && "SamplingVar not set properly");
828
829 // Create the condition for checking the burst duration.
830 Instruction *SamplingVarIncr;
831 Value *NewSamplingVarVal;
832 MDBuilder MDB(I->getContext());
833 MDNode *BranchWeight;
834 IRBuilder<> CondBuilder(I);
835 auto *LoadSamplingVar = CondBuilder.CreateLoad(Ty: SamplingVarTy, Ptr: SamplingVar);
836 if (config.IsSimpleSampling) {
837 // For the simple sampling, just create the load and increments.
838 IRBuilder<> IncBuilder(I);
839 NewSamplingVarVal =
840 IncBuilder.CreateAdd(LHS: LoadSamplingVar, RHS: GetConstant(IncBuilder, 1));
841 SamplingVarIncr = IncBuilder.CreateStore(Val: NewSamplingVarVal, Ptr: SamplingVar);
842 } else {
843 // For the burst-sampling, create the conditional update.
844 auto *DurationCond = CondBuilder.CreateICmpULE(
845 LHS: LoadSamplingVar, RHS: GetConstant(CondBuilder, config.BurstDuration - 1));
846 BranchWeight = MDB.createBranchWeights(
847 TrueWeight: config.BurstDuration, FalseWeight: config.Period - config.BurstDuration);
848 Instruction *ThenTerm = SplitBlockAndInsertIfThen(
849 Cond: DurationCond, SplitBefore: I, /* Unreachable */ false, BranchWeights: BranchWeight);
850 IRBuilder<> IncBuilder(I);
851 NewSamplingVarVal =
852 IncBuilder.CreateAdd(LHS: LoadSamplingVar, RHS: GetConstant(IncBuilder, 1));
853 SamplingVarIncr = IncBuilder.CreateStore(Val: NewSamplingVarVal, Ptr: SamplingVar);
854 I->moveBefore(InsertPos: ThenTerm->getIterator());
855 }
856
857 if (config.IsFastSampling)
858 return;
859
860 // Create the condition for checking the period.
861 Instruction *ThenTerm, *ElseTerm;
862 IRBuilder<> PeriodCondBuilder(SamplingVarIncr);
863 auto *PeriodCond = PeriodCondBuilder.CreateICmpUGE(
864 LHS: NewSamplingVarVal, RHS: GetConstant(PeriodCondBuilder, config.Period));
865 BranchWeight = MDB.createBranchWeights(TrueWeight: 1, FalseWeight: config.Period - 1);
866 SplitBlockAndInsertIfThenElse(Cond: PeriodCond, SplitBefore: SamplingVarIncr, ThenTerm: &ThenTerm,
867 ElseTerm: &ElseTerm, BranchWeights: BranchWeight);
868
869 // For the simple sampling, the counter update happens in sampling var reset.
870 if (config.IsSimpleSampling)
871 I->moveBefore(InsertPos: ThenTerm->getIterator());
872
873 IRBuilder<> ResetBuilder(ThenTerm);
874 ResetBuilder.CreateStore(Val: GetConstant(ResetBuilder, 0), Ptr: SamplingVar);
875 SamplingVarIncr->moveBefore(InsertPos: ElseTerm->getIterator());
876}
877
878bool InstrLowerer::lowerIntrinsics(Function *F) {
879 bool MadeChange = false;
880 PromotionCandidates.clear();
881 SmallVector<InstrProfInstBase *, 8> InstrProfInsts;
882
883 // To ensure compatibility with sampling, we save the intrinsics into
884 // a buffer to prevent potential breakage of the iterator (as the
885 // intrinsics will be moved to a different BB).
886 for (BasicBlock &BB : *F) {
887 for (Instruction &Instr : llvm::make_early_inc_range(Range&: BB)) {
888 if (auto *IP = dyn_cast<InstrProfInstBase>(Val: &Instr))
889 InstrProfInsts.push_back(Elt: IP);
890 }
891 }
892
893 for (auto *Instr : InstrProfInsts) {
894 doSampling(I: Instr);
895 if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(Val: Instr)) {
896 lowerIncrement(Inc: IPIS);
897 MadeChange = true;
898 } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(Val: Instr)) {
899 lowerIncrement(Inc: IPI);
900 MadeChange = true;
901 } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(Val: Instr)) {
902 lowerTimestamp(TimestampInstruction: IPC);
903 MadeChange = true;
904 } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(Val: Instr)) {
905 lowerCover(Inc: IPC);
906 MadeChange = true;
907 } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(Val: Instr)) {
908 lowerValueProfileInst(Ins: IPVP);
909 MadeChange = true;
910 } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(Val: Instr)) {
911 IPMP->eraseFromParent();
912 MadeChange = true;
913 } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(Val: Instr)) {
914 lowerMCDCTestVectorBitmapUpdate(Ins: IPBU);
915 MadeChange = true;
916 }
917 }
918
919 if (!MadeChange)
920 return false;
921
922 promoteCounterLoadStores(F);
923 return true;
924}
925
926bool InstrLowerer::isRuntimeCounterRelocationEnabled() const {
927 // Mach-O don't support weak external references.
928 if (TT.isOSBinFormatMachO())
929 return false;
930
931 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
932 return RuntimeCounterRelocation;
933
934 // Fuchsia uses runtime counter relocation by default.
935 return TT.isOSFuchsia();
936}
937
938bool InstrLowerer::isSamplingEnabled() const {
939 if (SampledInstr.getNumOccurrences() > 0)
940 return SampledInstr;
941 return Options.Sampling;
942}
943
944bool InstrLowerer::isCounterPromotionEnabled() const {
945 if (DoCounterPromotion.getNumOccurrences() > 0)
946 return DoCounterPromotion;
947 return Options.DoCounterPromotion;
948}
949
950bool InstrLowerer::isAtomic() const {
951 return Options.Atomic || AtomicCounterUpdateAll;
952}
953
954static void doAtomicCheck(Function *F) {
955 for (const llvm::Instruction &I : llvm::instructions(F)) {
956 const Value *Addr = nullptr;
957 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
958 Addr = LI->getOperand(i_nocapture: 0);
959 else if (const StoreInst *LI = dyn_cast<StoreInst>(Val: &I))
960 Addr = LI->getOperand(i_nocapture: 1);
961
962 if (Addr && Addr->stripInBoundsOffsets()->getName().starts_with(
963 Prefix: getInstrProfCountersVarPrefix())) {
964 LLVM_DEBUG(dbgs() << "Missed candidate: "; I.dump());
965 report_fatal_error(reason: "Candidate load/store not converted to atomic");
966 }
967 }
968}
969
970void InstrLowerer::promoteCounterLoadStores(Function *F) {
971 if (!isCounterPromotionEnabled())
972 return;
973
974 CycleInfo CI;
975 CI.compute(F&: *F);
976 LoopInfo LI;
977 LI.analyze(F);
978 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
979
980 std::unique_ptr<BlockFrequencyInfo> BFI;
981 if (Options.UseBFIInPromotion) {
982 std::unique_ptr<BranchProbabilityInfo> BPI;
983 BPI.reset(p: new BranchProbabilityInfo(*F, CI, &GetTLI(*F)));
984 BFI.reset(p: new BlockFrequencyInfo(*F, *BPI, CI));
985 }
986
987 for (const auto &LoadStore : PromotionCandidates) {
988 auto *CounterLoad = LoadStore.first;
989 auto *CounterStore = LoadStore.second;
990 BasicBlock *BB = CounterLoad->getParent();
991 Loop *ParentLoop = LI.getLoopFor(BB);
992 if (!ParentLoop) {
993 if (isAtomic())
994 makeAtomic(Load: CounterLoad, Store: CounterStore);
995 continue;
996 }
997 LoopPromotionCandidates[ParentLoop].emplace_back(Args&: CounterLoad, Args&: CounterStore);
998 }
999
1000 SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
1001
1002 // Do a post-order traversal of the loops so that counter updates can be
1003 // iteratively hoisted outside the loop nest.
1004 for (auto *Loop : llvm::reverse(C&: Loops)) {
1005 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get(),
1006 isAtomic());
1007 Promoter.run(NumPromoted: &TotalCountersPromoted);
1008 }
1009
1010 if (isAtomic() && VerifyAtomicPromotion)
1011 doAtomicCheck(F);
1012}
1013
1014static bool needsRuntimeHookUnconditionally(const Triple &TT) {
1015 // On Fuchsia, we only need runtime hook if any counters are present.
1016 if (TT.isOSFuchsia())
1017 return false;
1018
1019 return true;
1020}
1021
1022/// Check if the module contains uses of any profiling intrinsics.
1023static bool containsProfilingIntrinsics(Module &M) {
1024 auto containsIntrinsic = [&](int ID) {
1025 if (auto *F = Intrinsic::getDeclarationIfExists(M: &M, id: ID))
1026 return !F->use_empty();
1027 return false;
1028 };
1029 return containsIntrinsic(Intrinsic::instrprof_cover) ||
1030 containsIntrinsic(Intrinsic::instrprof_increment) ||
1031 containsIntrinsic(Intrinsic::instrprof_increment_step) ||
1032 containsIntrinsic(Intrinsic::instrprof_timestamp) ||
1033 containsIntrinsic(Intrinsic::instrprof_value_profile);
1034}
1035
1036bool InstrLowerer::lower() {
1037 bool MadeChange = false;
1038 bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
1039 if (NeedsRuntimeHook)
1040 MadeChange = emitRuntimeHook();
1041
1042 if (!IsCS && isSamplingEnabled())
1043 createProfileSamplingVar(M);
1044
1045 bool ContainsProfiling = containsProfilingIntrinsics(M);
1046 GlobalVariable *CoverageNamesVar =
1047 M.getNamedGlobal(Name: getCoverageUnusedNamesVarName());
1048 // Improve compile time by avoiding linear scans when there is no work.
1049 if (!ContainsProfiling && !CoverageNamesVar)
1050 return MadeChange;
1051
1052 // We did not know how many value sites there would be inside
1053 // the instrumented function. This is counting the number of instrumented
1054 // target value sites to enter it as field in the profile data variable.
1055 for (Function &F : M) {
1056 InstrProfCntrInstBase *FirstProfInst = nullptr;
1057 for (BasicBlock &BB : F) {
1058 for (auto I = BB.begin(), E = BB.end(); I != E; I++) {
1059 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Val&: I))
1060 computeNumValueSiteCounts(Ins: Ind);
1061 else {
1062 if (FirstProfInst == nullptr &&
1063 (isa<InstrProfIncrementInst>(Val: I) || isa<InstrProfCoverInst>(Val: I)))
1064 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(Val&: I);
1065 // If the MCDCBitmapParameters intrinsic seen, create the bitmaps.
1066 if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(Val&: I))
1067 static_cast<void>(getOrCreateRegionBitmaps(Inc: Params));
1068 }
1069 }
1070 }
1071
1072 // Use a profile intrinsic to create the region counters and data variable.
1073 // Also create the data variable based on the MCDCParams.
1074 if (FirstProfInst != nullptr) {
1075 static_cast<void>(getOrCreateRegionCounters(Inc: FirstProfInst));
1076 }
1077 }
1078
1079 if (EnableVTableValueProfiling)
1080 for (GlobalVariable &GV : M.globals())
1081 // Global variables with type metadata are virtual table variables.
1082 if (GV.hasMetadata(KindID: LLVMContext::MD_type))
1083 getOrCreateVTableProfData(GV: &GV);
1084
1085 for (Function &F : M)
1086 MadeChange |= lowerIntrinsics(F: &F);
1087
1088 if (CoverageNamesVar) {
1089 lowerCoverageData(CoverageNamesVar);
1090 MadeChange = true;
1091 }
1092
1093 if (!MadeChange)
1094 return false;
1095
1096 emitVNodes();
1097 emitNameData();
1098 emitVTableNames();
1099
1100 // Emit runtime hook for the cases where the target does not unconditionally
1101 // require pulling in profile runtime, and coverage is enabled on code that is
1102 // not eliminated by the front-end, e.g. unused functions with internal
1103 // linkage.
1104 if (!NeedsRuntimeHook && ContainsProfiling)
1105 emitRuntimeHook();
1106
1107 emitRegistration();
1108 emitUses();
1109 emitInitialization();
1110 return true;
1111}
1112
1113static FunctionCallee getOrInsertValueProfilingCall(
1114 Module &M, const TargetLibraryInfo &TLI,
1115 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
1116 LLVMContext &Ctx = M.getContext();
1117 auto *ReturnTy = Type::getVoidTy(C&: M.getContext());
1118
1119 AttributeList AL;
1120 if (auto AK = TLI.getExtAttrForI32Param(Signed: false))
1121 AL = AL.addParamAttribute(C&: M.getContext(), ArgNo: 2, Kind: AK);
1122
1123 assert((CallType == ValueProfilingCallType::Default ||
1124 CallType == ValueProfilingCallType::MemOp) &&
1125 "Must be Default or MemOp");
1126 Type *ParamTypes[] = {
1127#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
1128#include "llvm/ProfileData/InstrProfData.inc"
1129 };
1130 auto *ValueProfilingCallTy =
1131 FunctionType::get(Result: ReturnTy, Params: ArrayRef(ParamTypes), isVarArg: false);
1132 StringRef FuncName = CallType == ValueProfilingCallType::Default
1133 ? getInstrProfValueProfFuncName()
1134 : getInstrProfValueProfMemOpFuncName();
1135 return M.getOrInsertFunction(Name: FuncName, T: ValueProfilingCallTy, AttributeList: AL);
1136}
1137
1138void InstrLowerer::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
1139 GlobalVariable *Name = Ind->getName();
1140 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1141 uint64_t Index = Ind->getIndex()->getZExtValue();
1142 auto &PD = ProfileDataMap[Name];
1143 PD.NumValueSites[ValueKind] =
1144 std::max(a: PD.NumValueSites[ValueKind], b: (uint32_t)(Index + 1));
1145}
1146
1147void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
1148 // TODO: Value profiling heavily depends on the data section which is omitted
1149 // in lightweight mode. We need to move the value profile pointer to the
1150 // Counter struct to get this working.
1151 assert(
1152 ProfileCorrelate == InstrProfCorrelator::NONE &&
1153 "Value profiling is not yet supported with lightweight instrumentation");
1154 GlobalVariable *Name = Ind->getName();
1155 auto It = ProfileDataMap.find(Val: Name);
1156 assert(It != ProfileDataMap.end() && It->second.DataVar &&
1157 "value profiling detected in function with no counter increment");
1158
1159 GlobalVariable *DataVar = It->second.DataVar;
1160 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1161 uint64_t Index = Ind->getIndex()->getZExtValue();
1162 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
1163 Index += It->second.NumValueSites[Kind];
1164
1165 IRBuilder<> Builder(Ind);
1166 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
1167 llvm::InstrProfValueKind::IPVK_MemOPSize);
1168 CallInst *Call = nullptr;
1169 auto *TLI = &GetTLI(*Ind->getFunction());
1170 auto *NormalizedDataVarPtr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1171 C: DataVar, Ty: PointerType::get(C&: M.getContext(), AddressSpace: 0));
1172
1173 // To support value profiling calls within Windows exception handlers, funclet
1174 // information contained within operand bundles needs to be copied over to
1175 // the library call. This is required for the IR to be processed by the
1176 // WinEHPrepare pass.
1177 SmallVector<OperandBundleDef, 1> OpBundles;
1178 Ind->getOperandBundlesAsDefs(Defs&: OpBundles);
1179 if (!IsMemOpSize) {
1180 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1181 Builder.getInt32(C: Index)};
1182 Call = Builder.CreateCall(Callee: getOrInsertValueProfilingCall(M, TLI: *TLI), Args,
1183 OpBundles);
1184 } else {
1185 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1186 Builder.getInt32(C: Index)};
1187 Call = Builder.CreateCall(
1188 Callee: getOrInsertValueProfilingCall(M, TLI: *TLI, CallType: ValueProfilingCallType::MemOp),
1189 Args, OpBundles);
1190 }
1191 if (auto AK = TLI->getExtAttrForI32Param(Signed: false))
1192 Call->addParamAttr(ArgNo: 2, Kind: AK);
1193 Ind->replaceAllUsesWith(V: Call);
1194 Ind->eraseFromParent();
1195}
1196
1197GlobalVariable *InstrLowerer::getOrCreateBiasVar(StringRef VarName) {
1198 GlobalVariable *Bias = M.getGlobalVariable(Name: VarName);
1199 if (Bias)
1200 return Bias;
1201
1202 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1203
1204 // Compiler must define this variable when runtime counter relocation
1205 // is being used. Runtime has a weak external reference that is used
1206 // to check whether that's the case or not.
1207 Bias = new GlobalVariable(M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
1208 Constant::getNullValue(Ty: Int64Ty), VarName);
1209 Bias->setVisibility(GlobalVariable::HiddenVisibility);
1210 // A definition that's weak (linkonce_odr) without being in a COMDAT
1211 // section wouldn't lead to link errors, but it would lead to a dead
1212 // data word from every TU but one. Putting it in COMDAT ensures there
1213 // will be exactly one data slot in the link.
1214 if (TT.supportsCOMDAT())
1215 Bias->setComdat(M.getOrInsertComdat(Name: VarName));
1216
1217 return Bias;
1218}
1219
1220Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
1221 auto *Counters = getOrCreateRegionCounters(Inc: I);
1222 IRBuilder<> Builder(I);
1223
1224 if (isa<InstrProfTimestampInst>(Val: I))
1225 Counters->setAlignment(Align(8));
1226
1227 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
1228 Ty: Counters->getValueType(), Ptr: Counters, Idx0: 0, Idx1: I->getIndex()->getZExtValue());
1229
1230 if (!isRuntimeCounterRelocationEnabled())
1231 return Addr;
1232
1233 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1234 Function *Fn = I->getParent()->getParent();
1235 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
1236 if (!BiasLI) {
1237 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1238 auto *Bias = getOrCreateBiasVar(VarName: getInstrProfCounterBiasVarName());
1239 BiasLI = EntryBuilder.CreateLoad(Ty: Int64Ty, Ptr: Bias, Name: "profc_bias");
1240 // Bias doesn't change after startup.
1241 BiasLI->setMetadata(KindID: LLVMContext::MD_invariant_load,
1242 Node: MDNode::get(Context&: M.getContext(), MDs: {}));
1243 }
1244 auto *Add = Builder.CreateAdd(LHS: Builder.CreatePtrToInt(V: Addr, DestTy: Int64Ty), RHS: BiasLI);
1245 return Builder.CreateIntToPtr(V: Add, DestTy: Addr->getType());
1246}
1247
1248Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
1249 auto *Bitmaps = getOrCreateRegionBitmaps(Inc: I);
1250 if (!isRuntimeCounterRelocationEnabled())
1251 return Bitmaps;
1252
1253 // Put BiasLI onto the entry block.
1254 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1255 Function *Fn = I->getFunction();
1256 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1257 auto *Bias = getOrCreateBiasVar(VarName: getInstrProfBitmapBiasVarName());
1258 auto *BiasLI = EntryBuilder.CreateLoad(Ty: Int64Ty, Ptr: Bias, Name: "profbm_bias");
1259 // Assume BiasLI invariant (in the function at least)
1260 BiasLI->setMetadata(KindID: LLVMContext::MD_invariant_load,
1261 Node: MDNode::get(Context&: M.getContext(), MDs: {}));
1262
1263 // Add Bias to Bitmaps and put it before the intrinsic.
1264 IRBuilder<> Builder(I);
1265 return Builder.CreatePtrAdd(Ptr: Bitmaps, Offset: BiasLI, Name: "profbm_addr");
1266}
1267
1268void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
1269 auto *Addr = getCounterAddress(I: CoverInstruction);
1270 IRBuilder<> Builder(CoverInstruction);
1271 if (ConditionalCounterUpdate) {
1272 Instruction *SplitBefore = CoverInstruction->getNextNode();
1273 auto &Ctx = CoverInstruction->getParent()->getContext();
1274 auto *Int8Ty = llvm::Type::getInt8Ty(C&: Ctx);
1275 Value *Load = Builder.CreateLoad(Ty: Int8Ty, Ptr: Addr, Name: "pgocount");
1276 Value *Cmp = Builder.CreateIsNotNull(Arg: Load, Name: "pgocount.ifnonzero");
1277 Instruction *ThenBranch =
1278 SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore, Unreachable: false);
1279 Builder.SetInsertPoint(ThenBranch);
1280 }
1281
1282 // We store zero to represent that this block is covered.
1283 Builder.CreateStore(Val: Builder.getInt8(C: 0), Ptr: Addr);
1284 CoverInstruction->eraseFromParent();
1285}
1286
1287void InstrLowerer::lowerTimestamp(
1288 InstrProfTimestampInst *TimestampInstruction) {
1289 assert(TimestampInstruction->getIndex()->isNullValue() &&
1290 "timestamp probes are always the first probe for a function");
1291 auto &Ctx = M.getContext();
1292 auto *TimestampAddr = getCounterAddress(I: TimestampInstruction);
1293 IRBuilder<> Builder(TimestampInstruction);
1294 auto *CalleeTy =
1295 FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: TimestampAddr->getType(), isVarArg: false);
1296 auto Callee = M.getOrInsertFunction(
1297 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), T: CalleeTy);
1298 Builder.CreateCall(Callee, Args: {TimestampAddr});
1299 TimestampInstruction->eraseFromParent();
1300}
1301
1302InstrLowerer::GPUPGOInvariants &
1303InstrLowerer::getOrCreateGPUInvariants(Function *F) {
1304 auto It = GPUInvariantsCache.find(Val: F);
1305 if (It != GPUInvariantsCache.end())
1306 return It->second;
1307
1308 LLVMContext &Context = M.getContext();
1309 auto *Int32Ty = Type::getInt32Ty(C&: Context);
1310
1311 BasicBlock &EntryBB = F->getEntryBlock();
1312 IRBuilder<> Builder(&*EntryBB.getFirstInsertionPt());
1313
1314 Value *Matched = ConstantInt::getTrue(Context);
1315 if (OffloadPGOSampling > 0) {
1316 FunctionCallee IsSampledFn =
1317 M.getOrInsertFunction(Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(
1318 CallImpl: RTLIB::impl___llvm_profile_sampling_gpu),
1319 RetTy: Int32Ty, Args: Int32Ty);
1320 Value *SampledInt = Builder.CreateCall(
1321 Callee: IsSampledFn, Args: {ConstantInt::get(Ty: Int32Ty, V: OffloadPGOSampling)},
1322 Name: "pgo.sampled");
1323 Matched = Builder.CreateICmpNE(LHS: SampledInt, RHS: ConstantInt::get(Ty: Int32Ty, V: 0),
1324 Name: "pgo.matched");
1325 }
1326
1327 auto &Inv = GPUInvariantsCache[F];
1328 Inv.Matched = Matched;
1329 return Inv;
1330}
1331
1332void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
1333 IRBuilder<> Builder(Inc);
1334 if (isGPUProfTarget(M)) {
1335 Function *F = Inc->getFunction();
1336 auto &Inv = getOrCreateGPUInvariants(F);
1337
1338 LLVMContext &Context = M.getContext();
1339 auto *Int64Ty = Type::getInt64Ty(C&: Context);
1340 auto *PtrTy = PointerType::getUnqual(C&: Context);
1341
1342 auto *Addr = getCounterAddress(I: Inc);
1343
1344 // Store the device wave/warp size into the profile data struct once per
1345 // function. AMDGPU folds llvm.amdgcn.wavefrontsize to the subtarget's
1346 // constant; other GPUs use their fixed warp size.
1347 if (!Inv.WaveSizeStored) {
1348 Inv.WaveSizeStored = true;
1349 GlobalVariable *NamePtr = Inc->getName();
1350 auto &PD = ProfileDataMap[NamePtr];
1351 if (PD.DataVar) {
1352 IRBuilder<> EntryBuilder(&*F->getEntryBlock().getFirstInsertionPt());
1353 Value *WaveSize16 = nullptr;
1354 // Look the intrinsic up by name so this target-agnostic pass does not
1355 // pull in IntrinsicsAMDGPU.h. AMDGPU folds the intrinsic to the
1356 // subtarget's wavefront size; other GPUs fall back to a 32-lane warp.
1357 if (TT.isAMDGPU()) {
1358 Intrinsic::ID WaveSizeID =
1359 Intrinsic::lookupIntrinsicID(Name: "llvm.amdgcn.wavefrontsize");
1360 if (WaveSizeID != Intrinsic::not_intrinsic) {
1361 Function *WaveSizeFn =
1362 Intrinsic::getOrInsertDeclaration(M: &M, id: WaveSizeID);
1363 Value *WaveSize = EntryBuilder.CreateCall(Callee: WaveSizeFn);
1364 WaveSize16 = EntryBuilder.CreateTrunc(
1365 V: WaveSize, DestTy: Type::getInt16Ty(C&: Context), Name: "wavesize.i16");
1366 }
1367 }
1368 if (!WaveSize16)
1369 WaveSize16 = ConstantInt::get(Ty: Type::getInt16Ty(C&: Context), V: 32);
1370 Value *WaveSizeAddr = EntryBuilder.CreateStructGEP(
1371 Ty: PD.DataVar->getValueType(), Ptr: PD.DataVar, Idx: 9, Name: "profd.wavesize");
1372 EntryBuilder.CreateStore(Val: WaveSize16, Ptr: WaveSizeAddr);
1373 }
1374 }
1375
1376 GlobalVariable *UniformCounters = getOrCreateUniformCounters(Inc);
1377 Value *UniformAddrArg = ConstantPointerNull::get(T: PtrTy);
1378 if (UniformCounters) {
1379 Value *UniformIndices[] = {Builder.getInt32(C: 0), Inc->getIndex()};
1380 Value *UniformAddr = Builder.CreateInBoundsGEP(
1381 Ty: UniformCounters->getValueType(), Ptr: UniformCounters, IdxList: UniformIndices,
1382 Name: "unifctr.addr");
1383 UniformAddrArg =
1384 Builder.CreatePointerBitCastOrAddrSpaceCast(V: UniformAddr, DestTy: PtrTy);
1385 }
1386 Value *CastAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Addr, DestTy: PtrTy);
1387 Value *StepI64 =
1388 Builder.CreateZExtOrTrunc(V: Inc->getStep(), DestTy: Int64Ty, Name: "step.i64");
1389
1390 auto *CalleeTy = FunctionType::get(Result: Type::getVoidTy(C&: Context),
1391 Params: {PtrTy, PtrTy, Int64Ty}, isVarArg: false);
1392 FunctionCallee Callee =
1393 M.getOrInsertFunction(Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(
1394 CallImpl: RTLIB::impl___llvm_profile_instrument_gpu),
1395 T: CalleeTy);
1396
1397 if (OffloadPGOSampling > 0) {
1398 BasicBlock *CurBB = Builder.GetInsertBlock();
1399 BasicBlock *ContBB =
1400 CurBB->splitBasicBlock(I: BasicBlock::iterator(Inc), BBName: "po_cont");
1401 BasicBlock *ThenBB = BasicBlock::Create(Context, Name: "po_then", Parent: F);
1402
1403 CurBB->getTerminator()->eraseFromParent();
1404 IRBuilder<> HeadBuilder(CurBB);
1405 HeadBuilder.CreateCondBr(Cond: Inv.Matched, True: ThenBB, False: ContBB);
1406
1407 IRBuilder<> ThenBuilder(ThenBB);
1408 ThenBuilder.CreateCall(Callee, Args: {CastAddr, UniformAddrArg, StepI64});
1409 ThenBuilder.CreateBr(Dest: ContBB);
1410 } else {
1411 Builder.CreateCall(Callee, Args: {CastAddr, UniformAddrArg, StepI64});
1412 }
1413 Inc->eraseFromParent();
1414 return;
1415 }
1416
1417 auto *Addr = getCounterAddress(I: Inc);
1418 // If promotion is enabled then delay generating atomic updates until
1419 // after promotion is done.
1420 if ((!isCounterPromotionEnabled() && isAtomic()) ||
1421 (Inc->getIndex()->isNullValue() && AtomicFirstCounter)) {
1422 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: Inc->getStep(),
1423 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
1424 } else {
1425 Value *IncStep = Inc->getStep();
1426 Value *Load = Builder.CreateLoad(Ty: IncStep->getType(), Ptr: Addr, Name: "pgocount");
1427 auto *Count = Builder.CreateAdd(LHS: Load, RHS: Inc->getStep());
1428 auto *Store = Builder.CreateStore(Val: Count, Ptr: Addr);
1429 if (isCounterPromotionEnabled())
1430 PromotionCandidates.emplace_back(args: cast<Instruction>(Val: Load), args&: Store);
1431 }
1432 Inc->eraseFromParent();
1433}
1434
1435void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
1436 ConstantArray *Names =
1437 cast<ConstantArray>(Val: CoverageNamesVar->getInitializer());
1438 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
1439 Constant *NC = Names->getOperand(i_nocapture: I);
1440 Value *V = NC->stripPointerCasts();
1441 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
1442 GlobalVariable *Name = cast<GlobalVariable>(Val: V);
1443
1444 Name->setLinkage(GlobalValue::PrivateLinkage);
1445 ReferencedNames.push_back(x: Name);
1446 if (isa<ConstantExpr>(Val: NC))
1447 NC->dropAllReferences();
1448 }
1449 CoverageNamesVar->eraseFromParent();
1450}
1451
1452void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
1453 InstrProfMCDCTVBitmapUpdate *Update) {
1454 auto &Ctx = M.getContext();
1455 IRBuilder<> Builder(Update);
1456 auto *Int8Ty = Type::getInt8Ty(C&: Ctx);
1457 auto *Int32Ty = Type::getInt32Ty(C&: Ctx);
1458 auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
1459 auto *BitmapAddr = getBitmapAddress(I: Update);
1460
1461 // Load Temp Val + BitmapIdx.
1462 // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
1463 auto *Temp = Builder.CreateAdd(
1464 LHS: Builder.CreateLoad(Ty: Int32Ty, Ptr: MCDCCondBitmapAddr, Name: "mcdc.temp"),
1465 RHS: Update->getBitmapIndex());
1466
1467 // Calculate byte offset using div8.
1468 // %1 = lshr i32 %mcdc.temp, 3
1469 auto *BitmapByteOffset = Builder.CreateLShr(LHS: Temp, RHS: 0x3);
1470
1471 // Add byte offset to section base byte address.
1472 // %4 = getelementptr inbounds i8, ptr @__profbm_test, i32 %1
1473 auto *BitmapByteAddr =
1474 Builder.CreateInBoundsPtrAdd(Ptr: BitmapAddr, Offset: BitmapByteOffset);
1475
1476 // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8)
1477 // %5 = and i32 %mcdc.temp, 7
1478 // %6 = trunc i32 %5 to i8
1479 auto *BitToSet = Builder.CreateTrunc(V: Builder.CreateAnd(LHS: Temp, RHS: 0x7), DestTy: Int8Ty);
1480
1481 // Shift bit offset left to form a bitmap.
1482 // %7 = shl i8 1, %6
1483 auto *ShiftedVal = Builder.CreateShl(LHS: Builder.getInt8(C: 0x1), RHS: BitToSet);
1484
1485 // Load profile bitmap byte.
1486 // %mcdc.bits = load i8, ptr %4, align 1
1487 auto *Bitmap = Builder.CreateLoad(Ty: Int8Ty, Ptr: BitmapByteAddr, Name: "mcdc.bits");
1488
1489 if (isAtomic()) {
1490 // If ((Bitmap & Val) != Val), then execute atomic (Bitmap |= Val).
1491 // Note, just-loaded Bitmap might not be up-to-date. Use it just for
1492 // early testing.
1493 auto *Masked = Builder.CreateAnd(LHS: Bitmap, RHS: ShiftedVal);
1494 auto *ShouldStore = Builder.CreateICmpNE(LHS: Masked, RHS: ShiftedVal);
1495
1496 // Assume updating will be rare.
1497 auto *Unlikely = MDBuilder(Ctx).createUnlikelyBranchWeights();
1498 Instruction *ThenBranch =
1499 SplitBlockAndInsertIfThen(Cond: ShouldStore, SplitBefore: Update, Unreachable: false, BranchWeights: Unlikely);
1500
1501 // Execute if (unlikely(ShouldStore)).
1502 Builder.SetInsertPoint(ThenBranch);
1503 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: BitmapByteAddr, Val: ShiftedVal,
1504 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
1505 } else {
1506 // Perform logical OR of profile bitmap byte and shifted bit offset.
1507 // %8 = or i8 %mcdc.bits, %7
1508 auto *Result = Builder.CreateOr(LHS: Bitmap, RHS: ShiftedVal);
1509
1510 // Store the updated profile bitmap byte.
1511 // store i8 %8, ptr %3, align 1
1512 Builder.CreateStore(Val: Result, Ptr: BitmapByteAddr);
1513 }
1514
1515 Update->eraseFromParent();
1516}
1517
1518/// Get the name of a profiling variable for a particular function.
1519static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
1520 bool &Renamed) {
1521 StringRef NamePrefix = getInstrProfNameVarPrefix();
1522 StringRef Name = Inc->getName()->getName().substr(Start: NamePrefix.size());
1523 Function *F = Inc->getParent()->getParent();
1524 Module *M = F->getParent();
1525 if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
1526 !canRenameComdatFunc(F: *F)) {
1527 Renamed = false;
1528 return (Prefix + Name).str();
1529 }
1530 Renamed = true;
1531 uint64_t FuncHash = Inc->getHash()->getZExtValue();
1532 SmallVector<char, 24> HashPostfix;
1533 if (Name.ends_with(Suffix: (Twine(".") + Twine(FuncHash)).toStringRef(Out&: HashPostfix)))
1534 return (Prefix + Name).str();
1535 return (Prefix + Name + "." + Twine(FuncHash)).str();
1536}
1537
1538static inline bool shouldRecordFunctionAddr(Function *F) {
1539 // Only record function addresses if IR PGO is enabled or if clang value
1540 // profiling is enabled. Recording function addresses greatly increases object
1541 // file size, because it prevents the inliner from deleting functions that
1542 // have been inlined everywhere.
1543 if (!profDataReferencedByCode(M: *F->getParent()))
1544 return false;
1545
1546 // Check the linkage
1547 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
1548 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
1549 !HasAvailableExternallyLinkage)
1550 return true;
1551
1552 // A function marked 'alwaysinline' with available_externally linkage can't
1553 // have its address taken. Doing so would create an undefined external ref to
1554 // the function, which would fail to link.
1555 if (HasAvailableExternallyLinkage &&
1556 F->hasFnAttribute(Kind: Attribute::AlwaysInline))
1557 return false;
1558
1559 // Prohibit function address recording if the function is both internal and
1560 // COMDAT. This avoids the profile data variable referencing internal symbols
1561 // in COMDAT.
1562 if (F->hasLocalLinkage() && F->hasComdat())
1563 return false;
1564
1565 // Check uses of this function for other than direct calls or invokes to it.
1566 // Inline virtual functions have linkeOnceODR linkage. When a key method
1567 // exists, the vtable will only be emitted in the TU where the key method
1568 // is defined. In a TU where vtable is not available, the function won't
1569 // be 'addresstaken'. If its address is not recorded here, the profile data
1570 // with missing address may be picked by the linker leading to missing
1571 // indirect call target info.
1572 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
1573}
1574
1575static inline bool shouldUsePublicSymbol(Function *Fn) {
1576 // It isn't legal to make an alias of this function at all
1577 if (Fn->isDeclarationForLinker())
1578 return true;
1579
1580 // Symbols with local linkage can just use the symbol directly without
1581 // introducing relocations
1582 if (Fn->hasLocalLinkage())
1583 return true;
1584
1585 // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some
1586 // unfavorable interaction between the new alias and the alias renaming done
1587 // in LowerTypeTests under ThinLTO. For comdat functions that would normally
1588 // be deduplicated, but the renaming scheme ends up preventing renaming, since
1589 // it creates unique names for each alias, resulting in duplicated symbols. In
1590 // the future, we should update the CFI related passes to migrate these
1591 // aliases to the same module as the jump-table they refer to will be defined.
1592 if (Fn->hasMetadata(KindID: LLVMContext::MD_type))
1593 return true;
1594
1595 // For comdat functions, an alias would need the same linkage as the original
1596 // function and hidden visibility. There is no point in adding an alias with
1597 // identical linkage an visibility to avoid introducing symbolic relocations.
1598 if (Fn->hasComdat() &&
1599 (Fn->getVisibility() == GlobalValue::VisibilityTypes::HiddenVisibility))
1600 return true;
1601
1602 // its OK to use an alias
1603 return false;
1604}
1605
1606static inline Constant *getFuncAddrForProfData(Function *Fn) {
1607 auto *Int8PtrTy = PointerType::getUnqual(C&: Fn->getContext());
1608 // Store a nullptr in __llvm_profd, if we shouldn't use a real address
1609 if (!shouldRecordFunctionAddr(F: Fn))
1610 return ConstantPointerNull::get(T: Int8PtrTy);
1611
1612 // If we can't use an alias, we must use the public symbol, even though this
1613 // may require a symbolic relocation.
1614 if (shouldUsePublicSymbol(Fn))
1615 return Fn;
1616
1617 // For GPU targets, weak functions cannot use private aliases because
1618 // LTO may pick a different TU's copy, leaving the alias undefined
1619 if (isGPUProfTarget(M: *Fn->getParent()) &&
1620 GlobalValue::isWeakForLinker(Linkage: Fn->getLinkage()))
1621 return Fn;
1622
1623 // When possible use a private alias to avoid symbolic relocations.
1624 auto *GA = GlobalAlias::create(Linkage: GlobalValue::LinkageTypes::PrivateLinkage,
1625 Name: Fn->getName() + ".local", Aliasee: Fn);
1626
1627 // When the instrumented function is a COMDAT function, we cannot use a
1628 // private alias. If we did, we would create reference to a local label in
1629 // this function's section. If this version of the function isn't selected by
1630 // the linker, then the metadata would introduce a reference to a discarded
1631 // section. So, for COMDAT functions, we need to adjust the linkage of the
1632 // alias. Using hidden visibility avoids a dynamic relocation and an entry in
1633 // the dynamic symbol table.
1634 //
1635 // Note that this handles COMDAT functions with visibility other than Hidden,
1636 // since that case is covered in shouldUsePublicSymbol()
1637 if (Fn->hasComdat()) {
1638 GA->setLinkage(Fn->getLinkage());
1639 GA->setVisibility(GlobalValue::VisibilityTypes::HiddenVisibility);
1640 }
1641
1642 // appendToCompilerUsed(*Fn->getParent(), {GA});
1643
1644 return GA;
1645}
1646
1647static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
1648 // NVPTX is an ELF target but PTX does not expose sections or linker symbols.
1649 if (TT.isNVPTX())
1650 return true;
1651
1652 // compiler-rt uses linker support to get data/counters/name start/end for
1653 // ELF, COFF, Mach-O, XCOFF, and Wasm.
1654 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
1655 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF() ||
1656 TT.isOSBinFormatWasm())
1657 return false;
1658
1659 return true;
1660}
1661
1662void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO,
1663 StringRef CounterGroupName) {
1664 // Place lowered global variables in a comdat group if the associated function
1665 // or global variable is a COMDAT. This will make sure that only one copy of
1666 // global variable (e.g. function counters) of the COMDAT function will be
1667 // emitted after linking.
1668 bool NeedComdat = needsComdatForCounter(GV: *GO, M);
1669 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
1670
1671 if (!UseComdat)
1672 return;
1673
1674 // Keep in mind that this pass may run before the inliner, so we need to
1675 // create a new comdat group (for counters, profiling data, etc). If we use
1676 // the comdat of the parent function, that will result in relocations against
1677 // discarded sections.
1678 //
1679 // If the data variable is referenced by code, non-counter variables (notably
1680 // profiling data) and counters have to be in different comdats for COFF
1681 // because the Visual C++ linker will report duplicate symbol errors if there
1682 // are multiple external symbols with the same name marked
1683 // IMAGE_COMDAT_SELECT_ASSOCIATIVE.
1684 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
1685 ? GV->getName()
1686 : CounterGroupName;
1687 Comdat *C = M.getOrInsertComdat(Name: GroupName);
1688
1689 if (!NeedComdat) {
1690 // Object file format must be ELF since `UseComdat && !NeedComdat` is true.
1691 //
1692 // For ELF, when not using COMDAT, put counters, data and values into a
1693 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
1694 // allows -z start-stop-gc to discard the entire group when the function is
1695 // discarded.
1696 C->setSelectionKind(Comdat::NoDeduplicate);
1697 }
1698 GV->setComdat(C);
1699 // COFF doesn't allow the comdat group leader to have private linkage, so
1700 // upgrade private linkage to internal linkage to produce a symbol table
1701 // entry.
1702 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
1703 GV->setLinkage(GlobalValue::InternalLinkage);
1704}
1705
1706static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {
1707 if (!profDataReferencedByCode(M: *GV->getParent()))
1708 return false;
1709
1710 if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&
1711 !GV->hasAvailableExternallyLinkage())
1712 return true;
1713
1714 // This avoids the profile data from referencing internal symbols in
1715 // COMDAT.
1716 if (GV->hasLocalLinkage() && GV->hasComdat())
1717 return false;
1718
1719 return true;
1720}
1721
1722// FIXME: Introduce an internal alias like what's done for functions to reduce
1723// the number of relocation entries.
1724static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
1725 // Store a nullptr in __profvt_ if a real address shouldn't be used.
1726 if (!shouldRecordVTableAddr(GV))
1727 return ConstantPointerNull::get(T: PointerType::getUnqual(C&: GV->getContext()));
1728
1729 return GV;
1730}
1731
1732void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) {
1733 assert(ProfileCorrelate != InstrProfCorrelator::DEBUG_INFO &&
1734 "Value profiling is not supported with lightweight instrumentation");
1735 if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
1736 return;
1737
1738 // Skip llvm internal global variable or __prof variables.
1739 if (GV->getName().starts_with(Prefix: "llvm.") ||
1740 GV->getName().starts_with(Prefix: "__llvm") ||
1741 GV->getName().starts_with(Prefix: "__prof"))
1742 return;
1743
1744 // VTableProfData already created
1745 auto It = VTableDataMap.find(Val: GV);
1746 if (It != VTableDataMap.end() && It->second)
1747 return;
1748
1749 GlobalValue::LinkageTypes Linkage = GV->getLinkage();
1750 GlobalValue::VisibilityTypes Visibility = GV->getVisibility();
1751
1752 // This is to keep consistent with per-function profile data
1753 // for correctness.
1754 if (TT.isOSBinFormatXCOFF()) {
1755 Linkage = GlobalValue::InternalLinkage;
1756 Visibility = GlobalValue::DefaultVisibility;
1757 }
1758
1759 LLVMContext &Ctx = M.getContext();
1760 Type *DataTypes[] = {
1761#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
1762#include "llvm/ProfileData/InstrProfData.inc"
1763#undef INSTR_PROF_VTABLE_DATA
1764 };
1765
1766 auto *DataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes));
1767
1768 // Used by INSTR_PROF_VTABLE_DATA MACRO
1769 Constant *VTableAddr = getVTableAddrForProfData(GV);
1770 const std::string PGOVTableName = getPGOName(V: *GV);
1771 // Record the length of the vtable. This is needed since vtable pointers
1772 // loaded from C++ objects might be from the middle of a vtable definition.
1773 uint32_t VTableSizeVal = GV->getGlobalSize(DL: M.getDataLayout());
1774
1775 Constant *DataVals[] = {
1776#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
1777#include "llvm/ProfileData/InstrProfData.inc"
1778#undef INSTR_PROF_VTABLE_DATA
1779 };
1780
1781 auto *Data =
1782 new GlobalVariable(M, DataTy, /*constant=*/false, Linkage,
1783 ConstantStruct::get(T: DataTy, V: DataVals),
1784 getInstrProfVTableVarPrefix() + PGOVTableName);
1785
1786 Data->setVisibility(Visibility);
1787 Data->setSection(getInstrProfSectionName(IPSK: IPSK_vtab, OF: TT.getObjectFormat()));
1788 Data->setAlignment(Align(8));
1789
1790 maybeSetComdat(GV: Data, GO: GV, CounterGroupName: Data->getName());
1791
1792 VTableDataMap[GV] = Data;
1793
1794 ReferencedVTables.push_back(x: GV);
1795
1796 // VTable <Hash, Addr> is used by runtime but not referenced by other
1797 // sections. Conservatively mark it linker retained.
1798 UsedVars.push_back(x: Data);
1799}
1800
1801GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc,
1802 InstrProfSectKind IPSK) {
1803 GlobalVariable *NamePtr = Inc->getName();
1804
1805 // Match the linkage and visibility of the name global.
1806 Function *Fn = Inc->getParent()->getParent();
1807 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
1808 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1809
1810 // Use internal rather than private linkage so the counter variable shows up
1811 // in the symbol table when using debug info for correlation.
1812 if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO &&
1813 TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage)
1814 Linkage = GlobalValue::InternalLinkage;
1815
1816 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1817 // symbols in the same csect won't be discarded. When there are duplicate weak
1818 // symbols, we can NOT guarantee that the relocations get resolved to the
1819 // intended weak symbol, so we can not ensure the correctness of the relative
1820 // CounterPtr, so we have to use private linkage for counter and data symbols.
1821 if (TT.isOSBinFormatXCOFF()) {
1822 Linkage = GlobalValue::PrivateLinkage;
1823 Visibility = GlobalValue::DefaultVisibility;
1824 }
1825 // Move the name variable to the right section.
1826 bool Renamed;
1827 GlobalVariable *Ptr;
1828 StringRef VarPrefix;
1829 std::string VarName;
1830 if (IPSK == IPSK_cnts) {
1831 VarPrefix = getInstrProfCountersVarPrefix();
1832 VarName = getVarName(Inc, Prefix: VarPrefix, Renamed);
1833 InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Val: Inc);
1834 Ptr = createRegionCounters(Inc: CntrIncrement, Name: VarName, Linkage);
1835 } else if (IPSK == IPSK_bitmap) {
1836 VarPrefix = getInstrProfBitmapVarPrefix();
1837 VarName = getVarName(Inc, Prefix: VarPrefix, Renamed);
1838 InstrProfMCDCBitmapInstBase *BitmapUpdate =
1839 dyn_cast<InstrProfMCDCBitmapInstBase>(Val: Inc);
1840 Ptr = createRegionBitmaps(Inc: BitmapUpdate, Name: VarName, Linkage);
1841 } else {
1842 llvm_unreachable("Profile Section must be for Counters or Bitmaps");
1843 }
1844
1845 Ptr->setVisibility(Visibility);
1846 Ptr->setSection(getInstrProfSectionName(IPSK, OF: TT.getObjectFormat()));
1847 Ptr->setLinkage(Linkage);
1848 if (isGPUProfTarget(M) && !Ptr->hasComdat()) {
1849 Ptr->setComdat(M.getOrInsertComdat(Name: VarName));
1850 Ptr->setLinkage(GlobalValue::LinkOnceODRLinkage);
1851 Ptr->setVisibility(GlobalValue::ProtectedVisibility);
1852 } else {
1853 maybeSetComdat(GV: Ptr, GO: Fn, CounterGroupName: VarName);
1854 }
1855 return Ptr;
1856}
1857
1858GlobalVariable *
1859InstrLowerer::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
1860 StringRef Name,
1861 GlobalValue::LinkageTypes Linkage) {
1862 uint64_t NumBytes = Inc->getNumBitmapBytes();
1863 auto *BitmapTy = ArrayType::get(ElementType: Type::getInt8Ty(C&: M.getContext()), NumElements: NumBytes);
1864 auto GV = new GlobalVariable(M, BitmapTy, false, Linkage,
1865 Constant::getNullValue(Ty: BitmapTy), Name);
1866 GV->setAlignment(Align(1));
1867 return GV;
1868}
1869
1870GlobalVariable *
1871InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
1872 GlobalVariable *NamePtr = Inc->getName();
1873 auto &PD = ProfileDataMap[NamePtr];
1874 if (PD.RegionBitmaps)
1875 return PD.RegionBitmaps;
1876
1877 // If RegionBitmaps doesn't already exist, create it by first setting up
1878 // the corresponding profile section.
1879 auto *BitmapPtr = setupProfileSection(Inc, IPSK: IPSK_bitmap);
1880 PD.RegionBitmaps = BitmapPtr;
1881 PD.NumBitmapBytes = Inc->getNumBitmapBytes();
1882
1883 if (PD.NumBitmapBytes &&
1884 ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) {
1885 LLVMContext &Ctx = M.getContext();
1886 Function *Fn = Inc->getParent()->getParent();
1887 if (auto *SP = Fn->getSubprogram()) {
1888 DIBuilder DB(M, true, SP->getUnit());
1889 Metadata *FunctionNameAnnotation[] = {
1890 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::FunctionNameAttributeName),
1891 MDString::get(Context&: Ctx, Str: getPGOFuncNameVarInitializer(NameVar: NamePtr)),
1892 };
1893 Metadata *NumBitmapBitsAnnotation[] = {
1894 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::NumBitmapBitsAttributeName),
1895 ConstantAsMetadata::get(C: Inc->getNumBitmapBits()),
1896 };
1897 auto Annotations = DB.getOrCreateArray(Elements: {
1898 MDNode::get(Context&: Ctx, MDs: FunctionNameAnnotation),
1899 MDNode::get(Context&: Ctx, MDs: NumBitmapBitsAnnotation),
1900 });
1901 auto *DICounter = DB.createGlobalVariableExpression(
1902 Context: SP, Name: BitmapPtr->getName(), /*LinkageName=*/StringRef(), File: SP->getFile(),
1903 /*LineNo=*/0, Ty: DB.createUnspecifiedType(Name: "Profile Bitmap Type"),
1904 IsLocalToUnit: BitmapPtr->hasLocalLinkage(), /*IsDefined=*/isDefined: true, /*Expr=*/nullptr,
1905 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1906 Annotations);
1907 BitmapPtr->addDebugInfo(GV: DICounter);
1908 DB.finalizeSubprogram(SP);
1909 DB.finalize();
1910 }
1911
1912 // Mark the bitmap variable as used so that it isn't optimized out.
1913 CompilerUsedVars.push_back(x: PD.RegionBitmaps);
1914 }
1915
1916 return PD.RegionBitmaps;
1917}
1918
1919GlobalVariable *
1920InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
1921 GlobalValue::LinkageTypes Linkage) {
1922 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1923 auto &Ctx = M.getContext();
1924 GlobalVariable *GV;
1925 if (isa<InstrProfCoverInst>(Val: Inc)) {
1926 auto *CounterTy = Type::getInt8Ty(C&: Ctx);
1927 auto *CounterArrTy = ArrayType::get(ElementType: CounterTy, NumElements: NumCounters);
1928 // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
1929 std::vector<Constant *> InitialValues(NumCounters,
1930 Constant::getAllOnesValue(Ty: CounterTy));
1931 GV = new GlobalVariable(M, CounterArrTy, false, Linkage,
1932 ConstantArray::get(T: CounterArrTy, V: InitialValues),
1933 Name);
1934 GV->setAlignment(Align(1));
1935 } else {
1936 auto *CounterTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NumCounters);
1937 GV = new GlobalVariable(M, CounterTy, false, Linkage,
1938 Constant::getNullValue(Ty: CounterTy), Name);
1939 GV->setAlignment(Align(8));
1940 }
1941 return GV;
1942}
1943
1944GlobalVariable *
1945InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
1946 GlobalVariable *NamePtr = Inc->getName();
1947 auto &PD = ProfileDataMap[NamePtr];
1948 if (PD.RegionCounters)
1949 return PD.RegionCounters;
1950
1951 // If RegionCounters doesn't already exist, create it by first setting up
1952 // the corresponding profile section.
1953 auto *CounterPtr = setupProfileSection(Inc, IPSK: IPSK_cnts);
1954 PD.RegionCounters = CounterPtr;
1955
1956 if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) {
1957 LLVMContext &Ctx = M.getContext();
1958 Function *Fn = Inc->getParent()->getParent();
1959 if (auto *SP = Fn->getSubprogram()) {
1960 DIBuilder DB(M, true, SP->getUnit());
1961 Metadata *FunctionNameAnnotation[] = {
1962 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::FunctionNameAttributeName),
1963 MDString::get(Context&: Ctx, Str: getPGOFuncNameVarInitializer(NameVar: NamePtr)),
1964 };
1965 Metadata *CFGHashAnnotation[] = {
1966 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::CFGHashAttributeName),
1967 ConstantAsMetadata::get(C: Inc->getHash()),
1968 };
1969 Metadata *NumCountersAnnotation[] = {
1970 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::NumCountersAttributeName),
1971 ConstantAsMetadata::get(C: Inc->getNumCounters()),
1972 };
1973 auto Annotations = DB.getOrCreateArray(Elements: {
1974 MDNode::get(Context&: Ctx, MDs: FunctionNameAnnotation),
1975 MDNode::get(Context&: Ctx, MDs: CFGHashAnnotation),
1976 MDNode::get(Context&: Ctx, MDs: NumCountersAnnotation),
1977 });
1978 auto *DICounter = DB.createGlobalVariableExpression(
1979 Context: SP, Name: CounterPtr->getName(), /*LinkageName=*/StringRef(), File: SP->getFile(),
1980 /*LineNo=*/0, Ty: DB.createUnspecifiedType(Name: "Profile Data Type"),
1981 IsLocalToUnit: CounterPtr->hasLocalLinkage(), /*IsDefined=*/isDefined: true, /*Expr=*/nullptr,
1982 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1983 Annotations);
1984 CounterPtr->addDebugInfo(GV: DICounter);
1985 DB.finalizeSubprogram(SP);
1986 DB.finalize();
1987 }
1988
1989 // Mark the counter variable as used so that it isn't optimized out.
1990 CompilerUsedVars.push_back(x: PD.RegionCounters);
1991 }
1992
1993 // Create uniform counters before the data variable so that
1994 // UniformCounterPtr can reference them in createDataVariable().
1995 getOrCreateUniformCounters(Inc);
1996
1997 // Create the data variable (if it doesn't already exist).
1998 createDataVariable(Inc);
1999
2000 return PD.RegionCounters;
2001}
2002
2003GlobalVariable *
2004InstrLowerer::getOrCreateUniformCounters(InstrProfCntrInstBase *Inc) {
2005 // Uniform counters are only meaningful for GPU profile targets.
2006 if (!isGPUProfTarget(M))
2007 return nullptr;
2008
2009 GlobalVariable *NamePtr = Inc->getName();
2010 auto &PD = ProfileDataMap[NamePtr];
2011 if (PD.UniformCounters)
2012 return PD.UniformCounters;
2013
2014 assert(PD.RegionCounters && "region counters must be created first");
2015
2016 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
2017
2018 LLVMContext &Ctx = M.getContext();
2019 ArrayType *CounterTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NumCounters);
2020
2021 bool Renamed;
2022 std::string VarName = getVarName(Inc, Prefix: "__llvm_prf_unifcnt_", Renamed);
2023
2024 auto *GV = new GlobalVariable(M, CounterTy, false, NamePtr->getLinkage(),
2025 Constant::getNullValue(Ty: CounterTy), VarName);
2026 GV->setAlignment(Align(8));
2027
2028 GV->setSection(getInstrProfSectionName(IPSK: IPSK_ucnts, OF: TT.getObjectFormat()));
2029
2030 GV->setComdat(M.getOrInsertComdat(Name: VarName));
2031 GV->setLinkage(GlobalValue::LinkOnceODRLinkage);
2032 GV->setVisibility(GlobalValue::ProtectedVisibility);
2033
2034 PD.UniformCounters = GV;
2035 CompilerUsedVars.push_back(x: GV);
2036
2037 return PD.UniformCounters;
2038}
2039
2040void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
2041 // When debug information is correlated to profile data, a data variable
2042 // is not needed.
2043 if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)
2044 return;
2045
2046 GlobalVariable *NamePtr = Inc->getName();
2047 auto &PD = ProfileDataMap[NamePtr];
2048
2049 // Return if data variable was already created.
2050 if (PD.DataVar)
2051 return;
2052
2053 LLVMContext &Ctx = M.getContext();
2054
2055 Function *Fn = Inc->getParent()->getParent();
2056 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
2057 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
2058
2059 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
2060 // symbols in the same csect won't be discarded. When there are duplicate weak
2061 // symbols, we can NOT guarantee that the relocations get resolved to the
2062 // intended weak symbol, so we can not ensure the correctness of the relative
2063 // CounterPtr, so we have to use private linkage for counter and data symbols.
2064 if (TT.isOSBinFormatXCOFF()) {
2065 Linkage = GlobalValue::PrivateLinkage;
2066 Visibility = GlobalValue::DefaultVisibility;
2067 }
2068
2069 bool NeedComdat = needsComdatForCounter(GV: *Fn, M);
2070 bool Renamed;
2071
2072 // The Data Variable section is anchored to profile counters.
2073 std::string CntsVarName =
2074 getVarName(Inc, Prefix: getInstrProfCountersVarPrefix(), Renamed);
2075 std::string DataVarName =
2076 getVarName(Inc, Prefix: getInstrProfDataVarPrefix(), Renamed);
2077
2078 auto *Int8PtrTy = PointerType::getUnqual(C&: Ctx);
2079 // Allocate statically the array of pointers to value profile nodes for
2080 // the current function.
2081 Constant *ValuesPtrExpr = ConstantPointerNull::get(T: Int8PtrTy);
2082 uint64_t NS = 0;
2083 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
2084 NS += PD.NumValueSites[Kind];
2085 if (NS > 0 && ValueProfileStaticAlloc &&
2086 !needsRuntimeRegistrationOfSectionRange(TT)) {
2087 ArrayType *ValuesTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NS);
2088 auto *ValuesVar = new GlobalVariable(
2089 M, ValuesTy, false, Linkage, Constant::getNullValue(Ty: ValuesTy),
2090 getVarName(Inc, Prefix: getInstrProfValuesVarPrefix(), Renamed));
2091 ValuesVar->setVisibility(Visibility);
2092 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *ValuesVar);
2093 ValuesVar->setSection(
2094 getInstrProfSectionName(IPSK: IPSK_vals, OF: TT.getObjectFormat()));
2095 ValuesVar->setAlignment(Align(8));
2096 maybeSetComdat(GV: ValuesVar, GO: Fn, CounterGroupName: CntsVarName);
2097 ValuesPtrExpr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
2098 C: ValuesVar, Ty: PointerType::get(C&: Fn->getContext(), AddressSpace: 0));
2099 }
2100
2101 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
2102
2103 Constant *CounterPtr = PD.RegionCounters;
2104 Constant *UniformCounterPtr = PD.UniformCounters;
2105
2106 uint64_t NumBitmapBytes = PD.NumBitmapBytes;
2107
2108 // Create data variable.
2109 auto *IntPtrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
2110 auto *Int16Ty = Type::getInt16Ty(C&: Ctx);
2111 auto *Int16ArrayTy = ArrayType::get(ElementType: Int16Ty, NumElements: IPVK_Last + 1);
2112 auto *DataTy = getProfileDataTy();
2113
2114 Constant *FunctionAddr = getFuncAddrForProfData(Fn);
2115
2116 Constant *Int16ArrayVals[IPVK_Last + 1];
2117 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
2118 Int16ArrayVals[Kind] = ConstantInt::get(Ty: Int16Ty, V: PD.NumValueSites[Kind]);
2119
2120 uint16_t OffloadDeviceWaveSizeVal = 0;
2121
2122 if (isGPUProfTarget(M)) {
2123 // For GPU targets, weak functions need weak linkage for their profile data
2124 // aliases to allow linker deduplication across TUs
2125 if (GlobalValue::isWeakForLinker(Linkage: Fn->getLinkage()))
2126 Linkage = Fn->getLinkage();
2127 else
2128 Linkage = GlobalValue::ExternalLinkage;
2129 Visibility = GlobalValue::ProtectedVisibility;
2130 }
2131 // If the data variable is not referenced by code (if we don't emit
2132 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
2133 // data variable live under linker GC, the data variable can be private. This
2134 // optimization applies to ELF.
2135 //
2136 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
2137 // to be false.
2138 //
2139 // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
2140 // that other copies must have the same CFG and cannot have value profiling.
2141 // If no hash suffix, other profd copies may be referenced by code.
2142 if (!isGPUProfTarget(M) && NS == 0 &&
2143 !(DataReferencedByCode && NeedComdat && !Renamed) &&
2144 (TT.isOSBinFormatELF() ||
2145 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
2146 Linkage = GlobalValue::PrivateLinkage;
2147 Visibility = GlobalValue::DefaultVisibility;
2148 }
2149 // GPU-target ELF objects are always ET_DYN, so non-local symbols with
2150 // default visibility are preemptible. The CounterPtr label difference
2151 // emits a REL32 relocation that lld rejects against preemptible targets.
2152 if (TT.isGPU() && TT.isOSBinFormatELF() &&
2153 !GlobalValue::isLocalLinkage(Linkage))
2154 Visibility = GlobalValue::ProtectedVisibility;
2155 auto *Data =
2156 new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName);
2157
2158 Constant *RelativeCounterPtr;
2159 Constant *RelativeUniformCounterPtr = ConstantInt::get(Ty: IntPtrTy, V: 0);
2160 GlobalVariable *BitmapPtr = PD.RegionBitmaps;
2161 Constant *RelativeBitmapPtr = ConstantInt::get(Ty: IntPtrTy, V: 0);
2162 InstrProfSectKind DataSectionKind;
2163 // With binary profile correlation, profile data is not loaded into memory.
2164 // profile data must reference profile counter with an absolute relocation.
2165 if (ProfileCorrelate == InstrProfCorrelator::BINARY) {
2166 DataSectionKind = IPSK_covdata;
2167 RelativeCounterPtr = ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy);
2168 if (BitmapPtr != nullptr)
2169 RelativeBitmapPtr = ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy);
2170 if (UniformCounterPtr != nullptr)
2171 RelativeUniformCounterPtr =
2172 ConstantExpr::getPtrToInt(C: UniformCounterPtr, Ty: IntPtrTy);
2173 } else if (TT.isNVPTX()) {
2174 // The NVPTX target cannot handle self-referencing constant expressions in
2175 // global initializers at all. Use absolute pointers and have the runtime
2176 // registration convert them to relative offsets.
2177 DataSectionKind = IPSK_data;
2178 RelativeCounterPtr = ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy);
2179 } else {
2180 // Reference the counter variable with a label difference (link-time
2181 // constant).
2182 DataSectionKind = IPSK_data;
2183 RelativeCounterPtr =
2184 ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy),
2185 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2186 if (BitmapPtr != nullptr)
2187 RelativeBitmapPtr =
2188 ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy),
2189 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2190 if (UniformCounterPtr != nullptr)
2191 RelativeUniformCounterPtr = ConstantExpr::getSub(
2192 C1: ConstantExpr::getPtrToInt(C: UniformCounterPtr, Ty: IntPtrTy),
2193 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2194 }
2195
2196 Constant *DataVals[] = {
2197#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
2198#include "llvm/ProfileData/InstrProfData.inc"
2199 };
2200 Data->setInitializer(ConstantStruct::get(T: DataTy, V: DataVals));
2201
2202 Data->setVisibility(Visibility);
2203 Data->setSection(
2204 getInstrProfSectionName(IPSK: DataSectionKind, OF: TT.getObjectFormat()));
2205 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
2206 if (isGPUProfTarget(M) && !Data->hasComdat()) {
2207 Data->setComdat(M.getOrInsertComdat(Name: CntsVarName));
2208 Data->setLinkage(GlobalValue::LinkOnceODRLinkage);
2209 } else {
2210 maybeSetComdat(GV: Data, GO: Fn, CounterGroupName: CntsVarName);
2211 }
2212
2213 PD.DataVar = Data;
2214
2215 // Mark the data variable as used so that it isn't stripped out.
2216 CompilerUsedVars.push_back(x: Data);
2217 // Now that the linkage set by the FE has been passed to the data and counter
2218 // variables, reset Name variable's linkage and visibility to private so that
2219 // it can be removed later by the compiler.
2220 NamePtr->setLinkage(GlobalValue::PrivateLinkage);
2221 // Collect the referenced names to be used by emitNameData.
2222 ReferencedNames.push_back(x: NamePtr);
2223}
2224
2225void InstrLowerer::emitVNodes() {
2226 if (!ValueProfileStaticAlloc)
2227 return;
2228
2229 // For now only support this on platforms that do
2230 // not require runtime registration to discover
2231 // named section start/end.
2232 if (needsRuntimeRegistrationOfSectionRange(TT))
2233 return;
2234
2235 size_t TotalNS = 0;
2236 for (auto &PD : ProfileDataMap) {
2237 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
2238 TotalNS += PD.second.NumValueSites[Kind];
2239 }
2240
2241 if (!TotalNS)
2242 return;
2243
2244 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
2245// Heuristic for small programs with very few total value sites.
2246// The default value of vp-counters-per-site is chosen based on
2247// the observation that large apps usually have a low percentage
2248// of value sites that actually have any profile data, and thus
2249// the average number of counters per site is low. For small
2250// apps with very few sites, this may not be true. Bump up the
2251// number of counters in this case.
2252#define INSTR_PROF_MIN_VAL_COUNTS 10
2253 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
2254 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, b: (int)NumCounters * 2);
2255
2256 auto &Ctx = M.getContext();
2257 Type *VNodeTypes[] = {
2258#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
2259#include "llvm/ProfileData/InstrProfData.inc"
2260 };
2261 auto *VNodeTy = StructType::get(Context&: Ctx, Elements: ArrayRef(VNodeTypes));
2262
2263 ArrayType *VNodesTy = ArrayType::get(ElementType: VNodeTy, NumElements: NumCounters);
2264 auto *VNodesVar = new GlobalVariable(
2265 M, VNodesTy, false, GlobalValue::PrivateLinkage,
2266 Constant::getNullValue(Ty: VNodesTy), getInstrProfVNodesVarName());
2267 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *VNodesVar);
2268 VNodesVar->setSection(
2269 getInstrProfSectionName(IPSK: IPSK_vnodes, OF: TT.getObjectFormat()));
2270 VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(Ty: VNodesTy));
2271 // VNodesVar is used by runtime but not referenced via relocation by other
2272 // sections. Conservatively make it linker retained.
2273 UsedVars.push_back(x: VNodesVar);
2274}
2275
2276// Build the per-TU device-PGO sections struct: section start/stop bounds for
2277// names/counters/data/uniform-counters plus the raw version. Returns null if it
2278// already exists.
2279static GlobalVariable *emitGPUOffloadSectionsStruct(Module &M,
2280 StringRef CUIDPostfix) {
2281 std::string Name = ("__llvm_profile_sections" + CUIDPostfix).str();
2282 if (M.getNamedValue(Name))
2283 return nullptr;
2284
2285 LLVMContext &Ctx = M.getContext();
2286 unsigned AS = M.getDataLayout().getDefaultGlobalsAddressSpace();
2287 auto Extern = [&](StringRef Sym, Type *Ty, bool IsConst,
2288 GlobalValue::VisibilityTypes Vis) {
2289 GlobalVariable *GV = M.getNamedGlobal(Name: Sym);
2290 if (!GV) {
2291 GV = new GlobalVariable(M, Ty, IsConst, GlobalValue::ExternalLinkage,
2292 nullptr, Sym, nullptr,
2293 GlobalValue::NotThreadLocal, AS);
2294 GV->setVisibility(Vis);
2295 }
2296 return GV;
2297 };
2298 // Section bounds are hidden i8 markers; raw_version is an i64 constant.
2299 auto *I8 = Type::getInt8Ty(C&: Ctx);
2300 auto Hidden = GlobalValue::HiddenVisibility;
2301 Constant *Fields[] = {Extern("__start___llvm_prf_names", I8, false, Hidden),
2302 Extern("__stop___llvm_prf_names", I8, false, Hidden),
2303 Extern("__start___llvm_prf_cnts", I8, false, Hidden),
2304 Extern("__stop___llvm_prf_cnts", I8, false, Hidden),
2305 Extern("__start___llvm_prf_data", I8, false, Hidden),
2306 Extern("__stop___llvm_prf_data", I8, false, Hidden),
2307 Extern("__start___llvm_prf_ucnts", I8, false, Hidden),
2308 Extern("__stop___llvm_prf_ucnts", I8, false, Hidden),
2309 Extern("__llvm_profile_raw_version",
2310 Type::getInt64Ty(C&: Ctx), true,
2311 GlobalValue::DefaultVisibility)};
2312 auto *PtrTy = PointerType::get(C&: Ctx, AddressSpace: AS);
2313 auto *STy = StructType::get(
2314 Context&: Ctx, Elements: {PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy});
2315 auto *GV = new GlobalVariable(M, STy, /*isConstant=*/true,
2316 GlobalValue::ExternalLinkage,
2317 ConstantStruct::get(T: STy, V: Fields), Name, nullptr,
2318 GlobalValue::NotThreadLocal, AS);
2319 GV->setVisibility(GlobalValue::ProtectedVisibility);
2320 return GV;
2321}
2322
2323void InstrLowerer::emitNameData() {
2324 if (ReferencedNames.empty())
2325 return;
2326
2327 std::string CompressedNameStr;
2328 if (Error E = collectPGOFuncNameStrings(NameVars: ReferencedNames, Result&: CompressedNameStr,
2329 doCompression: DoInstrProfNameCompression)) {
2330 report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false);
2331 }
2332
2333 auto &Ctx = M.getContext();
2334 auto *NamesVal =
2335 ConstantDataArray::getString(Context&: Ctx, Initializer: StringRef(CompressedNameStr), AddNull: false);
2336 std::string NamesVarName = std::string(getInstrProfNamesVarName());
2337 GlobalValue::LinkageTypes NamesLinkage = GlobalValue::PrivateLinkage;
2338 GlobalValue::VisibilityTypes NamesVisibility = GlobalValue::DefaultVisibility;
2339 std::string GPUCUIDPostfix;
2340 if (isGPUProfTarget(M)) {
2341 if (auto *GV = M.getNamedGlobal(Name: getInstrProfNamesVarPostfixVarName())) {
2342 if (auto *Init =
2343 dyn_cast_or_null<ConstantDataArray>(Val: GV->getInitializer())) {
2344 if (Init->isCString()) {
2345 GPUCUIDPostfix = Init->getAsCString().str();
2346 NamesVarName += GPUCUIDPostfix;
2347 NamesLinkage = GlobalValue::ExternalLinkage;
2348 NamesVisibility = GlobalValue::ProtectedVisibility;
2349 removeFromUsedLists(
2350 M, ShouldRemove: [GV](Constant *C) { return C->stripPointerCasts() == GV; });
2351 GV->eraseFromParent();
2352 }
2353 }
2354 }
2355 }
2356 NamesVar = new GlobalVariable(M, NamesVal->getType(), true, NamesLinkage,
2357 NamesVal, NamesVarName);
2358 NamesVar->setVisibility(NamesVisibility);
2359
2360 NamesSize = CompressedNameStr.size();
2361 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *NamesVar);
2362 std::string NamesSectionName =
2363 ProfileCorrelate == InstrProfCorrelator::BINARY
2364 ? getInstrProfSectionName(IPSK: IPSK_covname, OF: TT.getObjectFormat())
2365 : getInstrProfSectionName(IPSK: IPSK_name, OF: TT.getObjectFormat());
2366 NamesVar->setSection(NamesSectionName);
2367 // On COFF, it's important to reduce the alignment down to 1 to prevent the
2368 // linker from inserting padding before the start of the names section or
2369 // between names entries.
2370 NamesVar->setAlignment(Align(1));
2371 // NamesVar is used by runtime but not referenced via relocation by other
2372 // sections. Conservatively make it linker retained.
2373 UsedVars.push_back(x: NamesVar);
2374
2375 for (auto *NamePtr : ReferencedNames)
2376 NamePtr->eraseFromParent();
2377
2378 // Emit the device sections struct only when this TU produced profile data, so
2379 // its section start/stop references are backed by a real section.
2380 bool HasData = llvm::any_of(Range&: ProfileDataMap,
2381 P: [](const auto &KV) { return KV.second.DataVar; });
2382 if (!GPUCUIDPostfix.empty() && HasData)
2383 if (GlobalVariable *GV = emitGPUOffloadSectionsStruct(M, CUIDPostfix: GPUCUIDPostfix))
2384 CompilerUsedVars.push_back(x: GV);
2385}
2386
2387void InstrLowerer::emitVTableNames() {
2388 if (!EnableVTableValueProfiling || ReferencedVTables.empty())
2389 return;
2390
2391 // Collect the PGO names of referenced vtables and compress them.
2392 std::string CompressedVTableNames;
2393 if (Error E = collectVTableStrings(VTables: ReferencedVTables, Result&: CompressedVTableNames,
2394 doCompression: DoInstrProfNameCompression)) {
2395 report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false);
2396 }
2397
2398 auto &Ctx = M.getContext();
2399 auto *VTableNamesVal = ConstantDataArray::getString(
2400 Context&: Ctx, Initializer: StringRef(CompressedVTableNames), AddNull: false /* AddNull */);
2401 GlobalVariable *VTableNamesVar =
2402 new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */,
2403 GlobalValue::PrivateLinkage, VTableNamesVal,
2404 getInstrProfVTableNamesVarName());
2405 VTableNamesVar->setSection(
2406 getInstrProfSectionName(IPSK: IPSK_vname, OF: TT.getObjectFormat()));
2407 VTableNamesVar->setAlignment(Align(1));
2408 // Make VTableNames linker retained.
2409 UsedVars.push_back(x: VTableNamesVar);
2410}
2411
2412void InstrLowerer::emitRegistration() {
2413 if (!needsRuntimeRegistrationOfSectionRange(TT))
2414 return;
2415
2416 // Construct the function.
2417 auto *VoidTy = Type::getVoidTy(C&: M.getContext());
2418 auto *VoidPtrTy = PointerType::getUnqual(C&: M.getContext());
2419 auto *Int64Ty = Type::getInt64Ty(C&: M.getContext());
2420 auto *RegisterFTy = FunctionType::get(Result: VoidTy, isVarArg: false);
2421 auto *RegisterF = Function::Create(Ty: RegisterFTy, Linkage: GlobalValue::InternalLinkage,
2422 N: getInstrProfRegFuncsName(), M);
2423 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2424 if (Options.NoRedZone)
2425 RegisterF->addFnAttr(Kind: Attribute::NoRedZone);
2426
2427 auto *RuntimeRegisterTy = FunctionType::get(Result: VoidTy, Params: VoidPtrTy, isVarArg: false);
2428 auto *RuntimeRegisterF =
2429 Function::Create(Ty: RuntimeRegisterTy, Linkage: GlobalVariable::ExternalLinkage,
2430 N: getInstrProfRegFuncName(), M);
2431
2432 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: RegisterF));
2433 for (Value *Data : CompilerUsedVars)
2434 if (!isa<Function>(Val: Data))
2435 // Check for addrspace cast when profiling GPU
2436 IRB.CreateCall(Callee: RuntimeRegisterF,
2437 Args: IRB.CreatePointerBitCastOrAddrSpaceCast(V: Data, DestTy: VoidPtrTy));
2438 for (Value *Data : UsedVars)
2439 if (Data != NamesVar && !isa<Function>(Val: Data))
2440 IRB.CreateCall(Callee: RuntimeRegisterF,
2441 Args: IRB.CreatePointerBitCastOrAddrSpaceCast(V: Data, DestTy: VoidPtrTy));
2442
2443 if (NamesVar) {
2444 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
2445 auto *NamesRegisterTy =
2446 FunctionType::get(Result: VoidTy, Params: ArrayRef(ParamTypes), isVarArg: false);
2447 auto *NamesRegisterF =
2448 Function::Create(Ty: NamesRegisterTy, Linkage: GlobalVariable::ExternalLinkage,
2449 N: getInstrProfNamesRegFuncName(), M);
2450 IRB.CreateCall(Callee: NamesRegisterF, Args: {IRB.CreatePointerBitCastOrAddrSpaceCast(
2451 V: NamesVar, DestTy: VoidPtrTy),
2452 IRB.getInt64(C: NamesSize)});
2453 }
2454
2455 IRB.CreateRetVoid();
2456}
2457
2458bool InstrLowerer::emitRuntimeHook() {
2459 // GPU profiling data is read directly by the host offload runtime. We do not
2460 // need the standard runtime hook.
2461 if (TT.isGPU())
2462 return false;
2463
2464 // We expect the linker to be invoked with -u<hook_var> flag for Linux
2465 // in which case there is no need to emit the external variable.
2466 if (TT.isOSLinux() || TT.isOSAIX())
2467 return false;
2468
2469 // If the module's provided its own runtime, we don't need to do anything.
2470 if (M.getGlobalVariable(Name: getInstrProfRuntimeHookVarName()))
2471 return false;
2472
2473 // Declare an external variable that will pull in the runtime initialization.
2474 auto *Int32Ty = Type::getInt32Ty(C&: M.getContext());
2475 auto *Var =
2476 new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
2477 nullptr, getInstrProfRuntimeHookVarName());
2478 Var->setVisibility(GlobalValue::HiddenVisibility);
2479
2480 if (TT.isOSBinFormatELF() && !TT.isPS()) {
2481 // Mark the user variable as used so that it isn't stripped out.
2482 CompilerUsedVars.push_back(x: Var);
2483 } else {
2484 // Make a function that uses it.
2485 auto *User = Function::Create(Ty: FunctionType::get(Result: Int32Ty, isVarArg: false),
2486 Linkage: GlobalValue::LinkOnceODRLinkage,
2487 N: getInstrProfRuntimeHookVarUseFuncName(), M);
2488 User->addFnAttr(Kind: Attribute::NoInline);
2489 if (Options.NoRedZone)
2490 User->addFnAttr(Kind: Attribute::NoRedZone);
2491 User->setVisibility(GlobalValue::HiddenVisibility);
2492 if (TT.supportsCOMDAT())
2493 User->setComdat(M.getOrInsertComdat(Name: User->getName()));
2494 // Explicitly mark this function as cold since it is never called.
2495 User->setEntryCount(Count: 0);
2496
2497 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: User));
2498 auto *Load = IRB.CreateLoad(Ty: Int32Ty, Ptr: Var);
2499 IRB.CreateRet(V: Load);
2500
2501 // Mark the function as used so that it isn't stripped out.
2502 CompilerUsedVars.push_back(x: User);
2503 }
2504 return true;
2505}
2506
2507void InstrLowerer::emitUses() {
2508 // The metadata sections are parallel arrays. Optimizers (e.g.
2509 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
2510 // we conservatively retain all unconditionally in the compiler.
2511 //
2512 // On ELF and Mach-O, the linker can guarantee the associated sections will be
2513 // retained or discarded as a unit, so llvm.compiler.used is sufficient.
2514 // Similarly on COFF, if prof data is not referenced by code we use one comdat
2515 // and ensure this GC property as well. Otherwise, we have to conservatively
2516 // make all of the sections retained by the linker.
2517 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
2518 (TT.isOSBinFormatCOFF() && !DataReferencedByCode))
2519 appendToCompilerUsed(M, Values: CompilerUsedVars);
2520 else
2521 appendToUsed(M, Values: CompilerUsedVars);
2522
2523 // We do not add proper references from used metadata sections to NamesVar and
2524 // VNodesVar, so we have to be conservative and place them in llvm.used
2525 // regardless of the target,
2526 appendToUsed(M, Values: UsedVars);
2527}
2528
2529void InstrLowerer::emitInitialization() {
2530 // Create ProfileFileName variable. Don't don't this for the
2531 // context-sensitive instrumentation lowering: This lowering is after
2532 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
2533 // have already create the variable before LTO/ThinLTO linking.
2534 if (!IsCS)
2535 createProfileFileNameVar(M, InstrProfileOutput: Options.InstrProfileOutput);
2536 Function *RegisterF = M.getFunction(Name: getInstrProfRegFuncsName());
2537 if (!RegisterF)
2538 return;
2539
2540 // Create the initialization function.
2541 auto *VoidTy = Type::getVoidTy(C&: M.getContext());
2542 auto *F = Function::Create(Ty: FunctionType::get(Result: VoidTy, isVarArg: false),
2543 Linkage: GlobalValue::InternalLinkage,
2544 N: getInstrProfInitFuncName(), M);
2545 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2546 F->addFnAttr(Kind: Attribute::NoInline);
2547 if (Options.NoRedZone)
2548 F->addFnAttr(Kind: Attribute::NoRedZone);
2549
2550 // Add the basic block and the necessary calls.
2551 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: F));
2552 IRB.CreateCall(Callee: RegisterF, Args: {});
2553 IRB.CreateRetVoid();
2554
2555 appendToGlobalCtors(M, F, Priority: 0);
2556}
2557
2558namespace llvm {
2559// Create the variable for profile sampling.
2560void createProfileSamplingVar(Module &M) {
2561 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
2562 IntegerType *SamplingVarTy;
2563 Constant *ValueZero;
2564 if (getSampledInstrumentationConfig().UseShort) {
2565 SamplingVarTy = Type::getInt16Ty(C&: M.getContext());
2566 ValueZero = Constant::getIntegerValue(Ty: SamplingVarTy, V: APInt(16, 0));
2567 } else {
2568 SamplingVarTy = Type::getInt32Ty(C&: M.getContext());
2569 ValueZero = Constant::getIntegerValue(Ty: SamplingVarTy, V: APInt(32, 0));
2570 }
2571 auto SamplingVar = new GlobalVariable(
2572 M, SamplingVarTy, false, GlobalValue::WeakAnyLinkage, ValueZero, VarName);
2573 SamplingVar->setVisibility(GlobalValue::DefaultVisibility);
2574 SamplingVar->setThreadLocal(true);
2575 Triple TT(M.getTargetTriple());
2576 if (TT.supportsCOMDAT()) {
2577 SamplingVar->setLinkage(GlobalValue::ExternalLinkage);
2578 SamplingVar->setComdat(M.getOrInsertComdat(Name: VarName));
2579 }
2580 appendToCompilerUsed(M, Values: SamplingVar);
2581}
2582} // namespace llvm
2583
2584// For GPU targets: Allocate contiguous arrays for all profile data.
2585// This solves the linker reordering problem by using ONE symbol per section
2586// type, so there's nothing for the linker to reorder.
2587StructType *InstrLowerer::getProfileDataTy() {
2588 if (ProfileDataTy)
2589 return ProfileDataTy;
2590
2591 auto &Ctx = M.getContext();
2592 auto *IntPtrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
2593 auto *Int16Ty = Type::getInt16Ty(C&: Ctx);
2594 auto *Int16ArrayTy = ArrayType::get(ElementType: Int16Ty, NumElements: IPVK_Last + 1);
2595 Type *DataTypes[] = {
2596#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
2597#include "llvm/ProfileData/InstrProfData.inc"
2598 };
2599 ProfileDataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes));
2600 return ProfileDataTy;
2601}
2602