1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by an instrumentor.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/Twine.h"
21#include "llvm/Analysis/BlockFrequencyInfo.h"
22#include "llvm/Analysis/BranchProbabilityInfo.h"
23#include "llvm/Analysis/CFG.h"
24#include "llvm/Analysis/LoopInfo.h"
25#include "llvm/Analysis/TargetLibraryInfo.h"
26#include "llvm/IR/Attributes.h"
27#include "llvm/IR/BasicBlock.h"
28#include "llvm/IR/CFG.h"
29#include "llvm/IR/Constant.h"
30#include "llvm/IR/Constants.h"
31#include "llvm/IR/CycleInfo.h"
32#include "llvm/IR/DIBuilder.h"
33#include "llvm/IR/DerivedTypes.h"
34#include "llvm/IR/DiagnosticInfo.h"
35#include "llvm/IR/Function.h"
36#include "llvm/IR/GlobalAlias.h"
37#include "llvm/IR/GlobalValue.h"
38#include "llvm/IR/GlobalVariable.h"
39#include "llvm/IR/IRBuilder.h"
40#include "llvm/IR/InstIterator.h"
41#include "llvm/IR/Instruction.h"
42#include "llvm/IR/Instructions.h"
43#include "llvm/IR/IntrinsicInst.h"
44#include "llvm/IR/Intrinsics.h"
45#include "llvm/IR/MDBuilder.h"
46#include "llvm/IR/Module.h"
47#include "llvm/IR/RuntimeLibcalls.h"
48#include "llvm/IR/Type.h"
49#include "llvm/Pass.h"
50#include "llvm/ProfileData/InstrProf.h"
51#include "llvm/ProfileData/InstrProfCorrelator.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CommandLine.h"
54#include "llvm/Support/Compiler.h"
55#include "llvm/Support/Error.h"
56#include "llvm/Support/ErrorHandling.h"
57#include "llvm/TargetParser/Triple.h"
58#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
59#include "llvm/Transforms/Utils/BasicBlockUtils.h"
60#include "llvm/Transforms/Utils/Instrumentation.h"
61#include "llvm/Transforms/Utils/ModuleUtils.h"
62#include "llvm/Transforms/Utils/SSAUpdater.h"
63#include <algorithm>
64#include <cassert>
65#include <cstdint>
66#include <string>
67
68using namespace llvm;
69
70#define DEBUG_TYPE "instrprof"
71
72namespace llvm {
73// Command line option to enable vtable value profiling. Defined in
74// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
75extern cl::opt<bool> EnableVTableValueProfiling;
76cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate(
77 "profile-correlate",
78 cl::desc("Use debug info or binary file to correlate profiles."),
79 cl::init(Val: InstrProfCorrelator::NONE),
80 cl::values(clEnumValN(InstrProfCorrelator::NONE, "",
81 "No profile correlation"),
82 clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info",
83 "Use debug info to correlate"),
84 clEnumValN(InstrProfCorrelator::BINARY, "binary",
85 "Use binary to correlate")));
86} // namespace llvm
87
88bool llvm::isProfileCorrelationEnabled() {
89 return ProfileCorrelate != InstrProfCorrelator::NONE;
90}
91
92namespace {
93
94cl::opt<bool> DoHashBasedCounterSplit(
95 "hash-based-counter-split",
96 cl::desc("Rename counter variable of a comdat function based on cfg hash"),
97 cl::init(Val: true));
98
99cl::opt<bool>
100 RuntimeCounterRelocation("runtime-counter-relocation",
101 cl::desc("Enable relocating counters at runtime."),
102 cl::init(Val: false));
103
104cl::opt<bool> ValueProfileStaticAlloc(
105 "vp-static-alloc",
106 cl::desc("Do static counter allocation for value profiler"),
107 cl::init(Val: true));
108
109cl::opt<double> NumCountersPerValueSite(
110 "vp-counters-per-site",
111 cl::desc("The average number of profile counters allocated "
112 "per value profiling site."),
113 // This is set to a very small value because in real programs, only
114 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
115 // For those sites with non-zero profile, the average number of targets
116 // is usually smaller than 2.
117 cl::init(Val: 1.0));
118
119cl::opt<bool> AtomicCounterUpdateAll(
120 "instrprof-atomic-counter-update-all",
121 cl::desc("Make all profile counter updates atomic (for testing only)"),
122 cl::init(Val: false));
123
124cl::opt<bool> VerifyAtomicPromotion(
125 "verify-atomic-counter-promoted",
126 cl::desc("Check that all profile counter updates were made atomic; no-op "
127 "if atomic updates are not requested (-fprofile-update=atomic)"),
128 cl::init(Val: false));
129
130cl::opt<bool> AtomicCounterUpdatePromoted(
131 "atomic-counter-update-promoted",
132 cl::desc("Do counter update using atomic fetch add "
133 " for promoted counters only"),
134 cl::init(Val: false));
135
136cl::opt<bool> AtomicFirstCounter(
137 "atomic-first-counter",
138 cl::desc("Use atomic fetch add for first counter in a function (usually "
139 "the entry counter)"),
140 cl::init(Val: false));
141
142cl::opt<bool> ConditionalCounterUpdate(
143 "conditional-counter-update",
144 cl::desc("Do conditional counter updates in single byte counters mode)"),
145 cl::init(Val: false));
146
147// If the option is not specified, the default behavior about whether
148// counter promotion is done depends on how instrumentation lowering
149// pipeline is setup, i.e., the default value of true of this option
150// does not mean the promotion will be done by default. Explicitly
151// setting this option can override the default behavior.
152cl::opt<bool> DoCounterPromotion("do-counter-promotion",
153 cl::desc("Do counter register promotion"),
154 cl::init(Val: false));
155cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
156 "max-counter-promotions-per-loop", cl::init(Val: 20),
157 cl::desc("Max number counter promotions per loop to avoid"
158 " increasing register pressure too much"));
159
160// A debug option
161cl::opt<int>
162 MaxNumOfPromotions("max-counter-promotions", cl::init(Val: -1),
163 cl::desc("Max number of allowed counter promotions"));
164
165cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
166 "speculative-counter-promotion-max-exiting", cl::init(Val: 3),
167 cl::desc("The max number of exiting blocks of a loop to allow "
168 " speculative counter promotion"));
169
170cl::opt<bool> SpeculativeCounterPromotionToLoop(
171 "speculative-counter-promotion-to-loop",
172 cl::desc("When the option is false, if the target block is in a loop, "
173 "the promotion will be disallowed unless the promoted counter "
174 " update can be further/iteratively promoted into an acyclic "
175 " region."));
176
177static cl::opt<unsigned> OffloadPGOSampling(
178 "offload-pgo-sampling",
179 cl::desc("Log2 of the sampling period for offload PGO instrumentation. "
180 "Only 1 in every 2^N blocks is instrumented. "
181 "0 = all blocks, 1 = 50%, 2 = 25%, 3 = 12.5% (default). "
182 "Higher values reduce overhead at the cost of sparser profiles."),
183 cl::init(Val: 3));
184
185cl::opt<bool> IterativeCounterPromotion(
186 "iterative-counter-promotion", cl::init(Val: true),
187 cl::desc("Allow counter promotion across the whole loop nest."));
188
189cl::opt<bool> SkipRetExitBlock(
190 "skip-ret-exit-block", cl::init(Val: true),
191 cl::desc("Suppress counter promotion if exit blocks contain ret."));
192
193static cl::opt<bool> SampledInstr("sampled-instrumentation",
194 cl::desc("Do PGO instrumentation sampling"));
195
196static cl::opt<unsigned> SampledInstrPeriod(
197 "sampled-instr-period",
198 cl::desc("Set the profile instrumentation sample period. A sample period "
199 "of 0 is invalid. For each sample period, a fixed number of "
200 "consecutive samples will be recorded. The number is controlled "
201 "by 'sampled-instr-burst-duration' flag. The default sample "
202 "period of 65536 is optimized for generating efficient code that "
203 "leverages unsigned short integer wrapping in overflow, but this "
204 "is disabled under simple sampling (burst duration = 1)."),
205 cl::init(USHRT_MAX + 1));
206
207static cl::opt<unsigned> SampledInstrBurstDuration(
208 "sampled-instr-burst-duration",
209 cl::desc("Set the profile instrumentation burst duration, which can range "
210 "from 1 to the value of 'sampled-instr-period' (0 is invalid). "
211 "This number of samples will be recorded for each "
212 "'sampled-instr-period' count update. Setting to 1 enables simple "
213 "sampling, in which case it is recommended to set "
214 "'sampled-instr-period' to a prime number."),
215 cl::init(Val: 200));
216
217struct SampledInstrumentationConfig {
218 unsigned BurstDuration;
219 unsigned Period;
220 bool UseShort;
221 bool IsSimpleSampling;
222 bool IsFastSampling;
223};
224
225static SampledInstrumentationConfig getSampledInstrumentationConfig() {
226 SampledInstrumentationConfig config;
227 config.BurstDuration = SampledInstrBurstDuration.getValue();
228 config.Period = SampledInstrPeriod.getValue();
229 if (config.BurstDuration > config.Period)
230 report_fatal_error(
231 reason: "SampledBurstDuration must be less than or equal to SampledPeriod");
232 if (config.Period == 0 || config.BurstDuration == 0)
233 report_fatal_error(
234 reason: "SampledPeriod and SampledBurstDuration must be greater than 0");
235 config.IsSimpleSampling = (config.BurstDuration == 1);
236 // If (BurstDuration == 1 && Period == 65536), generate the simple sampling
237 // style code.
238 config.IsFastSampling =
239 (!config.IsSimpleSampling && config.Period == USHRT_MAX + 1);
240 config.UseShort = (config.Period <= USHRT_MAX) || config.IsFastSampling;
241 return config;
242}
243
244using LoadStorePair = std::pair<Instruction *, Instruction *>;
245
246static void makeAtomic(Instruction *Load, Instruction *Store) {
247 auto *Addition = dyn_cast<BinaryOperator>(Val: Store->getOperand(i: 0));
248 assert(Addition && Addition->getOpcode() == Instruction::BinaryOps::Add);
249 auto *Addend = Addition->getOperand(i_nocapture: 1);
250
251 IRBuilder<> Builder(Load);
252 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Store->getOperand(i: 1), Val: Addend,
253 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
254 Store->eraseFromParent();
255 Addition->eraseFromParent();
256 Load->eraseFromParent();
257}
258
259static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {
260 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(Val: M.getModuleFlag(Key: Flag));
261 if (!MD)
262 return 0;
263
264 // If the flag is a ConstantAsMetadata, it should be an integer representable
265 // in 64-bits.
266 return cast<ConstantInt>(Val: MD->getValue())->getZExtValue();
267}
268
269static bool enablesValueProfiling(const Module &M) {
270 return isIRPGOFlagSet(M: &M) ||
271 getIntModuleFlagOrZero(M, Flag: "EnableValueProfiling") != 0;
272}
273
274// Conservatively returns true if value profiling is enabled.
275static bool profDataReferencedByCode(const Module &M) {
276 return enablesValueProfiling(M);
277}
278
279class InstrLowerer final {
280public:
281 InstrLowerer(Module &M, const InstrProfOptions &Options,
282 std::function<const TargetLibraryInfo &(Function &F)> GetTLI,
283 bool IsCS)
284 : M(M), Options(Options), TT(M.getTargetTriple()), IsCS(IsCS),
285 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {}
286
287 bool lower();
288
289private:
290 Module &M;
291 const InstrProfOptions Options;
292 const Triple TT;
293 // Is this lowering for the context-sensitive instrumentation.
294 const bool IsCS;
295
296 std::function<const TargetLibraryInfo &(Function &F)> GetTLI;
297
298 const bool DataReferencedByCode;
299
300 struct PerFunctionProfileData {
301 uint32_t NumValueSites[IPVK_Last + 1] = {};
302 GlobalVariable *RegionCounters = nullptr;
303 GlobalVariable *UniformCounters =
304 nullptr; // Per-block uniform-entry counters
305 GlobalVariable *DataVar = nullptr;
306 GlobalVariable *RegionBitmaps = nullptr;
307 uint32_t NumBitmapBytes = 0;
308
309 PerFunctionProfileData() = default;
310 };
311 DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
312 // Key is virtual table variable, value is 'VTableProfData' in the form of
313 // GlobalVariable.
314 DenseMap<GlobalVariable *, GlobalVariable *> VTableDataMap;
315 /// If runtime relocation is enabled, this maps functions to the load
316 /// instruction that produces the profile relocation bias.
317 DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;
318 std::vector<GlobalValue *> CompilerUsedVars;
319 std::vector<GlobalValue *> UsedVars;
320 std::vector<GlobalVariable *> ReferencedNames;
321 // The list of virtual table variables of which the VTableProfData is
322 // collected.
323 std::vector<GlobalVariable *> ReferencedVTables;
324 GlobalVariable *NamesVar = nullptr;
325 size_t NamesSize = 0;
326
327 StructType *ProfileDataTy = nullptr;
328
329 // vector of counter load/store pairs to be register promoted.
330 std::vector<LoadStorePair> PromotionCandidates;
331
332 int64_t TotalCountersPromoted = 0;
333
334 // Per-function cache of invariant values for GPU PGO instrumentation.
335 // Computed once at the function entry and reused across all instrumentation
336 // points to avoid redundant IR and help the optimizer.
337 struct GPUPGOInvariants {
338 Value *Matched = nullptr;
339 bool WaveSizeStored = false;
340 };
341 DenseMap<Function *, GPUPGOInvariants> GPUInvariantsCache;
342
343 /// Emit invariant PGO values at the function entry block and cache them.
344 GPUPGOInvariants &getOrCreateGPUInvariants(Function *F);
345
346 /// Lower instrumentation intrinsics in the function. Returns true if there
347 /// any lowering.
348 bool lowerIntrinsics(Function *F);
349
350 /// Register-promote counter loads and stores in loops.
351 void promoteCounterLoadStores(Function *F);
352
353 /// Returns true if relocating counters at runtime is enabled.
354 bool isRuntimeCounterRelocationEnabled() const;
355
356 /// Returns true if profile counter update register promotion is enabled.
357 bool isCounterPromotionEnabled() const;
358
359 /// Returns true if profile counter updates should be atomic.
360 bool isAtomic() const;
361
362 /// Return true if profile sampling is enabled.
363 bool isSamplingEnabled() const;
364
365 /// Count the number of instrumented value sites for the function.
366 void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
367
368 /// Replace instrprof.value.profile with a call to runtime library.
369 void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
370
371 /// Replace instrprof.cover with a store instruction to the coverage byte.
372 void lowerCover(InstrProfCoverInst *Inc);
373
374 /// Replace instrprof.timestamp with a call to
375 /// INSTR_PROF_PROFILE_SET_TIMESTAMP.
376 void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction);
377
378 /// Replace instrprof.increment with an increment of the appropriate value.
379 void lowerIncrement(InstrProfIncrementInst *Inc);
380
381 /// Force emitting of name vars for unused functions.
382 void lowerCoverageData(GlobalVariable *CoverageNamesVar);
383
384 /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction
385 /// using the index represented by the a temp value into a bitmap.
386 void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins);
387
388 /// Get the Bias value for data to access mmap-ed area.
389 /// Create it if it hasn't been seen.
390 GlobalVariable *getOrCreateBiasVar(StringRef VarName);
391
392 /// Compute the address of the counter value that this profiling instruction
393 /// acts on.
394 Value *getCounterAddress(InstrProfCntrInstBase *I);
395
396 /// Lower the incremental instructions under profile sampling predicates.
397 void doSampling(Instruction *I);
398
399 /// Get the region counters for an increment, creating them if necessary.
400 ///
401 /// If the counter array doesn't yet exist, the profile data variables
402 /// referring to them will also be created.
403 GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc);
404
405 /// Get the uniform entry counters for GPU divergence tracking.
406 /// These counters track how often blocks are entered with all lanes active.
407 GlobalVariable *getOrCreateUniformCounters(InstrProfCntrInstBase *Inc);
408
409 /// Create the region counters.
410 GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc,
411 StringRef Name,
412 GlobalValue::LinkageTypes Linkage);
413
414 /// Compute the address of the test vector bitmap that this profiling
415 /// instruction acts on.
416 Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I);
417
418 /// Get the region bitmaps for an increment, creating them if necessary.
419 ///
420 /// If the bitmap array doesn't yet exist, the profile data variables
421 /// referring to them will also be created.
422 GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc);
423
424 /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with
425 /// an MC/DC Decision region. The number of bytes required is indicated by
426 /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called
427 /// as part of setupProfileSection() and is conceptually very similar to
428 /// what is done for profile data counters in createRegionCounters().
429 GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
430 StringRef Name,
431 GlobalValue::LinkageTypes Linkage);
432
433 /// Set Comdat property of GV, if required.
434 void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName);
435
436 /// Setup the sections into which counters and bitmaps are allocated.
437 GlobalVariable *setupProfileSection(InstrProfInstBase *Inc,
438 InstrProfSectKind IPSK);
439
440 /// Create INSTR_PROF_DATA variable for counters and bitmaps.
441 void createDataVariable(InstrProfCntrInstBase *Inc);
442
443 /// Get the counters for virtual table values, creating them if necessary.
444 void getOrCreateVTableProfData(GlobalVariable *GV);
445
446 /// Emit the section with compressed function names.
447 void emitNameData();
448
449 /// Emit the section with compressed vtable names.
450 void emitVTableNames();
451
452 /// Emit value nodes section for value profiling.
453 void emitVNodes();
454
455 /// Emit runtime registration functions for each profile data variable.
456 void emitRegistration();
457
458 /// Emit the necessary plumbing to pull in the runtime initialization.
459 /// Returns true if a change was made.
460 bool emitRuntimeHook();
461
462 /// Add uses of our data variables and runtime hook.
463 void emitUses();
464
465 /// Create a static initializer for our data, on platforms that need it,
466 /// and for any profile output file that was specified.
467 void emitInitialization();
468
469 /// Return the __llvm_profile_data struct type.
470 StructType *getProfileDataTy();
471};
472
473///
474/// A helper class to promote one counter RMW operation in the loop
475/// into register update.
476///
477/// RWM update for the counter will be sinked out of the loop after
478/// the transformation.
479///
480class PGOCounterPromoterHelper : public LoadAndStorePromoter {
481public:
482 PGOCounterPromoterHelper(
483 Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,
484 BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
485 ArrayRef<Instruction *> InsertPts,
486 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
487 LoopInfo &LI, bool IsAtomic)
488 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
489 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI),
490 IsAtomic(IsAtomic) {
491 assert(isa<LoadInst>(L));
492 assert(isa<StoreInst>(S));
493 SSA.AddAvailableValue(BB: PH, V: Init);
494 }
495
496 void doExtraRewritesBeforeFinalDeletion() override {
497 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
498 BasicBlock *ExitBlock = ExitBlocks[i];
499 Instruction *InsertPos = InsertPts[i];
500 // Get LiveIn value into the ExitBlock. If there are multiple
501 // predecessors, the value is defined by a PHI node in this
502 // block.
503 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(BB: ExitBlock);
504 Value *Addr = cast<StoreInst>(Val: Store)->getPointerOperand();
505 Type *Ty = LiveInValue->getType();
506 IRBuilder<> Builder(InsertPos);
507 if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Val: Addr)) {
508 // If isRuntimeCounterRelocationEnabled() is true then the address of
509 // the store instruction is computed with two instructions in
510 // InstrProfiling::getCounterAddress(). We need to copy those
511 // instructions to this block to compute Addr correctly.
512 // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
513 // %Addr = inttoptr i64 %BiasAdd to i64*
514 auto *OrigBiasInst = dyn_cast<BinaryOperator>(Val: AddrInst->getOperand(i_nocapture: 0));
515 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
516 Value *BiasInst = Builder.Insert(I: OrigBiasInst->clone());
517 Addr = Builder.CreateIntToPtr(V: BiasInst,
518 DestTy: PointerType::getUnqual(C&: Ty->getContext()));
519 }
520 auto *TargetLoop =
521 IterativeCounterPromotion ? LI.getLoopFor(BB: ExitBlock) : nullptr;
522 // Generate the relaxed atomic RMW if we've asked for it and no more
523 // promotion is possible.
524 if ((IsAtomic && !TargetLoop) || AtomicCounterUpdatePromoted)
525 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: LiveInValue,
526 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
527 else {
528 LoadInst *OldVal = Builder.CreateLoad(Ty, Ptr: Addr, Name: "pgocount.promoted");
529 auto *NewVal = Builder.CreateAdd(LHS: OldVal, RHS: LiveInValue);
530 auto *NewStore = Builder.CreateStore(Val: NewVal, Ptr: Addr);
531
532 // Now update the parent loop's candidate list:
533 if (TargetLoop)
534 LoopToCandidates[TargetLoop].emplace_back(Args&: OldVal, Args&: NewStore);
535 }
536 }
537 }
538
539private:
540 Instruction *Store;
541 ArrayRef<BasicBlock *> ExitBlocks;
542 ArrayRef<Instruction *> InsertPts;
543 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
544 LoopInfo &LI;
545 const bool IsAtomic;
546};
547
548/// A helper class to do register promotion for all profile counter
549/// updates in a loop.
550///
551class PGOCounterPromoter {
552public:
553 PGOCounterPromoter(
554 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
555 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI, bool IsAtomic)
556 : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI),
557 IsAtomic(IsAtomic) {
558
559 // Skip collection of ExitBlocks and InsertPts for loops that will not be
560 // able to have counters promoted.
561 SmallVector<BasicBlock *, 8> LoopExitBlocks;
562 SmallPtrSet<BasicBlock *, 8> BlockSet;
563
564 L.getExitBlocks(ExitBlocks&: LoopExitBlocks);
565 if (!isPromotionPossible(LP: &L, LoopExitBlocks))
566 return;
567
568 for (BasicBlock *ExitBlock : LoopExitBlocks) {
569 if (BlockSet.insert(Ptr: ExitBlock).second &&
570 llvm::none_of(Range: predecessors(BB: ExitBlock), P: [&](const BasicBlock *Pred) {
571 return llvm::isPresplitCoroSuspendExitEdge(Src: *Pred, Dest: *ExitBlock);
572 })) {
573 ExitBlocks.push_back(Elt: ExitBlock);
574 InsertPts.push_back(Elt: &*ExitBlock->getFirstInsertionPt());
575 }
576 }
577 }
578
579 bool run(int64_t *NumPromoted) {
580 // Move L's candidates out of LoopToCandidates before promoting them, as
581 // promoting a counter to an enclosing loop may insert a new key into
582 // LoopToCandidates and trigger DenseMap::grow().
583 auto &OrigCandidates = LoopToCandidates[&L];
584 SmallVector<LoadStorePair, 8> Candidates = std::move(OrigCandidates);
585 OrigCandidates.clear();
586 bool RC = promoteCandidates(Candidates, NumPromoted);
587 assert(LoopToCandidates[&L].empty() &&
588 "Did not expect new candidates to be added to current loop");
589 // In certain case, e.g. with -fprofile-update=atomic, we want to generate
590 // atomic updates of the PGO counters, but also perform promotion of these
591 // updates out of loops to reduce train time. The strategy is:
592 // 1) generate non-atomic load-increment-store sequence of instructions
593 // during lowerIntrinsics phase,
594 // 2) perform the promotion (in promoteCandidates function), then
595 // 3) convert all (promoted and unpromotable) updates to atomicRMW.
596 // This requires that promoted candidates are set to nullptr in the
597 // Candidates array by the promoteCandidates() function.
598 if (IsAtomic)
599 for (auto &Cand : Candidates)
600 if (Cand.first != nullptr && Cand.second != nullptr)
601 makeAtomic(Load: Cand.first, Store: Cand.second);
602 return RC;
603 }
604
605private:
606 bool promoteCandidates(SmallVectorImpl<LoadStorePair> &Candidates,
607 int64_t *NumPromoted) {
608 // Skip 'infinite' loops:
609 if (ExitBlocks.size() == 0)
610 return false;
611
612 // Skip if any of the ExitBlocks contains a ret instruction.
613 // This is to prevent dumping of incomplete profile -- if the
614 // the loop is a long running loop and dump is called in the middle
615 // of the loop, the result profile is incomplete.
616 // FIXME: add other heuristics to detect long running loops.
617 if (SkipRetExitBlock) {
618 for (auto *BB : ExitBlocks)
619 if (isa<ReturnInst>(Val: BB->getTerminator()))
620 return false;
621 }
622
623 unsigned MaxProm = getMaxNumOfPromotionsInLoop(LP: &L);
624 if (MaxProm == 0)
625 return false;
626
627 unsigned Promoted = 0;
628 for (auto &Cand : Candidates) {
629 SmallVector<PHINode *, 4> NewPHIs;
630 SSAUpdater SSA(&NewPHIs);
631 Value *InitVal = ConstantInt::get(Ty: Cand.first->getType(), V: 0);
632
633 // If BFI is set, we will use it to guide the promotions.
634 if (BFI) {
635 auto *BB = Cand.first->getParent();
636 auto InstrCount = BFI->getBlockProfileCount(BB);
637 if (!InstrCount)
638 continue;
639 auto PreheaderCount = BFI->getBlockProfileCount(BB: L.getLoopPreheader());
640 // If the average loop trip count is not greater than 1.5, we skip
641 // promotion.
642 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
643 continue;
644 }
645
646 PGOCounterPromoterHelper Promoter(
647 Cand.first, Cand.second, SSA, InitVal, L.getLoopPreheader(),
648 ExitBlocks, InsertPts, LoopToCandidates, LI, IsAtomic);
649 Promoter.run(Insts: SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
650
651 Cand = {nullptr, nullptr};
652
653 Promoted++;
654 if (Promoted >= MaxProm)
655 break;
656
657 (*NumPromoted)++;
658 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
659 break;
660 }
661
662 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
663 << L.getLoopDepth() << ")\n");
664 return Promoted != 0;
665 }
666
667private:
668 bool allowSpeculativeCounterPromotion(Loop *LP) {
669 SmallVector<BasicBlock *, 8> ExitingBlocks;
670 L.getExitingBlocks(ExitingBlocks);
671 // Not considierered speculative.
672 if (ExitingBlocks.size() == 1)
673 return true;
674 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
675 return false;
676 return true;
677 }
678
679 // Check whether the loop satisfies the basic conditions needed to perform
680 // Counter Promotions.
681 bool
682 isPromotionPossible(Loop *LP,
683 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
684 // We can't insert into a catchswitch.
685 if (llvm::any_of(Range: LoopExitBlocks, P: [](BasicBlock *Exit) {
686 return isa<CatchSwitchInst>(Val: Exit->getTerminator());
687 }))
688 return false;
689
690 if (!LP->hasDedicatedExits())
691 return false;
692
693 BasicBlock *PH = LP->getLoopPreheader();
694 if (!PH)
695 return false;
696
697 return true;
698 }
699
700 // Returns the max number of Counter Promotions for LP.
701 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
702 SmallVector<BasicBlock *, 8> LoopExitBlocks;
703 LP->getExitBlocks(ExitBlocks&: LoopExitBlocks);
704 if (!isPromotionPossible(LP, LoopExitBlocks))
705 return 0;
706
707 SmallVector<BasicBlock *, 8> ExitingBlocks;
708 LP->getExitingBlocks(ExitingBlocks);
709
710 // If BFI is set, we do more aggressive promotions based on BFI.
711 if (BFI)
712 return (unsigned)-1;
713
714 // Not considierered speculative.
715 if (ExitingBlocks.size() == 1)
716 return MaxNumOfPromotionsPerLoop;
717
718 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
719 return 0;
720
721 // Whether the target block is in a loop does not matter:
722 if (SpeculativeCounterPromotionToLoop)
723 return MaxNumOfPromotionsPerLoop;
724
725 // Now check the target block:
726 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
727 for (auto *TargetBlock : LoopExitBlocks) {
728 auto *TargetLoop = LI.getLoopFor(BB: TargetBlock);
729 if (!TargetLoop)
730 continue;
731 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(LP: TargetLoop);
732 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
733 MaxProm =
734 std::min(a: MaxProm, b: std::max(a: MaxPromForTarget, b: PendingCandsInTarget) -
735 PendingCandsInTarget);
736 }
737 return MaxProm;
738 }
739
740 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
741 SmallVector<BasicBlock *, 8> ExitBlocks;
742 SmallVector<Instruction *, 8> InsertPts;
743 Loop &L;
744 LoopInfo &LI;
745 BlockFrequencyInfo *BFI;
746 const bool IsAtomic; // Whether to convert counter updates to atomics.
747};
748
749enum class ValueProfilingCallType {
750 // Individual values are tracked. Currently used for indiret call target
751 // profiling.
752 Default,
753
754 // MemOp: the memop size value profiling.
755 MemOp
756};
757
758} // end anonymous namespace
759
760PreservedAnalyses InstrProfilingLoweringPass::run(Module &M,
761 ModuleAnalysisManager &AM) {
762 FunctionAnalysisManager &FAM =
763 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
764 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
765 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
766 };
767 InstrLowerer Lowerer(M, Options, GetTLI, IsCS);
768 if (!Lowerer.lower())
769 return PreservedAnalyses::all();
770
771 return PreservedAnalyses::none();
772}
773
774//
775// Perform instrumentation sampling.
776//
777// There are 3 favors of sampling:
778// (1) Full burst sampling: We transform:
779// Increment_Instruction;
780// to:
781// if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
782// Increment_Instruction;
783// }
784// __llvm_profile_sampling__ += 1;
785// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
786// __llvm_profile_sampling__ = 0;
787// }
788//
789// "__llvm_profile_sampling__" is a thread-local global shared by all PGO
790// counters (value-instrumentation and edge instrumentation).
791//
792// (2) Fast burst sampling:
793// "__llvm_profile_sampling__" variable is an unsigned type, meaning it will
794// wrap around to zero when overflows. In this case, the second check is
795// unnecessary, so we won't generate check2 when the SampledInstrPeriod is
796// set to 65536 (64K). The code after:
797// if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
798// Increment_Instruction;
799// }
800// __llvm_profile_sampling__ += 1;
801//
802// (3) Simple sampling:
803// When SampledInstrBurstDuration is set to 1, we do a simple sampling:
804// __llvm_profile_sampling__ += 1;
805// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
806// __llvm_profile_sampling__ = 0;
807// Increment_Instruction;
808// }
809//
810// Note that, the code snippet after the transformation can still be counter
811// promoted. However, with sampling enabled, counter updates are expected to
812// be infrequent, making the benefits of counter promotion negligible.
813// Moreover, counter promotion can potentially cause issues in server
814// applications, particularly when the counters are dumped without a clean
815// exit. To mitigate this risk, counter promotion is disabled by default when
816// sampling is enabled. This behavior can be overridden using the internal
817// option.
818void InstrLowerer::doSampling(Instruction *I) {
819 if (!isSamplingEnabled())
820 return;
821
822 SampledInstrumentationConfig config = getSampledInstrumentationConfig();
823 auto GetConstant = [&config](IRBuilder<> &Builder, uint32_t C) {
824 if (config.UseShort)
825 return Builder.getInt16(C);
826 else
827 return Builder.getInt32(C);
828 };
829
830 IntegerType *SamplingVarTy;
831 if (config.UseShort)
832 SamplingVarTy = Type::getInt16Ty(C&: M.getContext());
833 else
834 SamplingVarTy = Type::getInt32Ty(C&: M.getContext());
835 auto *SamplingVar =
836 M.getGlobalVariable(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
837 assert(SamplingVar && "SamplingVar not set properly");
838
839 // Create the condition for checking the burst duration.
840 Instruction *SamplingVarIncr;
841 Value *NewSamplingVarVal;
842 MDBuilder MDB(I->getContext());
843 MDNode *BranchWeight;
844 IRBuilder<> CondBuilder(I);
845 auto *LoadSamplingVar = CondBuilder.CreateLoad(Ty: SamplingVarTy, Ptr: SamplingVar);
846 if (config.IsSimpleSampling) {
847 // For the simple sampling, just create the load and increments.
848 IRBuilder<> IncBuilder(I);
849 NewSamplingVarVal =
850 IncBuilder.CreateAdd(LHS: LoadSamplingVar, RHS: GetConstant(IncBuilder, 1));
851 SamplingVarIncr = IncBuilder.CreateStore(Val: NewSamplingVarVal, Ptr: SamplingVar);
852 } else {
853 // For the burst-sampling, create the conditional update.
854 auto *DurationCond = CondBuilder.CreateICmpULE(
855 LHS: LoadSamplingVar, RHS: GetConstant(CondBuilder, config.BurstDuration - 1));
856 BranchWeight = MDB.createBranchWeights(
857 TrueWeight: config.BurstDuration, FalseWeight: config.Period - config.BurstDuration);
858 Instruction *ThenTerm = SplitBlockAndInsertIfThen(
859 Cond: DurationCond, SplitBefore: I, /* Unreachable */ false, BranchWeights: BranchWeight);
860 IRBuilder<> IncBuilder(I);
861 NewSamplingVarVal =
862 IncBuilder.CreateAdd(LHS: LoadSamplingVar, RHS: GetConstant(IncBuilder, 1));
863 SamplingVarIncr = IncBuilder.CreateStore(Val: NewSamplingVarVal, Ptr: SamplingVar);
864 I->moveBefore(InsertPos: ThenTerm->getIterator());
865 }
866
867 if (config.IsFastSampling)
868 return;
869
870 // Create the condition for checking the period.
871 Instruction *ThenTerm, *ElseTerm;
872 IRBuilder<> PeriodCondBuilder(SamplingVarIncr);
873 auto *PeriodCond = PeriodCondBuilder.CreateICmpUGE(
874 LHS: NewSamplingVarVal, RHS: GetConstant(PeriodCondBuilder, config.Period));
875 BranchWeight = MDB.createBranchWeights(TrueWeight: 1, FalseWeight: config.Period - 1);
876 SplitBlockAndInsertIfThenElse(Cond: PeriodCond, SplitBefore: SamplingVarIncr, ThenTerm: &ThenTerm,
877 ElseTerm: &ElseTerm, BranchWeights: BranchWeight);
878
879 // For the simple sampling, the counter update happens in sampling var reset.
880 if (config.IsSimpleSampling)
881 I->moveBefore(InsertPos: ThenTerm->getIterator());
882
883 IRBuilder<> ResetBuilder(ThenTerm);
884 ResetBuilder.CreateStore(Val: GetConstant(ResetBuilder, 0), Ptr: SamplingVar);
885 SamplingVarIncr->moveBefore(InsertPos: ElseTerm->getIterator());
886}
887
888bool InstrLowerer::lowerIntrinsics(Function *F) {
889 bool MadeChange = false;
890 PromotionCandidates.clear();
891 SmallVector<InstrProfInstBase *, 8> InstrProfInsts;
892
893 // To ensure compatibility with sampling, we save the intrinsics into
894 // a buffer to prevent potential breakage of the iterator (as the
895 // intrinsics will be moved to a different BB).
896 for (BasicBlock &BB : *F) {
897 for (Instruction &Instr : llvm::make_early_inc_range(Range&: BB)) {
898 if (auto *IP = dyn_cast<InstrProfInstBase>(Val: &Instr))
899 InstrProfInsts.push_back(Elt: IP);
900 }
901 }
902
903 for (auto *Instr : InstrProfInsts) {
904 doSampling(I: Instr);
905 if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(Val: Instr)) {
906 lowerIncrement(Inc: IPIS);
907 MadeChange = true;
908 } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(Val: Instr)) {
909 lowerIncrement(Inc: IPI);
910 MadeChange = true;
911 } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(Val: Instr)) {
912 lowerTimestamp(TimestampInstruction: IPC);
913 MadeChange = true;
914 } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(Val: Instr)) {
915 lowerCover(Inc: IPC);
916 MadeChange = true;
917 } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(Val: Instr)) {
918 lowerValueProfileInst(Ins: IPVP);
919 MadeChange = true;
920 } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(Val: Instr)) {
921 IPMP->eraseFromParent();
922 MadeChange = true;
923 } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(Val: Instr)) {
924 lowerMCDCTestVectorBitmapUpdate(Ins: IPBU);
925 MadeChange = true;
926 }
927 }
928
929 if (!MadeChange)
930 return false;
931
932 promoteCounterLoadStores(F);
933 return true;
934}
935
936bool InstrLowerer::isRuntimeCounterRelocationEnabled() const {
937 // Mach-O don't support weak external references.
938 if (TT.isOSBinFormatMachO())
939 return false;
940
941 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
942 return RuntimeCounterRelocation;
943
944 // Fuchsia uses runtime counter relocation by default.
945 return TT.isOSFuchsia();
946}
947
948bool InstrLowerer::isSamplingEnabled() const {
949 if (SampledInstr.getNumOccurrences() > 0)
950 return SampledInstr;
951 return Options.Sampling;
952}
953
954bool InstrLowerer::isCounterPromotionEnabled() const {
955 if (DoCounterPromotion.getNumOccurrences() > 0)
956 return DoCounterPromotion;
957 return Options.DoCounterPromotion;
958}
959
960bool InstrLowerer::isAtomic() const {
961 return Options.Atomic || AtomicCounterUpdateAll;
962}
963
964static void doAtomicCheck(Function *F) {
965 for (const llvm::Instruction &I : llvm::instructions(F)) {
966 const Value *Addr = nullptr;
967 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
968 Addr = LI->getOperand(i_nocapture: 0);
969 else if (const StoreInst *LI = dyn_cast<StoreInst>(Val: &I))
970 Addr = LI->getOperand(i_nocapture: 1);
971
972 if (Addr && Addr->stripInBoundsOffsets()->getName().starts_with(
973 Prefix: getInstrProfCountersVarPrefix())) {
974 LLVM_DEBUG(dbgs() << "Missed candidate: "; I.dump());
975 report_fatal_error(reason: "Candidate load/store not converted to atomic");
976 }
977 }
978}
979
980void InstrLowerer::promoteCounterLoadStores(Function *F) {
981 if (!isCounterPromotionEnabled())
982 return;
983
984 CycleInfo CI;
985 CI.compute(F&: *F);
986 LoopInfo LI;
987 LI.analyze(F);
988 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
989
990 std::unique_ptr<BlockFrequencyInfo> BFI;
991 if (Options.UseBFIInPromotion) {
992 std::unique_ptr<BranchProbabilityInfo> BPI;
993 BPI.reset(p: new BranchProbabilityInfo(*F, CI, &GetTLI(*F)));
994 BFI.reset(p: new BlockFrequencyInfo(*F, *BPI, CI));
995 }
996
997 for (const auto &LoadStore : PromotionCandidates) {
998 auto *CounterLoad = LoadStore.first;
999 auto *CounterStore = LoadStore.second;
1000 BasicBlock *BB = CounterLoad->getParent();
1001 Loop *ParentLoop = LI.getLoopFor(BB);
1002 if (!ParentLoop) {
1003 if (isAtomic())
1004 makeAtomic(Load: CounterLoad, Store: CounterStore);
1005 continue;
1006 }
1007 LoopPromotionCandidates[ParentLoop].emplace_back(Args&: CounterLoad, Args&: CounterStore);
1008 }
1009
1010 SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
1011
1012 // Do a post-order traversal of the loops so that counter updates can be
1013 // iteratively hoisted outside the loop nest.
1014 for (auto *Loop : llvm::reverse(C&: Loops)) {
1015 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get(),
1016 isAtomic());
1017 Promoter.run(NumPromoted: &TotalCountersPromoted);
1018 }
1019
1020 if (isAtomic() && VerifyAtomicPromotion)
1021 doAtomicCheck(F);
1022}
1023
1024static bool needsRuntimeHookUnconditionally(const Triple &TT) {
1025 // On Fuchsia, we only need runtime hook if any counters are present.
1026 if (TT.isOSFuchsia())
1027 return false;
1028
1029 return true;
1030}
1031
1032/// Check if the module contains uses of any profiling intrinsics.
1033static bool containsProfilingIntrinsics(Module &M) {
1034 auto containsIntrinsic = [&](int ID) {
1035 if (auto *F = Intrinsic::getDeclarationIfExists(M: &M, id: ID))
1036 return !F->use_empty();
1037 return false;
1038 };
1039 return containsIntrinsic(Intrinsic::instrprof_cover) ||
1040 containsIntrinsic(Intrinsic::instrprof_increment) ||
1041 containsIntrinsic(Intrinsic::instrprof_increment_step) ||
1042 containsIntrinsic(Intrinsic::instrprof_timestamp) ||
1043 containsIntrinsic(Intrinsic::instrprof_value_profile);
1044}
1045
1046bool InstrLowerer::lower() {
1047 bool MadeChange = false;
1048 bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
1049 if (NeedsRuntimeHook)
1050 MadeChange = emitRuntimeHook();
1051
1052 if (!IsCS && isSamplingEnabled())
1053 createProfileSamplingVar(M);
1054
1055 bool ContainsProfiling = containsProfilingIntrinsics(M);
1056 GlobalVariable *CoverageNamesVar =
1057 M.getNamedGlobal(Name: getCoverageUnusedNamesVarName());
1058 // Improve compile time by avoiding linear scans when there is no work.
1059 if (!ContainsProfiling && !CoverageNamesVar)
1060 return MadeChange;
1061
1062 // We did not know how many value sites there would be inside
1063 // the instrumented function. This is counting the number of instrumented
1064 // target value sites to enter it as field in the profile data variable.
1065 for (Function &F : M) {
1066 InstrProfCntrInstBase *FirstProfInst = nullptr;
1067 for (BasicBlock &BB : F) {
1068 for (auto I = BB.begin(), E = BB.end(); I != E; I++) {
1069 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Val&: I))
1070 computeNumValueSiteCounts(Ins: Ind);
1071 else {
1072 if (FirstProfInst == nullptr &&
1073 (isa<InstrProfIncrementInst>(Val: I) || isa<InstrProfCoverInst>(Val: I)))
1074 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(Val&: I);
1075 // If the MCDCBitmapParameters intrinsic seen, create the bitmaps.
1076 if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(Val&: I))
1077 static_cast<void>(getOrCreateRegionBitmaps(Inc: Params));
1078 }
1079 }
1080 }
1081
1082 // Use a profile intrinsic to create the region counters and data variable.
1083 // Also create the data variable based on the MCDCParams.
1084 if (FirstProfInst != nullptr) {
1085 static_cast<void>(getOrCreateRegionCounters(Inc: FirstProfInst));
1086 }
1087 }
1088
1089 if (EnableVTableValueProfiling)
1090 for (GlobalVariable &GV : M.globals())
1091 // Global variables with type metadata are virtual table variables.
1092 if (GV.hasMetadata(KindID: LLVMContext::MD_type))
1093 getOrCreateVTableProfData(GV: &GV);
1094
1095 for (Function &F : M)
1096 MadeChange |= lowerIntrinsics(F: &F);
1097
1098 if (CoverageNamesVar) {
1099 lowerCoverageData(CoverageNamesVar);
1100 MadeChange = true;
1101 }
1102
1103 if (!MadeChange)
1104 return false;
1105
1106 emitVNodes();
1107 emitNameData();
1108 emitVTableNames();
1109
1110 // Emit runtime hook for the cases where the target does not unconditionally
1111 // require pulling in profile runtime, and coverage is enabled on code that is
1112 // not eliminated by the front-end, e.g. unused functions with internal
1113 // linkage.
1114 if (!NeedsRuntimeHook && ContainsProfiling)
1115 emitRuntimeHook();
1116
1117 emitRegistration();
1118 emitUses();
1119 emitInitialization();
1120 return true;
1121}
1122
1123static FunctionCallee getOrInsertValueProfilingCall(
1124 Module &M, const TargetLibraryInfo &TLI,
1125 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
1126 LLVMContext &Ctx = M.getContext();
1127 auto *ReturnTy = Type::getVoidTy(C&: M.getContext());
1128
1129 AttributeList AL;
1130 if (auto AK = TLI.getExtAttrForI32Param(Signed: false))
1131 AL = AL.addParamAttribute(C&: M.getContext(), ArgNo: 2, Kind: AK);
1132
1133 assert((CallType == ValueProfilingCallType::Default ||
1134 CallType == ValueProfilingCallType::MemOp) &&
1135 "Must be Default or MemOp");
1136 Type *ParamTypes[] = {
1137#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
1138#include "llvm/ProfileData/InstrProfData.inc"
1139 };
1140 auto *ValueProfilingCallTy =
1141 FunctionType::get(Result: ReturnTy, Params: ArrayRef(ParamTypes), isVarArg: false);
1142 StringRef FuncName = CallType == ValueProfilingCallType::Default
1143 ? getInstrProfValueProfFuncName()
1144 : getInstrProfValueProfMemOpFuncName();
1145 return M.getOrInsertFunction(Name: FuncName, T: ValueProfilingCallTy, AttributeList: AL);
1146}
1147
1148void InstrLowerer::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
1149 GlobalVariable *Name = Ind->getName();
1150 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1151 uint64_t Index = Ind->getIndex()->getZExtValue();
1152 auto &PD = ProfileDataMap[Name];
1153 PD.NumValueSites[ValueKind] =
1154 std::max(a: PD.NumValueSites[ValueKind], b: (uint32_t)(Index + 1));
1155}
1156
1157void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
1158 // TODO: Value profiling heavily depends on the data section which is omitted
1159 // in lightweight mode. We need to move the value profile pointer to the
1160 // Counter struct to get this working.
1161 assert(
1162 ProfileCorrelate == InstrProfCorrelator::NONE &&
1163 "Value profiling is not yet supported with lightweight instrumentation");
1164 GlobalVariable *Name = Ind->getName();
1165 auto It = ProfileDataMap.find(Val: Name);
1166 assert(It != ProfileDataMap.end() && It->second.DataVar &&
1167 "value profiling detected in function with no counter increment");
1168
1169 GlobalVariable *DataVar = It->second.DataVar;
1170 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1171 uint64_t Index = Ind->getIndex()->getZExtValue();
1172 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
1173 Index += It->second.NumValueSites[Kind];
1174
1175 IRBuilder<> Builder(Ind);
1176 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
1177 llvm::InstrProfValueKind::IPVK_MemOPSize);
1178 CallInst *Call = nullptr;
1179 auto *TLI = &GetTLI(*Ind->getFunction());
1180 auto *NormalizedDataVarPtr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1181 C: DataVar, Ty: PointerType::get(C&: M.getContext(), AddressSpace: 0));
1182
1183 // To support value profiling calls within Windows exception handlers, funclet
1184 // information contained within operand bundles needs to be copied over to
1185 // the library call. This is required for the IR to be processed by the
1186 // WinEHPrepare pass.
1187 SmallVector<OperandBundleDef, 1> OpBundles;
1188 Ind->getOperandBundlesAsDefs(Defs&: OpBundles);
1189 if (!IsMemOpSize) {
1190 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1191 Builder.getInt32(C: Index)};
1192 Call = Builder.CreateCall(Callee: getOrInsertValueProfilingCall(M, TLI: *TLI), Args,
1193 OpBundles);
1194 } else {
1195 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1196 Builder.getInt32(C: Index)};
1197 Call = Builder.CreateCall(
1198 Callee: getOrInsertValueProfilingCall(M, TLI: *TLI, CallType: ValueProfilingCallType::MemOp),
1199 Args, OpBundles);
1200 }
1201 if (auto AK = TLI->getExtAttrForI32Param(Signed: false))
1202 Call->addParamAttr(ArgNo: 2, Kind: AK);
1203 Ind->replaceAllUsesWith(V: Call);
1204 Ind->eraseFromParent();
1205}
1206
1207GlobalVariable *InstrLowerer::getOrCreateBiasVar(StringRef VarName) {
1208 GlobalVariable *Bias = M.getGlobalVariable(Name: VarName);
1209 if (Bias)
1210 return Bias;
1211
1212 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1213
1214 // Compiler must define this variable when runtime counter relocation
1215 // is being used. Runtime has a weak external reference that is used
1216 // to check whether that's the case or not.
1217 Bias = new GlobalVariable(M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
1218 Constant::getNullValue(Ty: Int64Ty), VarName);
1219 Bias->setVisibility(GlobalVariable::HiddenVisibility);
1220 // A definition that's weak (linkonce_odr) without being in a COMDAT
1221 // section wouldn't lead to link errors, but it would lead to a dead
1222 // data word from every TU but one. Putting it in COMDAT ensures there
1223 // will be exactly one data slot in the link.
1224 if (TT.supportsCOMDAT())
1225 Bias->setComdat(M.getOrInsertComdat(Name: VarName));
1226
1227 return Bias;
1228}
1229
1230Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
1231 auto *Counters = getOrCreateRegionCounters(Inc: I);
1232 IRBuilder<> Builder(I);
1233
1234 if (isa<InstrProfTimestampInst>(Val: I))
1235 Counters->setAlignment(Align(8));
1236
1237 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
1238 Ty: Counters->getValueType(), Ptr: Counters, Idx0: 0, Idx1: I->getIndex()->getZExtValue());
1239
1240 if (!isRuntimeCounterRelocationEnabled())
1241 return Addr;
1242
1243 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1244 Function *Fn = I->getParent()->getParent();
1245 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
1246 if (!BiasLI) {
1247 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1248 auto *Bias = getOrCreateBiasVar(VarName: getInstrProfCounterBiasVarName());
1249 BiasLI = EntryBuilder.CreateLoad(Ty: Int64Ty, Ptr: Bias, Name: "profc_bias");
1250 // Bias doesn't change after startup.
1251 BiasLI->setMetadata(KindID: LLVMContext::MD_invariant_load,
1252 Node: MDNode::get(Context&: M.getContext(), MDs: {}));
1253 }
1254 auto *Add = Builder.CreateAdd(LHS: Builder.CreatePtrToInt(V: Addr, DestTy: Int64Ty), RHS: BiasLI);
1255 return Builder.CreateIntToPtr(V: Add, DestTy: Addr->getType());
1256}
1257
1258Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
1259 auto *Bitmaps = getOrCreateRegionBitmaps(Inc: I);
1260 if (!isRuntimeCounterRelocationEnabled())
1261 return Bitmaps;
1262
1263 // Put BiasLI onto the entry block.
1264 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1265 Function *Fn = I->getFunction();
1266 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1267 auto *Bias = getOrCreateBiasVar(VarName: getInstrProfBitmapBiasVarName());
1268 auto *BiasLI = EntryBuilder.CreateLoad(Ty: Int64Ty, Ptr: Bias, Name: "profbm_bias");
1269 // Assume BiasLI invariant (in the function at least)
1270 BiasLI->setMetadata(KindID: LLVMContext::MD_invariant_load,
1271 Node: MDNode::get(Context&: M.getContext(), MDs: {}));
1272
1273 // Add Bias to Bitmaps and put it before the intrinsic.
1274 IRBuilder<> Builder(I);
1275 return Builder.CreatePtrAdd(Ptr: Bitmaps, Offset: BiasLI, Name: "profbm_addr");
1276}
1277
1278void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
1279 auto *Addr = getCounterAddress(I: CoverInstruction);
1280 IRBuilder<> Builder(CoverInstruction);
1281 if (ConditionalCounterUpdate) {
1282 Instruction *SplitBefore = CoverInstruction->getNextNode();
1283 auto &Ctx = CoverInstruction->getParent()->getContext();
1284 auto *Int8Ty = llvm::Type::getInt8Ty(C&: Ctx);
1285 Value *Load = Builder.CreateLoad(Ty: Int8Ty, Ptr: Addr, Name: "pgocount");
1286 Value *Cmp = Builder.CreateIsNotNull(Arg: Load, Name: "pgocount.ifnonzero");
1287 Instruction *ThenBranch =
1288 SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore, Unreachable: false);
1289 Builder.SetInsertPoint(ThenBranch);
1290 }
1291
1292 // We store zero to represent that this block is covered.
1293 Builder.CreateStore(Val: Builder.getInt8(C: 0), Ptr: Addr);
1294 CoverInstruction->eraseFromParent();
1295}
1296
1297void InstrLowerer::lowerTimestamp(
1298 InstrProfTimestampInst *TimestampInstruction) {
1299 assert(TimestampInstruction->getIndex()->isNullValue() &&
1300 "timestamp probes are always the first probe for a function");
1301 auto &Ctx = M.getContext();
1302 auto *TimestampAddr = getCounterAddress(I: TimestampInstruction);
1303 IRBuilder<> Builder(TimestampInstruction);
1304 auto *CalleeTy =
1305 FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: TimestampAddr->getType(), isVarArg: false);
1306 auto Callee = M.getOrInsertFunction(
1307 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), T: CalleeTy);
1308 Builder.CreateCall(Callee, Args: {TimestampAddr});
1309 TimestampInstruction->eraseFromParent();
1310}
1311
1312InstrLowerer::GPUPGOInvariants &
1313InstrLowerer::getOrCreateGPUInvariants(Function *F) {
1314 auto It = GPUInvariantsCache.find(Val: F);
1315 if (It != GPUInvariantsCache.end())
1316 return It->second;
1317
1318 LLVMContext &Context = M.getContext();
1319 auto *Int32Ty = Type::getInt32Ty(C&: Context);
1320
1321 BasicBlock &EntryBB = F->getEntryBlock();
1322 IRBuilder<> Builder(&*EntryBB.getFirstInsertionPt());
1323
1324 Value *Matched = ConstantInt::getTrue(Context);
1325 if (OffloadPGOSampling > 0) {
1326 FunctionCallee IsSampledFn =
1327 M.getOrInsertFunction(Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(
1328 CallImpl: RTLIB::impl___llvm_profile_sampling_gpu),
1329 RetTy: Int32Ty, Args: Int32Ty);
1330 Value *SampledInt = Builder.CreateCall(
1331 Callee: IsSampledFn, Args: {ConstantInt::get(Ty: Int32Ty, V: OffloadPGOSampling)},
1332 Name: "pgo.sampled");
1333 Matched = Builder.CreateICmpNE(LHS: SampledInt, RHS: ConstantInt::get(Ty: Int32Ty, V: 0),
1334 Name: "pgo.matched");
1335 }
1336
1337 auto &Inv = GPUInvariantsCache[F];
1338 Inv.Matched = Matched;
1339 return Inv;
1340}
1341
1342void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
1343 IRBuilder<> Builder(Inc);
1344 if (isGPUProfTarget(M)) {
1345 Function *F = Inc->getFunction();
1346 auto &Inv = getOrCreateGPUInvariants(F);
1347
1348 LLVMContext &Context = M.getContext();
1349 auto *Int64Ty = Type::getInt64Ty(C&: Context);
1350 auto *PtrTy = PointerType::getUnqual(C&: Context);
1351
1352 auto *Addr = getCounterAddress(I: Inc);
1353
1354 // Store the device wave/warp size into the profile data struct once per
1355 // function. AMDGPU folds llvm.amdgcn.wavefrontsize to the subtarget's
1356 // constant; other GPUs use their fixed warp size.
1357 if (!Inv.WaveSizeStored) {
1358 Inv.WaveSizeStored = true;
1359 GlobalVariable *NamePtr = Inc->getName();
1360 auto &PD = ProfileDataMap[NamePtr];
1361 if (PD.DataVar) {
1362 IRBuilder<> EntryBuilder(&*F->getEntryBlock().getFirstInsertionPt());
1363 Value *WaveSize16 = nullptr;
1364 // Look the intrinsic up by name so this target-agnostic pass does not
1365 // pull in IntrinsicsAMDGPU.h. AMDGPU folds the intrinsic to the
1366 // subtarget's wavefront size; other GPUs fall back to a 32-lane warp.
1367 if (TT.isAMDGPU()) {
1368 Intrinsic::ID WaveSizeID =
1369 Intrinsic::lookupIntrinsicID(Name: "llvm.amdgcn.wavefrontsize");
1370 if (WaveSizeID != Intrinsic::not_intrinsic) {
1371 Function *WaveSizeFn =
1372 Intrinsic::getOrInsertDeclaration(M: &M, id: WaveSizeID);
1373 Value *WaveSize = EntryBuilder.CreateCall(Callee: WaveSizeFn);
1374 WaveSize16 = EntryBuilder.CreateTrunc(
1375 V: WaveSize, DestTy: Type::getInt16Ty(C&: Context), Name: "wavesize.i16");
1376 }
1377 }
1378 if (!WaveSize16)
1379 WaveSize16 = ConstantInt::get(Ty: Type::getInt16Ty(C&: Context), V: 32);
1380 Value *WaveSizeAddr = EntryBuilder.CreateStructGEP(
1381 Ty: PD.DataVar->getValueType(), Ptr: PD.DataVar, Idx: 9, Name: "profd.wavesize");
1382 EntryBuilder.CreateStore(Val: WaveSize16, Ptr: WaveSizeAddr);
1383 }
1384 }
1385
1386 GlobalVariable *UniformCounters = getOrCreateUniformCounters(Inc);
1387 Value *UniformAddrArg = ConstantPointerNull::get(T: PtrTy);
1388 if (UniformCounters) {
1389 Value *UniformIndices[] = {Builder.getInt32(C: 0), Inc->getIndex()};
1390 Value *UniformAddr = Builder.CreateInBoundsGEP(
1391 Ty: UniformCounters->getValueType(), Ptr: UniformCounters, IdxList: UniformIndices,
1392 Name: "unifctr.addr");
1393 UniformAddrArg =
1394 Builder.CreatePointerBitCastOrAddrSpaceCast(V: UniformAddr, DestTy: PtrTy);
1395 }
1396 Value *CastAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Addr, DestTy: PtrTy);
1397 Value *StepI64 =
1398 Builder.CreateZExtOrTrunc(V: Inc->getStep(), DestTy: Int64Ty, Name: "step.i64");
1399
1400 auto *CalleeTy = FunctionType::get(Result: Type::getVoidTy(C&: Context),
1401 Params: {PtrTy, PtrTy, Int64Ty}, isVarArg: false);
1402 FunctionCallee Callee =
1403 M.getOrInsertFunction(Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(
1404 CallImpl: RTLIB::impl___llvm_profile_instrument_gpu),
1405 T: CalleeTy);
1406
1407 if (OffloadPGOSampling > 0) {
1408 BasicBlock *CurBB = Builder.GetInsertBlock();
1409 BasicBlock *ContBB =
1410 CurBB->splitBasicBlock(I: BasicBlock::iterator(Inc), BBName: "po_cont");
1411 BasicBlock *ThenBB = BasicBlock::Create(Context, Name: "po_then", Parent: F);
1412
1413 CurBB->getTerminator()->eraseFromParent();
1414 IRBuilder<> HeadBuilder(CurBB);
1415 HeadBuilder.CreateCondBr(Cond: Inv.Matched, True: ThenBB, False: ContBB);
1416
1417 IRBuilder<> ThenBuilder(ThenBB);
1418 ThenBuilder.CreateCall(Callee, Args: {CastAddr, UniformAddrArg, StepI64});
1419 ThenBuilder.CreateBr(Dest: ContBB);
1420 } else {
1421 Builder.CreateCall(Callee, Args: {CastAddr, UniformAddrArg, StepI64});
1422 }
1423 Inc->eraseFromParent();
1424 return;
1425 }
1426
1427 auto *Addr = getCounterAddress(I: Inc);
1428 // If promotion is enabled then delay generating atomic updates until
1429 // after promotion is done.
1430 if ((!isCounterPromotionEnabled() && isAtomic()) ||
1431 (Inc->getIndex()->isNullValue() && AtomicFirstCounter)) {
1432 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: Inc->getStep(),
1433 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
1434 } else {
1435 Value *IncStep = Inc->getStep();
1436 Value *Load = Builder.CreateLoad(Ty: IncStep->getType(), Ptr: Addr, Name: "pgocount");
1437 auto *Count = Builder.CreateAdd(LHS: Load, RHS: Inc->getStep());
1438 auto *Store = Builder.CreateStore(Val: Count, Ptr: Addr);
1439 if (isCounterPromotionEnabled())
1440 PromotionCandidates.emplace_back(args: cast<Instruction>(Val: Load), args&: Store);
1441 }
1442 Inc->eraseFromParent();
1443}
1444
1445void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
1446 ConstantArray *Names =
1447 cast<ConstantArray>(Val: CoverageNamesVar->getInitializer());
1448 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
1449 Constant *NC = Names->getOperand(i_nocapture: I);
1450 Value *V = NC->stripPointerCasts();
1451 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
1452 GlobalVariable *Name = cast<GlobalVariable>(Val: V);
1453
1454 Name->setLinkage(GlobalValue::PrivateLinkage);
1455 ReferencedNames.push_back(x: Name);
1456 if (isa<ConstantExpr>(Val: NC))
1457 NC->dropAllReferences();
1458 }
1459 CoverageNamesVar->eraseFromParent();
1460}
1461
1462void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
1463 InstrProfMCDCTVBitmapUpdate *Update) {
1464 auto &Ctx = M.getContext();
1465 IRBuilder<> Builder(Update);
1466 auto *Int8Ty = Type::getInt8Ty(C&: Ctx);
1467 auto *Int32Ty = Type::getInt32Ty(C&: Ctx);
1468 auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
1469 auto *BitmapAddr = getBitmapAddress(I: Update);
1470
1471 // Load Temp Val + BitmapIdx.
1472 // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
1473 auto *Temp = Builder.CreateAdd(
1474 LHS: Builder.CreateLoad(Ty: Int32Ty, Ptr: MCDCCondBitmapAddr, Name: "mcdc.temp"),
1475 RHS: Update->getBitmapIndex());
1476
1477 // Calculate byte offset using div8.
1478 // %1 = lshr i32 %mcdc.temp, 3
1479 auto *BitmapByteOffset = Builder.CreateLShr(LHS: Temp, RHS: 0x3);
1480
1481 // Add byte offset to section base byte address.
1482 // %4 = getelementptr inbounds i8, ptr @__profbm_test, i32 %1
1483 auto *BitmapByteAddr =
1484 Builder.CreateInBoundsPtrAdd(Ptr: BitmapAddr, Offset: BitmapByteOffset);
1485
1486 // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8)
1487 // %5 = and i32 %mcdc.temp, 7
1488 // %6 = trunc i32 %5 to i8
1489 auto *BitToSet = Builder.CreateTrunc(V: Builder.CreateAnd(LHS: Temp, RHS: 0x7), DestTy: Int8Ty);
1490
1491 // Shift bit offset left to form a bitmap.
1492 // %7 = shl i8 1, %6
1493 auto *ShiftedVal = Builder.CreateShl(LHS: Builder.getInt8(C: 0x1), RHS: BitToSet);
1494
1495 // Load profile bitmap byte.
1496 // %mcdc.bits = load i8, ptr %4, align 1
1497 auto *Bitmap = Builder.CreateLoad(Ty: Int8Ty, Ptr: BitmapByteAddr, Name: "mcdc.bits");
1498
1499 if (isAtomic()) {
1500 // If ((Bitmap & Val) != Val), then execute atomic (Bitmap |= Val).
1501 // Note, just-loaded Bitmap might not be up-to-date. Use it just for
1502 // early testing.
1503 auto *Masked = Builder.CreateAnd(LHS: Bitmap, RHS: ShiftedVal);
1504 auto *ShouldStore = Builder.CreateICmpNE(LHS: Masked, RHS: ShiftedVal);
1505
1506 // Assume updating will be rare.
1507 auto *Unlikely = MDBuilder(Ctx).createUnlikelyBranchWeights();
1508 Instruction *ThenBranch =
1509 SplitBlockAndInsertIfThen(Cond: ShouldStore, SplitBefore: Update, Unreachable: false, BranchWeights: Unlikely);
1510
1511 // Execute if (unlikely(ShouldStore)).
1512 Builder.SetInsertPoint(ThenBranch);
1513 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: BitmapByteAddr, Val: ShiftedVal,
1514 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
1515 } else {
1516 // Perform logical OR of profile bitmap byte and shifted bit offset.
1517 // %8 = or i8 %mcdc.bits, %7
1518 auto *Result = Builder.CreateOr(LHS: Bitmap, RHS: ShiftedVal);
1519
1520 // Store the updated profile bitmap byte.
1521 // store i8 %8, ptr %3, align 1
1522 Builder.CreateStore(Val: Result, Ptr: BitmapByteAddr);
1523 }
1524
1525 Update->eraseFromParent();
1526}
1527
1528/// Get the name of a profiling variable for a particular function.
1529static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
1530 bool &Renamed) {
1531 StringRef NamePrefix = getInstrProfNameVarPrefix();
1532 StringRef Name = Inc->getName()->getName().substr(Start: NamePrefix.size());
1533 Function *F = Inc->getParent()->getParent();
1534 Module *M = F->getParent();
1535 if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
1536 !canRenameComdatFunc(F: *F)) {
1537 Renamed = false;
1538 return (Prefix + Name).str();
1539 }
1540 Renamed = true;
1541 uint64_t FuncHash = Inc->getHash()->getZExtValue();
1542 SmallVector<char, 24> HashPostfix;
1543 if (Name.ends_with(Suffix: (Twine(".") + Twine(FuncHash)).toStringRef(Out&: HashPostfix)))
1544 return (Prefix + Name).str();
1545 return (Prefix + Name + "." + Twine(FuncHash)).str();
1546}
1547
1548static inline bool shouldRecordFunctionAddr(Function *F) {
1549 // Only record function addresses if IR PGO is enabled or if clang value
1550 // profiling is enabled. Recording function addresses greatly increases object
1551 // file size, because it prevents the inliner from deleting functions that
1552 // have been inlined everywhere.
1553 if (!profDataReferencedByCode(M: *F->getParent()))
1554 return false;
1555
1556 // Check the linkage
1557 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
1558 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
1559 !HasAvailableExternallyLinkage)
1560 return true;
1561
1562 // A function marked 'alwaysinline' with available_externally linkage can't
1563 // have its address taken. Doing so would create an undefined external ref to
1564 // the function, which would fail to link.
1565 if (HasAvailableExternallyLinkage &&
1566 F->hasFnAttribute(Kind: Attribute::AlwaysInline))
1567 return false;
1568
1569 // Prohibit function address recording if the function is both internal and
1570 // COMDAT. This avoids the profile data variable referencing internal symbols
1571 // in COMDAT.
1572 if (F->hasLocalLinkage() && F->hasComdat())
1573 return false;
1574
1575 // Check uses of this function for other than direct calls or invokes to it.
1576 // Inline virtual functions have linkeOnceODR linkage. When a key method
1577 // exists, the vtable will only be emitted in the TU where the key method
1578 // is defined. In a TU where vtable is not available, the function won't
1579 // be 'addresstaken'. If its address is not recorded here, the profile data
1580 // with missing address may be picked by the linker leading to missing
1581 // indirect call target info.
1582 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
1583}
1584
1585static inline bool shouldUsePublicSymbol(Function *Fn) {
1586 // It isn't legal to make an alias of this function at all
1587 if (Fn->isDeclarationForLinker())
1588 return true;
1589
1590 // Symbols with local linkage can just use the symbol directly without
1591 // introducing relocations
1592 if (Fn->hasLocalLinkage())
1593 return true;
1594
1595 // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some
1596 // unfavorable interaction between the new alias and the alias renaming done
1597 // in LowerTypeTests under ThinLTO. For comdat functions that would normally
1598 // be deduplicated, but the renaming scheme ends up preventing renaming, since
1599 // it creates unique names for each alias, resulting in duplicated symbols. In
1600 // the future, we should update the CFI related passes to migrate these
1601 // aliases to the same module as the jump-table they refer to will be defined.
1602 if (Fn->hasMetadata(KindID: LLVMContext::MD_type))
1603 return true;
1604
1605 // For comdat functions, an alias would need the same linkage as the original
1606 // function and hidden visibility. There is no point in adding an alias with
1607 // identical linkage an visibility to avoid introducing symbolic relocations.
1608 if (Fn->hasComdat() &&
1609 (Fn->getVisibility() == GlobalValue::VisibilityTypes::HiddenVisibility))
1610 return true;
1611
1612 // its OK to use an alias
1613 return false;
1614}
1615
1616static inline Constant *getFuncAddrForProfData(Function *Fn) {
1617 auto *Int8PtrTy = PointerType::getUnqual(C&: Fn->getContext());
1618 // Store a nullptr in __llvm_profd, if we shouldn't use a real address
1619 if (!shouldRecordFunctionAddr(F: Fn))
1620 return ConstantPointerNull::get(T: Int8PtrTy);
1621
1622 // If we can't use an alias, we must use the public symbol, even though this
1623 // may require a symbolic relocation.
1624 if (shouldUsePublicSymbol(Fn))
1625 return Fn;
1626
1627 // For GPU targets, weak functions cannot use private aliases because
1628 // LTO may pick a different TU's copy, leaving the alias undefined
1629 if (isGPUProfTarget(M: *Fn->getParent()) &&
1630 GlobalValue::isWeakForLinker(Linkage: Fn->getLinkage()))
1631 return Fn;
1632
1633 // When possible use a private alias to avoid symbolic relocations.
1634 auto *GA = GlobalAlias::create(Linkage: GlobalValue::LinkageTypes::PrivateLinkage,
1635 Name: Fn->getName() + ".local", Aliasee: Fn);
1636
1637 // When the instrumented function is a COMDAT function, we cannot use a
1638 // private alias. If we did, we would create reference to a local label in
1639 // this function's section. If this version of the function isn't selected by
1640 // the linker, then the metadata would introduce a reference to a discarded
1641 // section. So, for COMDAT functions, we need to adjust the linkage of the
1642 // alias. Using hidden visibility avoids a dynamic relocation and an entry in
1643 // the dynamic symbol table.
1644 //
1645 // Note that this handles COMDAT functions with visibility other than Hidden,
1646 // since that case is covered in shouldUsePublicSymbol()
1647 if (Fn->hasComdat()) {
1648 GA->setLinkage(Fn->getLinkage());
1649 GA->setVisibility(GlobalValue::VisibilityTypes::HiddenVisibility);
1650 }
1651
1652 // appendToCompilerUsed(*Fn->getParent(), {GA});
1653
1654 return GA;
1655}
1656
1657static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
1658 // NVPTX is an ELF target but PTX does not expose sections or linker symbols.
1659 if (TT.isNVPTX())
1660 return true;
1661
1662 // compiler-rt uses linker support to get data/counters/name start/end for
1663 // ELF, COFF, Mach-O, XCOFF, and Wasm.
1664 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
1665 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF() ||
1666 TT.isOSBinFormatWasm())
1667 return false;
1668
1669 return true;
1670}
1671
1672void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO,
1673 StringRef CounterGroupName) {
1674 // Place lowered global variables in a comdat group if the associated function
1675 // or global variable is a COMDAT. This will make sure that only one copy of
1676 // global variable (e.g. function counters) of the COMDAT function will be
1677 // emitted after linking.
1678 bool NeedComdat = needsComdatForCounter(GV: *GO, M);
1679 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
1680
1681 if (!UseComdat)
1682 return;
1683
1684 // Keep in mind that this pass may run before the inliner, so we need to
1685 // create a new comdat group (for counters, profiling data, etc). If we use
1686 // the comdat of the parent function, that will result in relocations against
1687 // discarded sections.
1688 //
1689 // If the data variable is referenced by code, non-counter variables (notably
1690 // profiling data) and counters have to be in different comdats for COFF
1691 // because the Visual C++ linker will report duplicate symbol errors if there
1692 // are multiple external symbols with the same name marked
1693 // IMAGE_COMDAT_SELECT_ASSOCIATIVE.
1694 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
1695 ? GV->getName()
1696 : CounterGroupName;
1697 Comdat *C = M.getOrInsertComdat(Name: GroupName);
1698
1699 if (!NeedComdat) {
1700 // Object file format must be ELF since `UseComdat && !NeedComdat` is true.
1701 //
1702 // For ELF, when not using COMDAT, put counters, data and values into a
1703 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
1704 // allows -z start-stop-gc to discard the entire group when the function is
1705 // discarded.
1706 C->setSelectionKind(Comdat::NoDeduplicate);
1707 }
1708 GV->setComdat(C);
1709 // COFF doesn't allow the comdat group leader to have private linkage, so
1710 // upgrade private linkage to internal linkage to produce a symbol table
1711 // entry.
1712 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
1713 GV->setLinkage(GlobalValue::InternalLinkage);
1714}
1715
1716static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {
1717 if (!profDataReferencedByCode(M: *GV->getParent()))
1718 return false;
1719
1720 if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&
1721 !GV->hasAvailableExternallyLinkage())
1722 return true;
1723
1724 // This avoids the profile data from referencing internal symbols in
1725 // COMDAT.
1726 if (GV->hasLocalLinkage() && GV->hasComdat())
1727 return false;
1728
1729 return true;
1730}
1731
1732// FIXME: Introduce an internal alias like what's done for functions to reduce
1733// the number of relocation entries.
1734static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
1735 // Store a nullptr in __profvt_ if a real address shouldn't be used.
1736 if (!shouldRecordVTableAddr(GV))
1737 return ConstantPointerNull::get(T: PointerType::getUnqual(C&: GV->getContext()));
1738
1739 return GV;
1740}
1741
1742void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) {
1743 assert(ProfileCorrelate != InstrProfCorrelator::DEBUG_INFO &&
1744 "Value profiling is not supported with lightweight instrumentation");
1745 if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
1746 return;
1747
1748 // Skip llvm internal global variable or __prof variables.
1749 if (GV->getName().starts_with(Prefix: "llvm.") ||
1750 GV->getName().starts_with(Prefix: "__llvm") ||
1751 GV->getName().starts_with(Prefix: "__prof"))
1752 return;
1753
1754 // VTableProfData already created
1755 auto It = VTableDataMap.find(Val: GV);
1756 if (It != VTableDataMap.end() && It->second)
1757 return;
1758
1759 GlobalValue::LinkageTypes Linkage = GV->getLinkage();
1760 GlobalValue::VisibilityTypes Visibility = GV->getVisibility();
1761
1762 // This is to keep consistent with per-function profile data
1763 // for correctness.
1764 if (TT.isOSBinFormatXCOFF()) {
1765 Linkage = GlobalValue::InternalLinkage;
1766 Visibility = GlobalValue::DefaultVisibility;
1767 }
1768
1769 LLVMContext &Ctx = M.getContext();
1770 Type *DataTypes[] = {
1771#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
1772#include "llvm/ProfileData/InstrProfData.inc"
1773#undef INSTR_PROF_VTABLE_DATA
1774 };
1775
1776 auto *DataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes));
1777
1778 // Used by INSTR_PROF_VTABLE_DATA MACRO
1779 Constant *VTableAddr = getVTableAddrForProfData(GV);
1780 const std::string PGOVTableName = getIRPGOObjectName(GO: *GV);
1781 // Record the length of the vtable. This is needed since vtable pointers
1782 // loaded from C++ objects might be from the middle of a vtable definition.
1783 uint32_t VTableSizeVal = GV->getGlobalSize(DL: M.getDataLayout());
1784
1785 Constant *DataVals[] = {
1786#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
1787#include "llvm/ProfileData/InstrProfData.inc"
1788#undef INSTR_PROF_VTABLE_DATA
1789 };
1790
1791 auto *Data =
1792 new GlobalVariable(M, DataTy, /*constant=*/false, Linkage,
1793 ConstantStruct::get(T: DataTy, V: DataVals),
1794 getInstrProfVTableVarPrefix() + PGOVTableName);
1795
1796 Data->setVisibility(Visibility);
1797 Data->setSection(getInstrProfSectionName(IPSK: IPSK_vtab, OF: TT.getObjectFormat()));
1798 Data->setAlignment(Align(8));
1799
1800 maybeSetComdat(GV: Data, GO: GV, CounterGroupName: Data->getName());
1801
1802 VTableDataMap[GV] = Data;
1803
1804 ReferencedVTables.push_back(x: GV);
1805
1806 // VTable <Hash, Addr> is used by runtime but not referenced by other
1807 // sections. Conservatively mark it linker retained.
1808 UsedVars.push_back(x: Data);
1809}
1810
1811GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc,
1812 InstrProfSectKind IPSK) {
1813 GlobalVariable *NamePtr = Inc->getName();
1814
1815 // Match the linkage and visibility of the name global.
1816 Function *Fn = Inc->getParent()->getParent();
1817 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
1818 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1819
1820 // Use internal rather than private linkage so the counter variable shows up
1821 // in the symbol table when using debug info for correlation.
1822 if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO &&
1823 TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage)
1824 Linkage = GlobalValue::InternalLinkage;
1825
1826 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1827 // symbols in the same csect won't be discarded. When there are duplicate weak
1828 // symbols, we can NOT guarantee that the relocations get resolved to the
1829 // intended weak symbol, so we can not ensure the correctness of the relative
1830 // CounterPtr, so we have to use private linkage for counter and data symbols.
1831 if (TT.isOSBinFormatXCOFF()) {
1832 Linkage = GlobalValue::PrivateLinkage;
1833 Visibility = GlobalValue::DefaultVisibility;
1834 }
1835 // Move the name variable to the right section.
1836 bool Renamed;
1837 GlobalVariable *Ptr;
1838 StringRef VarPrefix;
1839 std::string VarName;
1840 if (IPSK == IPSK_cnts) {
1841 VarPrefix = getInstrProfCountersVarPrefix();
1842 VarName = getVarName(Inc, Prefix: VarPrefix, Renamed);
1843 InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Val: Inc);
1844 Ptr = createRegionCounters(Inc: CntrIncrement, Name: VarName, Linkage);
1845 } else if (IPSK == IPSK_bitmap) {
1846 VarPrefix = getInstrProfBitmapVarPrefix();
1847 VarName = getVarName(Inc, Prefix: VarPrefix, Renamed);
1848 InstrProfMCDCBitmapInstBase *BitmapUpdate =
1849 dyn_cast<InstrProfMCDCBitmapInstBase>(Val: Inc);
1850 Ptr = createRegionBitmaps(Inc: BitmapUpdate, Name: VarName, Linkage);
1851 } else {
1852 llvm_unreachable("Profile Section must be for Counters or Bitmaps");
1853 }
1854
1855 Ptr->setVisibility(Visibility);
1856 Ptr->setSection(getInstrProfSectionName(IPSK, OF: TT.getObjectFormat()));
1857 Ptr->setLinkage(Linkage);
1858 if (isGPUProfTarget(M) && !Ptr->hasComdat()) {
1859 Ptr->setComdat(M.getOrInsertComdat(Name: VarName));
1860 Ptr->setLinkage(GlobalValue::LinkOnceODRLinkage);
1861 Ptr->setVisibility(GlobalValue::ProtectedVisibility);
1862 } else {
1863 maybeSetComdat(GV: Ptr, GO: Fn, CounterGroupName: VarName);
1864 }
1865 return Ptr;
1866}
1867
1868GlobalVariable *
1869InstrLowerer::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
1870 StringRef Name,
1871 GlobalValue::LinkageTypes Linkage) {
1872 uint64_t NumBytes = Inc->getNumBitmapBytes();
1873 auto *BitmapTy = ArrayType::get(ElementType: Type::getInt8Ty(C&: M.getContext()), NumElements: NumBytes);
1874 auto GV = new GlobalVariable(M, BitmapTy, false, Linkage,
1875 Constant::getNullValue(Ty: BitmapTy), Name);
1876 GV->setAlignment(Align(1));
1877 return GV;
1878}
1879
1880GlobalVariable *
1881InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
1882 GlobalVariable *NamePtr = Inc->getName();
1883 auto &PD = ProfileDataMap[NamePtr];
1884 if (PD.RegionBitmaps)
1885 return PD.RegionBitmaps;
1886
1887 // If RegionBitmaps doesn't already exist, create it by first setting up
1888 // the corresponding profile section.
1889 auto *BitmapPtr = setupProfileSection(Inc, IPSK: IPSK_bitmap);
1890 PD.RegionBitmaps = BitmapPtr;
1891 PD.NumBitmapBytes = Inc->getNumBitmapBytes();
1892
1893 if (PD.NumBitmapBytes &&
1894 ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) {
1895 LLVMContext &Ctx = M.getContext();
1896 Function *Fn = Inc->getParent()->getParent();
1897 if (auto *SP = Fn->getSubprogram()) {
1898 DIBuilder DB(M, true, SP->getUnit());
1899 Metadata *FunctionNameAnnotation[] = {
1900 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::FunctionNameAttributeName),
1901 MDString::get(Context&: Ctx, Str: getPGOFuncNameVarInitializer(NameVar: NamePtr)),
1902 };
1903 Metadata *NumBitmapBitsAnnotation[] = {
1904 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::NumBitmapBitsAttributeName),
1905 ConstantAsMetadata::get(C: Inc->getNumBitmapBits()),
1906 };
1907 auto Annotations = DB.getOrCreateArray(Elements: {
1908 MDNode::get(Context&: Ctx, MDs: FunctionNameAnnotation),
1909 MDNode::get(Context&: Ctx, MDs: NumBitmapBitsAnnotation),
1910 });
1911 auto *DICounter = DB.createGlobalVariableExpression(
1912 Context: SP, Name: BitmapPtr->getName(), /*LinkageName=*/StringRef(), File: SP->getFile(),
1913 /*LineNo=*/0, Ty: DB.createUnspecifiedType(Name: "Profile Bitmap Type"),
1914 IsLocalToUnit: BitmapPtr->hasLocalLinkage(), /*IsDefined=*/isDefined: true, /*Expr=*/nullptr,
1915 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1916 Annotations);
1917 BitmapPtr->addDebugInfo(GV: DICounter);
1918 DB.finalizeSubprogram(SP);
1919 DB.finalize();
1920 }
1921
1922 // Mark the bitmap variable as used so that it isn't optimized out.
1923 CompilerUsedVars.push_back(x: PD.RegionBitmaps);
1924 }
1925
1926 return PD.RegionBitmaps;
1927}
1928
1929GlobalVariable *
1930InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
1931 GlobalValue::LinkageTypes Linkage) {
1932 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1933 auto &Ctx = M.getContext();
1934 GlobalVariable *GV;
1935 if (isa<InstrProfCoverInst>(Val: Inc)) {
1936 auto *CounterTy = Type::getInt8Ty(C&: Ctx);
1937 auto *CounterArrTy = ArrayType::get(ElementType: CounterTy, NumElements: NumCounters);
1938 // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
1939 std::vector<Constant *> InitialValues(NumCounters,
1940 Constant::getAllOnesValue(Ty: CounterTy));
1941 GV = new GlobalVariable(M, CounterArrTy, false, Linkage,
1942 ConstantArray::get(T: CounterArrTy, V: InitialValues),
1943 Name);
1944 GV->setAlignment(Align(1));
1945 } else {
1946 auto *CounterTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NumCounters);
1947 GV = new GlobalVariable(M, CounterTy, false, Linkage,
1948 Constant::getNullValue(Ty: CounterTy), Name);
1949 GV->setAlignment(Align(8));
1950 }
1951 return GV;
1952}
1953
1954GlobalVariable *
1955InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
1956 GlobalVariable *NamePtr = Inc->getName();
1957 auto &PD = ProfileDataMap[NamePtr];
1958 if (PD.RegionCounters)
1959 return PD.RegionCounters;
1960
1961 // If RegionCounters doesn't already exist, create it by first setting up
1962 // the corresponding profile section.
1963 auto *CounterPtr = setupProfileSection(Inc, IPSK: IPSK_cnts);
1964 PD.RegionCounters = CounterPtr;
1965
1966 if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) {
1967 LLVMContext &Ctx = M.getContext();
1968 Function *Fn = Inc->getParent()->getParent();
1969 if (auto *SP = Fn->getSubprogram()) {
1970 DIBuilder DB(M, true, SP->getUnit());
1971 Metadata *FunctionNameAnnotation[] = {
1972 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::FunctionNameAttributeName),
1973 MDString::get(Context&: Ctx, Str: getPGOFuncNameVarInitializer(NameVar: NamePtr)),
1974 };
1975 Metadata *CFGHashAnnotation[] = {
1976 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::CFGHashAttributeName),
1977 ConstantAsMetadata::get(C: Inc->getHash()),
1978 };
1979 Metadata *NumCountersAnnotation[] = {
1980 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::NumCountersAttributeName),
1981 ConstantAsMetadata::get(C: Inc->getNumCounters()),
1982 };
1983 auto Annotations = DB.getOrCreateArray(Elements: {
1984 MDNode::get(Context&: Ctx, MDs: FunctionNameAnnotation),
1985 MDNode::get(Context&: Ctx, MDs: CFGHashAnnotation),
1986 MDNode::get(Context&: Ctx, MDs: NumCountersAnnotation),
1987 });
1988 auto *DICounter = DB.createGlobalVariableExpression(
1989 Context: SP, Name: CounterPtr->getName(), /*LinkageName=*/StringRef(), File: SP->getFile(),
1990 /*LineNo=*/0, Ty: DB.createUnspecifiedType(Name: "Profile Data Type"),
1991 IsLocalToUnit: CounterPtr->hasLocalLinkage(), /*IsDefined=*/isDefined: true, /*Expr=*/nullptr,
1992 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1993 Annotations);
1994 CounterPtr->addDebugInfo(GV: DICounter);
1995 DB.finalizeSubprogram(SP);
1996 DB.finalize();
1997 }
1998
1999 // Mark the counter variable as used so that it isn't optimized out.
2000 CompilerUsedVars.push_back(x: PD.RegionCounters);
2001 }
2002
2003 // Create uniform counters before the data variable so that
2004 // UniformCounterPtr can reference them in createDataVariable().
2005 getOrCreateUniformCounters(Inc);
2006
2007 // Create the data variable (if it doesn't already exist).
2008 createDataVariable(Inc);
2009
2010 return PD.RegionCounters;
2011}
2012
2013GlobalVariable *
2014InstrLowerer::getOrCreateUniformCounters(InstrProfCntrInstBase *Inc) {
2015 // Uniform counters are only meaningful for GPU profile targets.
2016 if (!isGPUProfTarget(M))
2017 return nullptr;
2018
2019 GlobalVariable *NamePtr = Inc->getName();
2020 auto &PD = ProfileDataMap[NamePtr];
2021 if (PD.UniformCounters)
2022 return PD.UniformCounters;
2023
2024 assert(PD.RegionCounters && "region counters must be created first");
2025
2026 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
2027
2028 LLVMContext &Ctx = M.getContext();
2029 ArrayType *CounterTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NumCounters);
2030
2031 bool Renamed;
2032 std::string VarName = getVarName(Inc, Prefix: "__llvm_prf_unifcnt_", Renamed);
2033
2034 auto *GV = new GlobalVariable(M, CounterTy, false, NamePtr->getLinkage(),
2035 Constant::getNullValue(Ty: CounterTy), VarName);
2036 GV->setAlignment(Align(8));
2037
2038 GV->setSection(getInstrProfSectionName(IPSK: IPSK_ucnts, OF: TT.getObjectFormat()));
2039
2040 GV->setComdat(M.getOrInsertComdat(Name: VarName));
2041 GV->setLinkage(GlobalValue::LinkOnceODRLinkage);
2042 GV->setVisibility(GlobalValue::ProtectedVisibility);
2043
2044 PD.UniformCounters = GV;
2045 CompilerUsedVars.push_back(x: GV);
2046
2047 return PD.UniformCounters;
2048}
2049
2050void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
2051 // When debug information is correlated to profile data, a data variable
2052 // is not needed.
2053 if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)
2054 return;
2055
2056 GlobalVariable *NamePtr = Inc->getName();
2057 auto &PD = ProfileDataMap[NamePtr];
2058
2059 // Return if data variable was already created.
2060 if (PD.DataVar)
2061 return;
2062
2063 LLVMContext &Ctx = M.getContext();
2064
2065 Function *Fn = Inc->getParent()->getParent();
2066 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
2067 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
2068
2069 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
2070 // symbols in the same csect won't be discarded. When there are duplicate weak
2071 // symbols, we can NOT guarantee that the relocations get resolved to the
2072 // intended weak symbol, so we can not ensure the correctness of the relative
2073 // CounterPtr, so we have to use private linkage for counter and data symbols.
2074 if (TT.isOSBinFormatXCOFF()) {
2075 Linkage = GlobalValue::PrivateLinkage;
2076 Visibility = GlobalValue::DefaultVisibility;
2077 }
2078
2079 bool NeedComdat = needsComdatForCounter(GV: *Fn, M);
2080 bool Renamed;
2081
2082 // The Data Variable section is anchored to profile counters.
2083 std::string CntsVarName =
2084 getVarName(Inc, Prefix: getInstrProfCountersVarPrefix(), Renamed);
2085 std::string DataVarName =
2086 getVarName(Inc, Prefix: getInstrProfDataVarPrefix(), Renamed);
2087
2088 auto *Int8PtrTy = PointerType::getUnqual(C&: Ctx);
2089 // Allocate statically the array of pointers to value profile nodes for
2090 // the current function.
2091 Constant *ValuesPtrExpr = ConstantPointerNull::get(T: Int8PtrTy);
2092 uint64_t NS = 0;
2093 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
2094 NS += PD.NumValueSites[Kind];
2095 if (NS > 0 && ValueProfileStaticAlloc &&
2096 !needsRuntimeRegistrationOfSectionRange(TT)) {
2097 ArrayType *ValuesTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NS);
2098 auto *ValuesVar = new GlobalVariable(
2099 M, ValuesTy, false, Linkage, Constant::getNullValue(Ty: ValuesTy),
2100 getVarName(Inc, Prefix: getInstrProfValuesVarPrefix(), Renamed));
2101 ValuesVar->setVisibility(Visibility);
2102 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *ValuesVar);
2103 ValuesVar->setSection(
2104 getInstrProfSectionName(IPSK: IPSK_vals, OF: TT.getObjectFormat()));
2105 ValuesVar->setAlignment(Align(8));
2106 maybeSetComdat(GV: ValuesVar, GO: Fn, CounterGroupName: CntsVarName);
2107 ValuesPtrExpr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
2108 C: ValuesVar, Ty: PointerType::get(C&: Fn->getContext(), AddressSpace: 0));
2109 }
2110
2111 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
2112
2113 Constant *CounterPtr = PD.RegionCounters;
2114 Constant *UniformCounterPtr = PD.UniformCounters;
2115
2116 uint64_t NumBitmapBytes = PD.NumBitmapBytes;
2117
2118 // Create data variable.
2119 auto *IntPtrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
2120 auto *Int16Ty = Type::getInt16Ty(C&: Ctx);
2121 auto *Int16ArrayTy = ArrayType::get(ElementType: Int16Ty, NumElements: IPVK_Last + 1);
2122 auto *DataTy = getProfileDataTy();
2123
2124 Constant *FunctionAddr = getFuncAddrForProfData(Fn);
2125
2126 Constant *Int16ArrayVals[IPVK_Last + 1];
2127 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
2128 Int16ArrayVals[Kind] = ConstantInt::get(Ty: Int16Ty, V: PD.NumValueSites[Kind]);
2129
2130 uint16_t OffloadDeviceWaveSizeVal = 0;
2131
2132 if (isGPUProfTarget(M)) {
2133 // For GPU targets, weak functions need weak linkage for their profile data
2134 // aliases to allow linker deduplication across TUs
2135 if (GlobalValue::isWeakForLinker(Linkage: Fn->getLinkage()))
2136 Linkage = Fn->getLinkage();
2137 else
2138 Linkage = GlobalValue::ExternalLinkage;
2139 Visibility = GlobalValue::ProtectedVisibility;
2140 }
2141 // If the data variable is not referenced by code (if we don't emit
2142 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
2143 // data variable live under linker GC, the data variable can be private. This
2144 // optimization applies to ELF.
2145 //
2146 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
2147 // to be false.
2148 //
2149 // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
2150 // that other copies must have the same CFG and cannot have value profiling.
2151 // If no hash suffix, other profd copies may be referenced by code.
2152 if (!isGPUProfTarget(M) && NS == 0 &&
2153 !(DataReferencedByCode && NeedComdat && !Renamed) &&
2154 (TT.isOSBinFormatELF() ||
2155 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
2156 Linkage = GlobalValue::PrivateLinkage;
2157 Visibility = GlobalValue::DefaultVisibility;
2158 }
2159 // GPU-target ELF objects are always ET_DYN, so non-local symbols with
2160 // default visibility are preemptible. The CounterPtr label difference
2161 // emits a REL32 relocation that lld rejects against preemptible targets.
2162 if (TT.isGPU() && TT.isOSBinFormatELF() &&
2163 !GlobalValue::isLocalLinkage(Linkage))
2164 Visibility = GlobalValue::ProtectedVisibility;
2165 auto *Data =
2166 new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName);
2167
2168 Constant *RelativeCounterPtr;
2169 Constant *RelativeUniformCounterPtr = ConstantInt::get(Ty: IntPtrTy, V: 0);
2170 GlobalVariable *BitmapPtr = PD.RegionBitmaps;
2171 Constant *RelativeBitmapPtr = ConstantInt::get(Ty: IntPtrTy, V: 0);
2172 InstrProfSectKind DataSectionKind;
2173 // With binary profile correlation, profile data is not loaded into memory.
2174 // profile data must reference profile counter with an absolute relocation.
2175 if (ProfileCorrelate == InstrProfCorrelator::BINARY) {
2176 DataSectionKind = IPSK_covdata;
2177 RelativeCounterPtr = ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy);
2178 if (BitmapPtr != nullptr)
2179 RelativeBitmapPtr = ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy);
2180 if (UniformCounterPtr != nullptr)
2181 RelativeUniformCounterPtr =
2182 ConstantExpr::getPtrToInt(C: UniformCounterPtr, Ty: IntPtrTy);
2183 } else if (TT.isNVPTX()) {
2184 // The NVPTX target cannot handle self-referencing constant expressions in
2185 // global initializers at all. Use absolute pointers and have the runtime
2186 // registration convert them to relative offsets.
2187 DataSectionKind = IPSK_data;
2188 RelativeCounterPtr = ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy);
2189 } else {
2190 // Reference the counter variable with a label difference (link-time
2191 // constant).
2192 DataSectionKind = IPSK_data;
2193 RelativeCounterPtr =
2194 ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy),
2195 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2196 if (BitmapPtr != nullptr)
2197 RelativeBitmapPtr =
2198 ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy),
2199 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2200 if (UniformCounterPtr != nullptr)
2201 RelativeUniformCounterPtr = ConstantExpr::getSub(
2202 C1: ConstantExpr::getPtrToInt(C: UniformCounterPtr, Ty: IntPtrTy),
2203 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2204 }
2205
2206 Constant *DataVals[] = {
2207#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
2208#include "llvm/ProfileData/InstrProfData.inc"
2209 };
2210 Data->setInitializer(ConstantStruct::get(T: DataTy, V: DataVals));
2211
2212 Data->setVisibility(Visibility);
2213 Data->setSection(
2214 getInstrProfSectionName(IPSK: DataSectionKind, OF: TT.getObjectFormat()));
2215 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
2216 if (isGPUProfTarget(M) && !Data->hasComdat()) {
2217 Data->setComdat(M.getOrInsertComdat(Name: CntsVarName));
2218 Data->setLinkage(GlobalValue::LinkOnceODRLinkage);
2219 } else {
2220 maybeSetComdat(GV: Data, GO: Fn, CounterGroupName: CntsVarName);
2221 }
2222
2223 PD.DataVar = Data;
2224
2225 // Mark the data variable as used so that it isn't stripped out.
2226 CompilerUsedVars.push_back(x: Data);
2227 // Now that the linkage set by the FE has been passed to the data and counter
2228 // variables, reset Name variable's linkage and visibility to private so that
2229 // it can be removed later by the compiler.
2230 NamePtr->setLinkage(GlobalValue::PrivateLinkage);
2231 // Collect the referenced names to be used by emitNameData.
2232 ReferencedNames.push_back(x: NamePtr);
2233}
2234
2235void InstrLowerer::emitVNodes() {
2236 if (!ValueProfileStaticAlloc)
2237 return;
2238
2239 // For now only support this on platforms that do
2240 // not require runtime registration to discover
2241 // named section start/end.
2242 if (needsRuntimeRegistrationOfSectionRange(TT))
2243 return;
2244
2245 size_t TotalNS = 0;
2246 for (auto &PD : ProfileDataMap) {
2247 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
2248 TotalNS += PD.second.NumValueSites[Kind];
2249 }
2250
2251 if (!TotalNS)
2252 return;
2253
2254 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
2255// Heuristic for small programs with very few total value sites.
2256// The default value of vp-counters-per-site is chosen based on
2257// the observation that large apps usually have a low percentage
2258// of value sites that actually have any profile data, and thus
2259// the average number of counters per site is low. For small
2260// apps with very few sites, this may not be true. Bump up the
2261// number of counters in this case.
2262#define INSTR_PROF_MIN_VAL_COUNTS 10
2263 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
2264 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, b: (int)NumCounters * 2);
2265
2266 auto &Ctx = M.getContext();
2267 Type *VNodeTypes[] = {
2268#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
2269#include "llvm/ProfileData/InstrProfData.inc"
2270 };
2271 auto *VNodeTy = StructType::get(Context&: Ctx, Elements: ArrayRef(VNodeTypes));
2272
2273 ArrayType *VNodesTy = ArrayType::get(ElementType: VNodeTy, NumElements: NumCounters);
2274 auto *VNodesVar = new GlobalVariable(
2275 M, VNodesTy, false, GlobalValue::PrivateLinkage,
2276 Constant::getNullValue(Ty: VNodesTy), getInstrProfVNodesVarName());
2277 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *VNodesVar);
2278 VNodesVar->setSection(
2279 getInstrProfSectionName(IPSK: IPSK_vnodes, OF: TT.getObjectFormat()));
2280 VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(Ty: VNodesTy));
2281 // VNodesVar is used by runtime but not referenced via relocation by other
2282 // sections. Conservatively make it linker retained.
2283 UsedVars.push_back(x: VNodesVar);
2284}
2285
2286// Build the per-TU device-PGO sections struct: section start/stop bounds for
2287// names/counters/data/uniform-counters plus the raw version. Returns null if it
2288// already exists.
2289static GlobalVariable *emitGPUOffloadSectionsStruct(Module &M,
2290 StringRef CUIDPostfix) {
2291 std::string Name = ("__llvm_profile_sections" + CUIDPostfix).str();
2292 if (M.getNamedValue(Name))
2293 return nullptr;
2294
2295 LLVMContext &Ctx = M.getContext();
2296 unsigned AS = M.getDataLayout().getDefaultGlobalsAddressSpace();
2297 auto Extern = [&](StringRef Sym, Type *Ty, bool IsConst,
2298 GlobalValue::VisibilityTypes Vis) {
2299 GlobalVariable *GV = M.getNamedGlobal(Name: Sym);
2300 if (!GV) {
2301 GV = new GlobalVariable(M, Ty, IsConst, GlobalValue::ExternalLinkage,
2302 nullptr, Sym, nullptr,
2303 GlobalValue::NotThreadLocal, AS);
2304 GV->setVisibility(Vis);
2305 }
2306 return GV;
2307 };
2308 // Section bounds are hidden i8 markers; raw_version is an i64 constant.
2309 auto *I8 = Type::getInt8Ty(C&: Ctx);
2310 auto Hidden = GlobalValue::HiddenVisibility;
2311 Constant *Fields[] = {Extern("__start___llvm_prf_names", I8, false, Hidden),
2312 Extern("__stop___llvm_prf_names", I8, false, Hidden),
2313 Extern("__start___llvm_prf_cnts", I8, false, Hidden),
2314 Extern("__stop___llvm_prf_cnts", I8, false, Hidden),
2315 Extern("__start___llvm_prf_data", I8, false, Hidden),
2316 Extern("__stop___llvm_prf_data", I8, false, Hidden),
2317 Extern("__start___llvm_prf_ucnts", I8, false, Hidden),
2318 Extern("__stop___llvm_prf_ucnts", I8, false, Hidden),
2319 Extern("__llvm_profile_raw_version",
2320 Type::getInt64Ty(C&: Ctx), true,
2321 GlobalValue::DefaultVisibility)};
2322 auto *PtrTy = PointerType::get(C&: Ctx, AddressSpace: AS);
2323 auto *STy = StructType::get(
2324 Context&: Ctx, Elements: {PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy});
2325 auto *GV = new GlobalVariable(M, STy, /*isConstant=*/true,
2326 GlobalValue::ExternalLinkage,
2327 ConstantStruct::get(T: STy, V: Fields), Name, nullptr,
2328 GlobalValue::NotThreadLocal, AS);
2329 GV->setVisibility(GlobalValue::ProtectedVisibility);
2330 return GV;
2331}
2332
2333void InstrLowerer::emitNameData() {
2334 if (ReferencedNames.empty())
2335 return;
2336
2337 std::string CompressedNameStr;
2338 if (Error E = collectPGOFuncNameStrings(NameVars: ReferencedNames, Result&: CompressedNameStr,
2339 doCompression: DoInstrProfNameCompression)) {
2340 report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false);
2341 }
2342
2343 auto &Ctx = M.getContext();
2344 auto *NamesVal =
2345 ConstantDataArray::getString(Context&: Ctx, Initializer: StringRef(CompressedNameStr), AddNull: false);
2346 std::string NamesVarName = std::string(getInstrProfNamesVarName());
2347 GlobalValue::LinkageTypes NamesLinkage = GlobalValue::PrivateLinkage;
2348 GlobalValue::VisibilityTypes NamesVisibility = GlobalValue::DefaultVisibility;
2349 std::string GPUCUIDPostfix;
2350 if (isGPUProfTarget(M)) {
2351 if (auto *GV = M.getNamedGlobal(Name: getInstrProfNamesVarPostfixVarName())) {
2352 if (auto *Init =
2353 dyn_cast_or_null<ConstantDataArray>(Val: GV->getInitializer())) {
2354 if (Init->isCString()) {
2355 GPUCUIDPostfix = Init->getAsCString().str();
2356 NamesVarName += GPUCUIDPostfix;
2357 NamesLinkage = GlobalValue::ExternalLinkage;
2358 NamesVisibility = GlobalValue::ProtectedVisibility;
2359 removeFromUsedLists(
2360 M, ShouldRemove: [GV](Constant *C) { return C->stripPointerCasts() == GV; });
2361 GV->eraseFromParent();
2362 }
2363 }
2364 }
2365 }
2366 NamesVar = new GlobalVariable(M, NamesVal->getType(), true, NamesLinkage,
2367 NamesVal, NamesVarName);
2368 NamesVar->setVisibility(NamesVisibility);
2369
2370 NamesSize = CompressedNameStr.size();
2371 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *NamesVar);
2372 std::string NamesSectionName =
2373 ProfileCorrelate == InstrProfCorrelator::BINARY
2374 ? getInstrProfSectionName(IPSK: IPSK_covname, OF: TT.getObjectFormat())
2375 : getInstrProfSectionName(IPSK: IPSK_name, OF: TT.getObjectFormat());
2376 NamesVar->setSection(NamesSectionName);
2377 // On COFF, it's important to reduce the alignment down to 1 to prevent the
2378 // linker from inserting padding before the start of the names section or
2379 // between names entries.
2380 NamesVar->setAlignment(Align(1));
2381 // NamesVar is used by runtime but not referenced via relocation by other
2382 // sections. Conservatively make it linker retained.
2383 UsedVars.push_back(x: NamesVar);
2384
2385 for (auto *NamePtr : ReferencedNames)
2386 NamePtr->eraseFromParent();
2387
2388 // Emit the device sections struct only when this TU produced profile data, so
2389 // its section start/stop references are backed by a real section.
2390 bool HasData = llvm::any_of(Range&: ProfileDataMap,
2391 P: [](const auto &KV) { return KV.second.DataVar; });
2392 if (!GPUCUIDPostfix.empty() && HasData)
2393 if (GlobalVariable *GV = emitGPUOffloadSectionsStruct(M, CUIDPostfix: GPUCUIDPostfix))
2394 CompilerUsedVars.push_back(x: GV);
2395}
2396
2397void InstrLowerer::emitVTableNames() {
2398 if (!EnableVTableValueProfiling || ReferencedVTables.empty())
2399 return;
2400
2401 // Collect the PGO names of referenced vtables and compress them.
2402 std::string CompressedVTableNames;
2403 if (Error E = collectVTableStrings(VTables: ReferencedVTables, Result&: CompressedVTableNames,
2404 doCompression: DoInstrProfNameCompression)) {
2405 report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false);
2406 }
2407
2408 auto &Ctx = M.getContext();
2409 auto *VTableNamesVal = ConstantDataArray::getString(
2410 Context&: Ctx, Initializer: StringRef(CompressedVTableNames), AddNull: false /* AddNull */);
2411 GlobalVariable *VTableNamesVar =
2412 new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */,
2413 GlobalValue::PrivateLinkage, VTableNamesVal,
2414 getInstrProfVTableNamesVarName());
2415 VTableNamesVar->setSection(
2416 getInstrProfSectionName(IPSK: IPSK_vname, OF: TT.getObjectFormat()));
2417 VTableNamesVar->setAlignment(Align(1));
2418 // Make VTableNames linker retained.
2419 UsedVars.push_back(x: VTableNamesVar);
2420}
2421
2422void InstrLowerer::emitRegistration() {
2423 if (!needsRuntimeRegistrationOfSectionRange(TT))
2424 return;
2425
2426 // Construct the function.
2427 auto *VoidTy = Type::getVoidTy(C&: M.getContext());
2428 auto *VoidPtrTy = PointerType::getUnqual(C&: M.getContext());
2429 auto *Int64Ty = Type::getInt64Ty(C&: M.getContext());
2430 auto *RegisterFTy = FunctionType::get(Result: VoidTy, isVarArg: false);
2431 auto *RegisterF = Function::Create(Ty: RegisterFTy, Linkage: GlobalValue::InternalLinkage,
2432 N: getInstrProfRegFuncsName(), M);
2433 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2434 if (Options.NoRedZone)
2435 RegisterF->addFnAttr(Kind: Attribute::NoRedZone);
2436
2437 auto *RuntimeRegisterTy = FunctionType::get(Result: VoidTy, Params: VoidPtrTy, isVarArg: false);
2438 auto *RuntimeRegisterF =
2439 Function::Create(Ty: RuntimeRegisterTy, Linkage: GlobalVariable::ExternalLinkage,
2440 N: getInstrProfRegFuncName(), M);
2441
2442 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: RegisterF));
2443 for (Value *Data : CompilerUsedVars)
2444 if (!isa<Function>(Val: Data))
2445 // Check for addrspace cast when profiling GPU
2446 IRB.CreateCall(Callee: RuntimeRegisterF,
2447 Args: IRB.CreatePointerBitCastOrAddrSpaceCast(V: Data, DestTy: VoidPtrTy));
2448 for (Value *Data : UsedVars)
2449 if (Data != NamesVar && !isa<Function>(Val: Data))
2450 IRB.CreateCall(Callee: RuntimeRegisterF,
2451 Args: IRB.CreatePointerBitCastOrAddrSpaceCast(V: Data, DestTy: VoidPtrTy));
2452
2453 if (NamesVar) {
2454 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
2455 auto *NamesRegisterTy =
2456 FunctionType::get(Result: VoidTy, Params: ArrayRef(ParamTypes), isVarArg: false);
2457 auto *NamesRegisterF =
2458 Function::Create(Ty: NamesRegisterTy, Linkage: GlobalVariable::ExternalLinkage,
2459 N: getInstrProfNamesRegFuncName(), M);
2460 IRB.CreateCall(Callee: NamesRegisterF, Args: {IRB.CreatePointerBitCastOrAddrSpaceCast(
2461 V: NamesVar, DestTy: VoidPtrTy),
2462 IRB.getInt64(C: NamesSize)});
2463 }
2464
2465 IRB.CreateRetVoid();
2466}
2467
2468bool InstrLowerer::emitRuntimeHook() {
2469 // GPU profiling data is read directly by the host offload runtime. We do not
2470 // need the standard runtime hook.
2471 if (TT.isGPU())
2472 return false;
2473
2474 // We expect the linker to be invoked with -u<hook_var> flag for Linux
2475 // in which case there is no need to emit the external variable.
2476 if (TT.isOSLinux() || TT.isOSAIX())
2477 return false;
2478
2479 // If the module's provided its own runtime, we don't need to do anything.
2480 if (M.getGlobalVariable(Name: getInstrProfRuntimeHookVarName()))
2481 return false;
2482
2483 // Declare an external variable that will pull in the runtime initialization.
2484 auto *Int32Ty = Type::getInt32Ty(C&: M.getContext());
2485 auto *Var =
2486 new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
2487 nullptr, getInstrProfRuntimeHookVarName());
2488 Var->setVisibility(GlobalValue::HiddenVisibility);
2489
2490 if (TT.isOSBinFormatELF() && !TT.isPS()) {
2491 // Mark the user variable as used so that it isn't stripped out.
2492 CompilerUsedVars.push_back(x: Var);
2493 } else {
2494 // Make a function that uses it.
2495 auto *User = Function::Create(Ty: FunctionType::get(Result: Int32Ty, isVarArg: false),
2496 Linkage: GlobalValue::LinkOnceODRLinkage,
2497 N: getInstrProfRuntimeHookVarUseFuncName(), M);
2498 User->addFnAttr(Kind: Attribute::NoInline);
2499 if (Options.NoRedZone)
2500 User->addFnAttr(Kind: Attribute::NoRedZone);
2501 User->setVisibility(GlobalValue::HiddenVisibility);
2502 if (TT.supportsCOMDAT())
2503 User->setComdat(M.getOrInsertComdat(Name: User->getName()));
2504 // Explicitly mark this function as cold since it is never called.
2505 User->setEntryCount(Count: 0);
2506
2507 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: User));
2508 auto *Load = IRB.CreateLoad(Ty: Int32Ty, Ptr: Var);
2509 IRB.CreateRet(V: Load);
2510
2511 // Mark the function as used so that it isn't stripped out.
2512 CompilerUsedVars.push_back(x: User);
2513 }
2514 return true;
2515}
2516
2517void InstrLowerer::emitUses() {
2518 // The metadata sections are parallel arrays. Optimizers (e.g.
2519 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
2520 // we conservatively retain all unconditionally in the compiler.
2521 //
2522 // On ELF and Mach-O, the linker can guarantee the associated sections will be
2523 // retained or discarded as a unit, so llvm.compiler.used is sufficient.
2524 // Similarly on COFF, if prof data is not referenced by code we use one comdat
2525 // and ensure this GC property as well. Otherwise, we have to conservatively
2526 // make all of the sections retained by the linker.
2527 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
2528 (TT.isOSBinFormatCOFF() && !DataReferencedByCode))
2529 appendToCompilerUsed(M, Values: CompilerUsedVars);
2530 else
2531 appendToUsed(M, Values: CompilerUsedVars);
2532
2533 // We do not add proper references from used metadata sections to NamesVar and
2534 // VNodesVar, so we have to be conservative and place them in llvm.used
2535 // regardless of the target,
2536 appendToUsed(M, Values: UsedVars);
2537}
2538
2539void InstrLowerer::emitInitialization() {
2540 // Create ProfileFileName variable. Don't don't this for the
2541 // context-sensitive instrumentation lowering: This lowering is after
2542 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
2543 // have already create the variable before LTO/ThinLTO linking.
2544 if (!IsCS)
2545 createProfileFileNameVar(M, InstrProfileOutput: Options.InstrProfileOutput);
2546 Function *RegisterF = M.getFunction(Name: getInstrProfRegFuncsName());
2547 if (!RegisterF)
2548 return;
2549
2550 // Create the initialization function.
2551 auto *VoidTy = Type::getVoidTy(C&: M.getContext());
2552 auto *F = Function::Create(Ty: FunctionType::get(Result: VoidTy, isVarArg: false),
2553 Linkage: GlobalValue::InternalLinkage,
2554 N: getInstrProfInitFuncName(), M);
2555 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2556 F->addFnAttr(Kind: Attribute::NoInline);
2557 if (Options.NoRedZone)
2558 F->addFnAttr(Kind: Attribute::NoRedZone);
2559
2560 // Add the basic block and the necessary calls.
2561 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: F));
2562 IRB.CreateCall(Callee: RegisterF, Args: {});
2563 IRB.CreateRetVoid();
2564
2565 appendToGlobalCtors(M, F, Priority: 0);
2566}
2567
2568namespace llvm {
2569// Create the variable for profile sampling.
2570void createProfileSamplingVar(Module &M) {
2571 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
2572 IntegerType *SamplingVarTy;
2573 Constant *ValueZero;
2574 if (getSampledInstrumentationConfig().UseShort) {
2575 SamplingVarTy = Type::getInt16Ty(C&: M.getContext());
2576 ValueZero = Constant::getIntegerValue(Ty: SamplingVarTy, V: APInt(16, 0));
2577 } else {
2578 SamplingVarTy = Type::getInt32Ty(C&: M.getContext());
2579 ValueZero = Constant::getIntegerValue(Ty: SamplingVarTy, V: APInt(32, 0));
2580 }
2581 auto SamplingVar = new GlobalVariable(
2582 M, SamplingVarTy, false, GlobalValue::WeakAnyLinkage, ValueZero, VarName);
2583 SamplingVar->setVisibility(GlobalValue::DefaultVisibility);
2584 SamplingVar->setThreadLocal(true);
2585 Triple TT(M.getTargetTriple());
2586 if (TT.supportsCOMDAT()) {
2587 SamplingVar->setLinkage(GlobalValue::ExternalLinkage);
2588 SamplingVar->setComdat(M.getOrInsertComdat(Name: VarName));
2589 }
2590 appendToCompilerUsed(M, Values: SamplingVar);
2591}
2592} // namespace llvm
2593
2594// For GPU targets: Allocate contiguous arrays for all profile data.
2595// This solves the linker reordering problem by using ONE symbol per section
2596// type, so there's nothing for the linker to reorder.
2597StructType *InstrLowerer::getProfileDataTy() {
2598 if (ProfileDataTy)
2599 return ProfileDataTy;
2600
2601 auto &Ctx = M.getContext();
2602 auto *IntPtrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
2603 auto *Int16Ty = Type::getInt16Ty(C&: Ctx);
2604 auto *Int16ArrayTy = ArrayType::get(ElementType: Int16Ty, NumElements: IPVK_Last + 1);
2605 Type *DataTypes[] = {
2606#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
2607#include "llvm/ProfileData/InstrProfData.inc"
2608 };
2609 ProfileDataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes));
2610 return ProfileDataTy;
2611}
2612