1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by an instrumentor.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
16#include "InstrumentationOptions.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/ADT/Twine.h"
22#include "llvm/Analysis/BlockFrequencyInfo.h"
23#include "llvm/Analysis/BranchProbabilityInfo.h"
24#include "llvm/Analysis/CFG.h"
25#include "llvm/Analysis/LoopInfo.h"
26#include "llvm/Analysis/TargetLibraryInfo.h"
27#include "llvm/IR/Attributes.h"
28#include "llvm/IR/BasicBlock.h"
29#include "llvm/IR/CFG.h"
30#include "llvm/IR/Constant.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/CycleInfo.h"
33#include "llvm/IR/DIBuilder.h"
34#include "llvm/IR/DerivedTypes.h"
35#include "llvm/IR/DiagnosticInfo.h"
36#include "llvm/IR/Function.h"
37#include "llvm/IR/GlobalAlias.h"
38#include "llvm/IR/GlobalValue.h"
39#include "llvm/IR/GlobalVariable.h"
40#include "llvm/IR/IRBuilder.h"
41#include "llvm/IR/InstIterator.h"
42#include "llvm/IR/Instruction.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/MDBuilder.h"
47#include "llvm/IR/Module.h"
48#include "llvm/IR/RuntimeLibcalls.h"
49#include "llvm/IR/Type.h"
50#include "llvm/Pass.h"
51#include "llvm/ProfileData/InstrProf.h"
52#include "llvm/ProfileData/InstrProfCorrelator.h"
53#include "llvm/Support/Casting.h"
54#include "llvm/Support/CommandLine.h"
55#include "llvm/Support/Compiler.h"
56#include "llvm/Support/Error.h"
57#include "llvm/Support/ErrorHandling.h"
58#include "llvm/TargetParser/Triple.h"
59#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
60#include "llvm/Transforms/Utils/BasicBlockUtils.h"
61#include "llvm/Transforms/Utils/Instrumentation.h"
62#include "llvm/Transforms/Utils/ModuleUtils.h"
63#include "llvm/Transforms/Utils/SSAUpdater.h"
64#include <algorithm>
65#include <cassert>
66#include <cstdint>
67#include <string>
68
69using namespace llvm;
70
71#define DEBUG_TYPE "instrprof"
72
73namespace llvm {
74// Command line option to enable vtable value profiling. Defined in
75// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
76extern cl::opt<bool> EnableVTableValueProfiling;
77} // namespace llvm
78
79bool llvm::isProfileCorrelationEnabled(
80 const cl::LibraryOptionsContext &LibraryOptions) {
81 return LibraryOptions.get<InstrumentationOptions>()
82 .profile_correlate.has_value();
83}
84
85namespace {
86
87struct SampledInstrumentationConfig {
88 unsigned BurstDuration;
89 unsigned Period;
90 bool UseShort;
91 bool IsSimpleSampling;
92 bool IsFastSampling;
93};
94
95static SampledInstrumentationConfig
96getSampledInstrumentationConfig(const InstrumentationOptions &Opts) {
97 SampledInstrumentationConfig config;
98 config.BurstDuration = Opts.sampled_instr_burst_duration;
99 config.Period = Opts.sampled_instr_period;
100 if (config.BurstDuration > config.Period)
101 report_fatal_error(
102 reason: "SampledBurstDuration must be less than or equal to SampledPeriod");
103 if (config.Period == 0 || config.BurstDuration == 0)
104 report_fatal_error(
105 reason: "SampledPeriod and SampledBurstDuration must be greater than 0");
106 config.IsSimpleSampling = (config.BurstDuration == 1);
107 // If (BurstDuration == 1 && Period == 65536), generate the simple sampling
108 // style code.
109 config.IsFastSampling =
110 (!config.IsSimpleSampling && config.Period == USHRT_MAX + 1);
111 config.UseShort = (config.Period <= USHRT_MAX) || config.IsFastSampling;
112 return config;
113}
114
115using LoadStorePair = std::pair<Instruction *, Instruction *>;
116
117static void makeAtomic(Instruction *Load, Instruction *Store) {
118 auto *Addition = dyn_cast<BinaryOperator>(Val: Store->getOperand(i: 0));
119 assert(Addition && Addition->getOpcode() == Instruction::BinaryOps::Add);
120 auto *Addend = Addition->getOperand(i_nocapture: 1);
121
122 IRBuilder<> Builder(Load);
123 Builder
124 .CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Store->getOperand(i: 1), Val: Addend,
125 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic)
126 ->setNoSanitizeMetadata();
127 Store->eraseFromParent();
128 Addition->eraseFromParent();
129 Load->eraseFromParent();
130}
131
132static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {
133 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(Val: M.getModuleFlag(Key: Flag));
134 if (!MD)
135 return 0;
136
137 // If the flag is a ConstantAsMetadata, it should be an integer representable
138 // in 64-bits.
139 return cast<ConstantInt>(Val: MD->getValue())->getZExtValue();
140}
141
142static bool enablesValueProfiling(const Module &M) {
143 return isIRPGOFlagSet(M: &M) ||
144 getIntModuleFlagOrZero(M, Flag: "EnableValueProfiling") != 0;
145}
146
147// Conservatively returns true if value profiling is enabled.
148static bool profDataReferencedByCode(const Module &M) {
149 return enablesValueProfiling(M);
150}
151
152class InstrLowerer final {
153public:
154 InstrLowerer(const InstrumentationOptions &Opts, Module &M,
155 const InstrProfOptions &Options,
156 std::function<const TargetLibraryInfo &(Function &F)> GetTLI,
157 bool IsCS)
158 : Opts(Opts), M(M), Options(Options), TT(M.getTargetTriple()), IsCS(IsCS),
159 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {}
160
161 bool lower();
162
163private:
164 const InstrumentationOptions &Opts;
165 Module &M;
166 const InstrProfOptions Options;
167 const Triple TT;
168 // Is this lowering for the context-sensitive instrumentation.
169 const bool IsCS;
170
171 std::function<const TargetLibraryInfo &(Function &F)> GetTLI;
172
173 const bool DataReferencedByCode;
174
175 struct PerFunctionProfileData {
176 uint32_t NumValueSites[IPVK_Last + 1] = {};
177 GlobalVariable *RegionCounters = nullptr;
178 GlobalVariable *UniformCounters =
179 nullptr; // Per-block uniform-entry counters
180 GlobalVariable *DataVar = nullptr;
181 GlobalVariable *RegionBitmaps = nullptr;
182 uint32_t NumBitmapBytes = 0;
183
184 PerFunctionProfileData() = default;
185 };
186 DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
187 // Key is virtual table variable, value is 'VTableProfData' in the form of
188 // GlobalVariable.
189 DenseMap<GlobalVariable *, GlobalVariable *> VTableDataMap;
190 /// If runtime relocation is enabled, this maps functions to the load
191 /// instruction that produces the profile relocation bias.
192 DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;
193 std::vector<GlobalValue *> CompilerUsedVars;
194 std::vector<GlobalValue *> UsedVars;
195 std::vector<GlobalVariable *> ReferencedNames;
196 // The list of virtual table variables of which the VTableProfData is
197 // collected.
198 std::vector<GlobalVariable *> ReferencedVTables;
199 GlobalVariable *NamesVar = nullptr;
200 size_t NamesSize = 0;
201
202 StructType *ProfileDataTy = nullptr;
203
204 // vector of counter load/store pairs to be register promoted.
205 std::vector<LoadStorePair> PromotionCandidates;
206
207 int64_t TotalCountersPromoted = 0;
208
209 // Per-function cache of invariant values for GPU PGO instrumentation.
210 // Computed once at the function entry and reused across all instrumentation
211 // points to avoid redundant IR and help the optimizer.
212 struct GPUPGOInvariants {
213 Value *Matched = nullptr;
214 bool WaveSizeStored = false;
215 };
216 DenseMap<Function *, GPUPGOInvariants> GPUInvariantsCache;
217
218 /// Emit invariant PGO values at the function entry block and cache them.
219 GPUPGOInvariants &getOrCreateGPUInvariants(Function *F);
220
221 /// Lower instrumentation intrinsics in the function. Returns true if there
222 /// any lowering.
223 bool lowerIntrinsics(Function *F);
224
225 /// Register-promote counter loads and stores in loops.
226 void promoteCounterLoadStores(Function *F);
227
228 /// Returns true if relocating counters at runtime is enabled.
229 bool isRuntimeCounterRelocationEnabled() const;
230
231 /// Returns true if profile counter update register promotion is enabled.
232 bool isCounterPromotionEnabled() const;
233
234 /// Returns true if profile counter updates in \p F should be atomic.
235 bool isAtomic(const Function &F) const;
236
237 /// Return true if profile sampling is enabled.
238 bool isSamplingEnabled() const;
239
240 /// Count the number of instrumented value sites for the function.
241 void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
242
243 /// Replace instrprof.value.profile with a call to runtime library.
244 void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
245
246 /// Replace instrprof.cover with a store instruction to the coverage byte.
247 void lowerCover(InstrProfCoverInst *Inc);
248
249 /// Replace instrprof.timestamp with a call to
250 /// INSTR_PROF_PROFILE_SET_TIMESTAMP.
251 void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction);
252
253 /// Replace instrprof.increment with an increment of the appropriate value.
254 void lowerIncrement(InstrProfIncrementInst *Inc);
255
256 /// Force emitting of name vars for unused functions.
257 void lowerCoverageData(GlobalVariable *CoverageNamesVar);
258
259 /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction
260 /// using the index represented by the a temp value into a bitmap.
261 void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins);
262
263 /// Get the Bias value for data to access mmap-ed area.
264 /// Create it if it hasn't been seen.
265 GlobalVariable *getOrCreateBiasVar(StringRef VarName);
266
267 /// Compute the address of the counter value that this profiling instruction
268 /// acts on.
269 Value *getCounterAddress(InstrProfCntrInstBase *I);
270
271 /// Lower the incremental instructions under profile sampling predicates.
272 void doSampling(Instruction *I);
273
274 /// Get the region counters for an increment, creating them if necessary.
275 ///
276 /// If the counter array doesn't yet exist, the profile data variables
277 /// referring to them will also be created.
278 GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc);
279
280 /// Get the uniform entry counters for GPU divergence tracking.
281 /// These counters track how often blocks are entered with all lanes active.
282 GlobalVariable *getOrCreateUniformCounters(InstrProfCntrInstBase *Inc);
283
284 /// Create the region counters.
285 GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc,
286 StringRef Name,
287 GlobalValue::LinkageTypes Linkage);
288
289 /// Compute the address of the test vector bitmap that this profiling
290 /// instruction acts on.
291 Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I);
292
293 /// Get the region bitmaps for an increment, creating them if necessary.
294 ///
295 /// If the bitmap array doesn't yet exist, the profile data variables
296 /// referring to them will also be created.
297 GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc);
298
299 /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with
300 /// an MC/DC Decision region. The number of bytes required is indicated by
301 /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called
302 /// as part of setupProfileSection() and is conceptually very similar to
303 /// what is done for profile data counters in createRegionCounters().
304 GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
305 StringRef Name,
306 GlobalValue::LinkageTypes Linkage);
307
308 /// Set Comdat property of GV, if required.
309 void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName);
310
311 /// Setup the sections into which counters and bitmaps are allocated.
312 GlobalVariable *setupProfileSection(InstrProfInstBase *Inc,
313 InstrProfSectKind IPSK);
314
315 /// Create INSTR_PROF_DATA variable for counters and bitmaps.
316 void createDataVariable(InstrProfCntrInstBase *Inc);
317
318 /// Get the counters for virtual table values, creating them if necessary.
319 void getOrCreateVTableProfData(GlobalVariable *GV);
320
321 /// Emit the section with compressed function names.
322 void emitNameData();
323
324 /// Emit the section with compressed vtable names.
325 void emitVTableNames();
326
327 /// Emit value nodes section for value profiling.
328 void emitVNodes();
329
330 /// Emit runtime registration functions for each profile data variable.
331 void emitRegistration();
332
333 /// Emit the necessary plumbing to pull in the runtime initialization.
334 /// Returns true if a change was made.
335 bool emitRuntimeHook();
336
337 /// Add uses of our data variables and runtime hook.
338 void emitUses();
339
340 /// Create a static initializer for our data, on platforms that need it,
341 /// and for any profile output file that was specified.
342 void emitInitialization();
343
344 /// Return the __llvm_profile_data struct type.
345 StructType *getProfileDataTy();
346};
347
348///
349/// A helper class to promote one counter RMW operation in the loop
350/// into register update.
351///
352/// RWM update for the counter will be sinked out of the loop after
353/// the transformation.
354///
355class PGOCounterPromoterHelper : public LoadAndStorePromoter {
356public:
357 PGOCounterPromoterHelper(
358 const InstrumentationOptions &Opts, Instruction *L, Instruction *S,
359 SSAUpdater &SSA, Value *Init, BasicBlock *PH,
360 ArrayRef<BasicBlock *> ExitBlocks, ArrayRef<Instruction *> InsertPts,
361 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
362 LoopInfo &LI, bool IsAtomic)
363 : LoadAndStorePromoter({L, S}, SSA), Opts(Opts), Store(S),
364 ExitBlocks(ExitBlocks), InsertPts(InsertPts),
365 LoopToCandidates(LoopToCands), LI(LI), IsAtomic(IsAtomic) {
366 assert(isa<LoadInst>(L));
367 assert(isa<StoreInst>(S));
368 SSA.AddAvailableValue(BB: PH, V: Init);
369 }
370
371 void doExtraRewritesBeforeFinalDeletion() override {
372 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
373 BasicBlock *ExitBlock = ExitBlocks[i];
374 Instruction *InsertPos = InsertPts[i];
375 // Get LiveIn value into the ExitBlock. If there are multiple
376 // predecessors, the value is defined by a PHI node in this
377 // block.
378 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(BB: ExitBlock);
379 Value *Addr = cast<StoreInst>(Val: Store)->getPointerOperand();
380 Type *Ty = LiveInValue->getType();
381 IRBuilder<> Builder(InsertPos);
382 if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Val: Addr)) {
383 // If isRuntimeCounterRelocationEnabled() is true then the address of
384 // the store instruction is computed with two instructions in
385 // InstrProfiling::getCounterAddress(). We need to copy those
386 // instructions to this block to compute Addr correctly.
387 // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
388 // %Addr = inttoptr i64 %BiasAdd to i64*
389 auto *OrigBiasInst = dyn_cast<BinaryOperator>(Val: AddrInst->getOperand(i_nocapture: 0));
390 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
391 Value *BiasInst = Builder.Insert(I: OrigBiasInst->clone());
392 Addr = Builder.CreateIntToPtr(V: BiasInst,
393 DestTy: PointerType::getUnqual(C&: Ty->getContext()));
394 }
395 auto *TargetLoop =
396 Opts.iterative_counter_promotion ? LI.getLoopFor(BB: ExitBlock) : nullptr;
397 // Generate the relaxed atomic RMW if we've asked for it and no more
398 // promotion is possible.
399 if ((IsAtomic && !TargetLoop) || Opts.atomic_counter_update_promoted)
400 Builder
401 .CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: LiveInValue,
402 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic)
403 ->setNoSanitizeMetadata();
404 else {
405 LoadInst *OldVal = Builder.CreateLoad(Ty, Ptr: Addr, Name: "pgocount.promoted");
406 OldVal->setNoSanitizeMetadata();
407 auto *NewVal = Builder.CreateAdd(LHS: OldVal, RHS: LiveInValue);
408 auto *NewStore = Builder.CreateStore(Val: NewVal, Ptr: Addr);
409 NewStore->setNoSanitizeMetadata();
410
411 // Now update the parent loop's candidate list:
412 if (TargetLoop)
413 LoopToCandidates[TargetLoop].emplace_back(Args&: OldVal, Args&: NewStore);
414 }
415 }
416 }
417
418private:
419 const InstrumentationOptions &Opts;
420 Instruction *Store;
421 ArrayRef<BasicBlock *> ExitBlocks;
422 ArrayRef<Instruction *> InsertPts;
423 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
424 LoopInfo &LI;
425 const bool IsAtomic;
426};
427
428/// A helper class to do register promotion for all profile counter
429/// updates in a loop.
430///
431class PGOCounterPromoter {
432public:
433 PGOCounterPromoter(
434 const InstrumentationOptions &Opts,
435 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
436 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI, bool IsAtomic)
437 : Opts(Opts), LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI),
438 IsAtomic(IsAtomic) {
439
440 // Skip collection of ExitBlocks and InsertPts for loops that will not be
441 // able to have counters promoted.
442 SmallVector<BasicBlock *, 8> LoopExitBlocks;
443 SmallPtrSet<BasicBlock *, 8> BlockSet;
444
445 L.getExitBlocks(ExitBlocks&: LoopExitBlocks);
446 if (!isPromotionPossible(LP: &L, LoopExitBlocks))
447 return;
448
449 for (BasicBlock *ExitBlock : LoopExitBlocks) {
450 if (BlockSet.insert(Ptr: ExitBlock).second &&
451 llvm::none_of(Range: predecessors(BB: ExitBlock), P: [&](const BasicBlock *Pred) {
452 return llvm::isPresplitCoroSuspendExitEdge(Src: *Pred, Dest: *ExitBlock);
453 })) {
454 ExitBlocks.push_back(Elt: ExitBlock);
455 InsertPts.push_back(Elt: &*ExitBlock->getFirstInsertionPt());
456 }
457 }
458 }
459
460 bool run(int64_t *NumPromoted) {
461 // Move L's candidates out of LoopToCandidates before promoting them, as
462 // promoting a counter to an enclosing loop may insert a new key into
463 // LoopToCandidates and trigger DenseMap::grow().
464 auto &OrigCandidates = LoopToCandidates[&L];
465 SmallVector<LoadStorePair, 8> Candidates = std::move(OrigCandidates);
466 OrigCandidates.clear();
467 bool RC = promoteCandidates(Candidates, NumPromoted);
468 assert(LoopToCandidates[&L].empty() &&
469 "Did not expect new candidates to be added to current loop");
470 // In certain case, e.g. with -fprofile-update=atomic, we want to generate
471 // atomic updates of the PGO counters, but also perform promotion of these
472 // updates out of loops to reduce train time. The strategy is:
473 // 1) generate non-atomic load-increment-store sequence of instructions
474 // during lowerIntrinsics phase,
475 // 2) perform the promotion (in promoteCandidates function), then
476 // 3) convert all (promoted and unpromotable) updates to atomicRMW.
477 // This requires that promoted candidates are set to nullptr in the
478 // Candidates array by the promoteCandidates() function.
479 if (IsAtomic)
480 for (auto &Cand : Candidates)
481 if (Cand.first != nullptr && Cand.second != nullptr)
482 makeAtomic(Load: Cand.first, Store: Cand.second);
483 return RC;
484 }
485
486private:
487 bool promoteCandidates(SmallVectorImpl<LoadStorePair> &Candidates,
488 int64_t *NumPromoted) {
489 // Skip 'infinite' loops:
490 if (ExitBlocks.size() == 0)
491 return false;
492
493 // Skip if any of the ExitBlocks contains a ret instruction.
494 // This is to prevent dumping of incomplete profile -- if the
495 // the loop is a long running loop and dump is called in the middle
496 // of the loop, the result profile is incomplete.
497 // FIXME: add other heuristics to detect long running loops.
498 if (Opts.skip_ret_exit_block) {
499 for (auto *BB : ExitBlocks)
500 if (isa<ReturnInst>(Val: BB->getTerminator()))
501 return false;
502 }
503
504 unsigned MaxProm = getMaxNumOfPromotionsInLoop(LP: &L);
505 if (MaxProm == 0)
506 return false;
507
508 unsigned Promoted = 0;
509 for (auto &Cand : Candidates) {
510 SmallVector<PHINode *, 4> NewPHIs;
511 SSAUpdater SSA(&NewPHIs);
512 Value *InitVal = ConstantInt::get(Ty: Cand.first->getType(), V: 0);
513
514 // If BFI is set, we will use it to guide the promotions.
515 if (BFI) {
516 auto *BB = Cand.first->getParent();
517 auto InstrCount = BFI->getBlockProfileCount(BB);
518 if (!InstrCount)
519 continue;
520 auto PreheaderCount = BFI->getBlockProfileCount(BB: L.getLoopPreheader());
521 // If the average loop trip count is not greater than 1.5, we skip
522 // promotion.
523 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
524 continue;
525 }
526
527 PGOCounterPromoterHelper Promoter(
528 Opts, Cand.first, Cand.second, SSA, InitVal, L.getLoopPreheader(),
529 ExitBlocks, InsertPts, LoopToCandidates, LI, IsAtomic);
530 Promoter.run(Insts: SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
531
532 Cand = {nullptr, nullptr};
533
534 Promoted++;
535 if (Promoted >= MaxProm)
536 break;
537
538 (*NumPromoted)++;
539 if (Opts.max_counter_promotions != -1 &&
540 *NumPromoted >= Opts.max_counter_promotions)
541 break;
542 }
543
544 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
545 << L.getLoopDepth() << ")\n");
546 return Promoted != 0;
547 }
548
549private:
550 bool allowSpeculativeCounterPromotion(Loop *LP) {
551 SmallVector<BasicBlock *, 8> ExitingBlocks;
552 L.getExitingBlocks(ExitingBlocks);
553 // Not considierered speculative.
554 if (ExitingBlocks.size() == 1)
555 return true;
556 if (ExitingBlocks.size() > Opts.speculative_counter_promotion_max_exiting)
557 return false;
558 return true;
559 }
560
561 // Check whether the loop satisfies the basic conditions needed to perform
562 // Counter Promotions.
563 bool
564 isPromotionPossible(Loop *LP,
565 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
566 // We can't insert into a catchswitch.
567 if (llvm::any_of(Range: LoopExitBlocks, P: [](BasicBlock *Exit) {
568 return isa<CatchSwitchInst>(Val: Exit->getTerminator());
569 }))
570 return false;
571
572 if (!LP->hasDedicatedExits())
573 return false;
574
575 BasicBlock *PH = LP->getLoopPreheader();
576 if (!PH)
577 return false;
578
579 return true;
580 }
581
582 // Returns the max number of Counter Promotions for LP.
583 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
584 SmallVector<BasicBlock *, 8> LoopExitBlocks;
585 LP->getExitBlocks(ExitBlocks&: LoopExitBlocks);
586 if (!isPromotionPossible(LP, LoopExitBlocks))
587 return 0;
588
589 SmallVector<BasicBlock *, 8> ExitingBlocks;
590 LP->getExitingBlocks(ExitingBlocks);
591
592 // If BFI is set, we do more aggressive promotions based on BFI.
593 if (BFI)
594 return (unsigned)-1;
595
596 // Not considierered speculative.
597 if (ExitingBlocks.size() == 1)
598 return Opts.max_counter_promotions_per_loop;
599
600 if (ExitingBlocks.size() > Opts.speculative_counter_promotion_max_exiting)
601 return 0;
602
603 // Whether the target block is in a loop does not matter:
604 if (Opts.speculative_counter_promotion_to_loop)
605 return Opts.max_counter_promotions_per_loop;
606
607 // Now check the target block:
608 unsigned MaxProm = Opts.max_counter_promotions_per_loop;
609 for (auto *TargetBlock : LoopExitBlocks) {
610 auto *TargetLoop = LI.getLoopFor(BB: TargetBlock);
611 if (!TargetLoop)
612 continue;
613 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(LP: TargetLoop);
614 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
615 MaxProm =
616 std::min(a: MaxProm, b: std::max(a: MaxPromForTarget, b: PendingCandsInTarget) -
617 PendingCandsInTarget);
618 }
619 return MaxProm;
620 }
621
622 const InstrumentationOptions &Opts;
623 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
624 SmallVector<BasicBlock *, 8> ExitBlocks;
625 SmallVector<Instruction *, 8> InsertPts;
626 Loop &L;
627 LoopInfo &LI;
628 BlockFrequencyInfo *BFI;
629 const bool IsAtomic; // Whether to convert counter updates to atomics.
630};
631
632enum class ValueProfilingCallType {
633 // Individual values are tracked. Currently used for indiret call target
634 // profiling.
635 Default,
636
637 // MemOp: the memop size value profiling.
638 MemOp
639};
640
641} // end anonymous namespace
642
643PreservedAnalyses InstrProfilingLoweringPass::run(Module &M,
644 ModuleAnalysisManager &AM) {
645 FunctionAnalysisManager &FAM =
646 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
647 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
648 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
649 };
650 const InstrumentationOptions &Opts = getInstrumentationOptions(M);
651 InstrLowerer Lowerer(Opts, M, Options, GetTLI, IsCS);
652 if (!Lowerer.lower())
653 return PreservedAnalyses::all();
654
655 return PreservedAnalyses::none();
656}
657
658//
659// Perform instrumentation sampling.
660//
661// There are 3 favors of sampling:
662// (1) Full burst sampling: We transform:
663// Increment_Instruction;
664// to:
665// if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
666// Increment_Instruction;
667// }
668// __llvm_profile_sampling__ += 1;
669// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
670// __llvm_profile_sampling__ = 0;
671// }
672//
673// "__llvm_profile_sampling__" is a thread-local global shared by all PGO
674// counters (value-instrumentation and edge instrumentation).
675//
676// (2) Fast burst sampling:
677// "__llvm_profile_sampling__" variable is an unsigned type, meaning it will
678// wrap around to zero when overflows. In this case, the second check is
679// unnecessary, so we won't generate check2 when the SampledInstrPeriod is
680// set to 65536 (64K). The code after:
681// if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
682// Increment_Instruction;
683// }
684// __llvm_profile_sampling__ += 1;
685//
686// (3) Simple sampling:
687// When SampledInstrBurstDuration is set to 1, we do a simple sampling:
688// __llvm_profile_sampling__ += 1;
689// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
690// __llvm_profile_sampling__ = 0;
691// Increment_Instruction;
692// }
693//
694// Note that, the code snippet after the transformation can still be counter
695// promoted. However, with sampling enabled, counter updates are expected to
696// be infrequent, making the benefits of counter promotion negligible.
697// Moreover, counter promotion can potentially cause issues in server
698// applications, particularly when the counters are dumped without a clean
699// exit. To mitigate this risk, counter promotion is disabled by default when
700// sampling is enabled. This behavior can be overridden using the internal
701// option.
702void InstrLowerer::doSampling(Instruction *I) {
703 if (!isSamplingEnabled())
704 return;
705
706 SampledInstrumentationConfig config = getSampledInstrumentationConfig(Opts);
707 auto GetConstant = [&config](IRBuilder<> &Builder, uint32_t C) {
708 if (config.UseShort)
709 return Builder.getInt16(C);
710 else
711 return Builder.getInt32(C);
712 };
713
714 IntegerType *SamplingVarTy;
715 if (config.UseShort)
716 SamplingVarTy = Type::getInt16Ty(C&: M.getContext());
717 else
718 SamplingVarTy = Type::getInt32Ty(C&: M.getContext());
719 auto *SamplingVar =
720 M.getGlobalVariable(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
721 assert(SamplingVar && "SamplingVar not set properly");
722
723 // Create the condition for checking the burst duration.
724 Instruction *SamplingVarIncr;
725 Value *NewSamplingVarVal;
726 MDBuilder MDB(I->getContext());
727 MDNode *BranchWeight;
728 IRBuilder<> CondBuilder(I);
729 auto *LoadSamplingVar = CondBuilder.CreateLoad(Ty: SamplingVarTy, Ptr: SamplingVar);
730 if (config.IsSimpleSampling) {
731 // For the simple sampling, just create the load and increments.
732 IRBuilder<> IncBuilder(I);
733 NewSamplingVarVal =
734 IncBuilder.CreateAdd(LHS: LoadSamplingVar, RHS: GetConstant(IncBuilder, 1));
735 SamplingVarIncr = IncBuilder.CreateStore(Val: NewSamplingVarVal, Ptr: SamplingVar);
736 } else {
737 // For the burst-sampling, create the conditional update.
738 auto *DurationCond = CondBuilder.CreateICmpULE(
739 LHS: LoadSamplingVar, RHS: GetConstant(CondBuilder, config.BurstDuration - 1));
740 BranchWeight = MDB.createBranchWeights(
741 TrueWeight: config.BurstDuration, FalseWeight: config.Period - config.BurstDuration);
742 Instruction *ThenTerm = SplitBlockAndInsertIfThen(
743 Cond: DurationCond, SplitBefore: I, /* Unreachable */ false, BranchWeights: BranchWeight);
744 IRBuilder<> IncBuilder(I);
745 NewSamplingVarVal =
746 IncBuilder.CreateAdd(LHS: LoadSamplingVar, RHS: GetConstant(IncBuilder, 1));
747 SamplingVarIncr = IncBuilder.CreateStore(Val: NewSamplingVarVal, Ptr: SamplingVar);
748 I->moveBefore(InsertPos: ThenTerm->getIterator());
749 }
750
751 if (config.IsFastSampling)
752 return;
753
754 // Create the condition for checking the period.
755 Instruction *ThenTerm, *ElseTerm;
756 IRBuilder<> PeriodCondBuilder(SamplingVarIncr);
757 auto *PeriodCond = PeriodCondBuilder.CreateICmpUGE(
758 LHS: NewSamplingVarVal, RHS: GetConstant(PeriodCondBuilder, config.Period));
759 BranchWeight = MDB.createBranchWeights(TrueWeight: 1, FalseWeight: config.Period - 1);
760 SplitBlockAndInsertIfThenElse(Cond: PeriodCond, SplitBefore: SamplingVarIncr, ThenTerm: &ThenTerm,
761 ElseTerm: &ElseTerm, BranchWeights: BranchWeight);
762
763 // For the simple sampling, the counter update happens in sampling var reset.
764 if (config.IsSimpleSampling)
765 I->moveBefore(InsertPos: ThenTerm->getIterator());
766
767 IRBuilder<> ResetBuilder(ThenTerm);
768 ResetBuilder.CreateStore(Val: GetConstant(ResetBuilder, 0), Ptr: SamplingVar);
769 SamplingVarIncr->moveBefore(InsertPos: ElseTerm->getIterator());
770}
771
772bool InstrLowerer::lowerIntrinsics(Function *F) {
773 bool MadeChange = false;
774 PromotionCandidates.clear();
775 SmallVector<InstrProfInstBase *, 8> InstrProfInsts;
776
777 // To ensure compatibility with sampling, we save the intrinsics into
778 // a buffer to prevent potential breakage of the iterator (as the
779 // intrinsics will be moved to a different BB).
780 for (BasicBlock &BB : *F) {
781 for (Instruction &Instr : llvm::make_early_inc_range(Range&: BB)) {
782 if (auto *IP = dyn_cast<InstrProfInstBase>(Val: &Instr))
783 InstrProfInsts.push_back(Elt: IP);
784 }
785 }
786
787 for (auto *Instr : InstrProfInsts) {
788 doSampling(I: Instr);
789 if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(Val: Instr)) {
790 lowerIncrement(Inc: IPIS);
791 MadeChange = true;
792 } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(Val: Instr)) {
793 lowerIncrement(Inc: IPI);
794 MadeChange = true;
795 } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(Val: Instr)) {
796 lowerTimestamp(TimestampInstruction: IPC);
797 MadeChange = true;
798 } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(Val: Instr)) {
799 lowerCover(Inc: IPC);
800 MadeChange = true;
801 } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(Val: Instr)) {
802 lowerValueProfileInst(Ins: IPVP);
803 MadeChange = true;
804 } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(Val: Instr)) {
805 IPMP->eraseFromParent();
806 MadeChange = true;
807 } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(Val: Instr)) {
808 lowerMCDCTestVectorBitmapUpdate(Ins: IPBU);
809 MadeChange = true;
810 }
811 }
812
813 if (!MadeChange)
814 return false;
815
816 promoteCounterLoadStores(F);
817 return true;
818}
819
820bool InstrLowerer::isRuntimeCounterRelocationEnabled() const {
821 // Mach-O don't support weak external references.
822 if (TT.isOSBinFormatMachO())
823 return false;
824
825 // Fuchsia uses runtime counter relocation by default.
826 return valueOr(X: Opts.runtime_counter_relocation, Default: TT.isOSFuchsia());
827}
828
829bool InstrLowerer::isSamplingEnabled() const {
830 return valueOr(X: Opts.sampled_instrumentation, Default: Options.Sampling);
831}
832
833bool InstrLowerer::isCounterPromotionEnabled() const {
834 return valueOr(X: Opts.do_counter_promotion, Default: Options.DoCounterPromotion);
835}
836
837bool InstrLowerer::isAtomic(const Function &F) const {
838 // Non-atomic counter updates are data races, which ThreadSanitizer would
839 // report. Use atomic updates in functions that TSan instruments.
840 return valueOr(X: Opts.instrprof_atomic_counter_update_all,
841 Default: Options.Atomic || F.hasFnAttribute(Kind: Attribute::SanitizeThread));
842}
843
844static void doAtomicCheck(Function *F) {
845 for (const llvm::Instruction &I : llvm::instructions(F)) {
846 const Value *Addr = nullptr;
847 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
848 Addr = LI->getOperand(i_nocapture: 0);
849 else if (const StoreInst *LI = dyn_cast<StoreInst>(Val: &I))
850 Addr = LI->getOperand(i_nocapture: 1);
851
852 if (Addr && Addr->stripInBoundsOffsets()->getName().starts_with(
853 Prefix: getInstrProfCountersVarPrefix())) {
854 LLVM_DEBUG(dbgs() << "Missed candidate: "; I.dump());
855 report_fatal_error(reason: "Candidate load/store not converted to atomic");
856 }
857 }
858}
859
860void InstrLowerer::promoteCounterLoadStores(Function *F) {
861 if (!isCounterPromotionEnabled())
862 return;
863
864 CycleInfo CI;
865 CI.compute(F&: *F);
866 LoopInfo LI;
867 LI.analyze(F);
868 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
869
870 std::unique_ptr<BlockFrequencyInfo> BFI;
871 if (Options.UseBFIInPromotion) {
872 std::unique_ptr<BranchProbabilityInfo> BPI;
873 BPI.reset(p: new BranchProbabilityInfo(*F, CI, &GetTLI(*F)));
874 BFI.reset(p: new BlockFrequencyInfo(*F, *BPI, CI));
875 }
876
877 for (const auto &LoadStore : PromotionCandidates) {
878 auto *CounterLoad = LoadStore.first;
879 auto *CounterStore = LoadStore.second;
880 BasicBlock *BB = CounterLoad->getParent();
881 Loop *ParentLoop = LI.getLoopFor(BB);
882 if (!ParentLoop) {
883 if (isAtomic(F: *F))
884 makeAtomic(Load: CounterLoad, Store: CounterStore);
885 continue;
886 }
887 LoopPromotionCandidates[ParentLoop].emplace_back(Args&: CounterLoad, Args&: CounterStore);
888 }
889
890 SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
891
892 // Do a post-order traversal of the loops so that counter updates can be
893 // iteratively hoisted outside the loop nest.
894 for (auto *Loop : llvm::reverse(C&: Loops)) {
895 PGOCounterPromoter Promoter(Opts, LoopPromotionCandidates, *Loop, LI,
896 BFI.get(), isAtomic(F: *F));
897 Promoter.run(NumPromoted: &TotalCountersPromoted);
898 }
899
900 if (isAtomic(F: *F) && Opts.verify_atomic_counter_promoted)
901 doAtomicCheck(F);
902}
903
904static bool needsRuntimeHookUnconditionally(const Triple &TT) {
905 // On Fuchsia, we only need runtime hook if any counters are present.
906 if (TT.isOSFuchsia())
907 return false;
908
909 return true;
910}
911
912/// Check if the module contains uses of any profiling intrinsics.
913static bool containsProfilingIntrinsics(Module &M) {
914 auto containsIntrinsic = [&](int ID) {
915 if (auto *F = Intrinsic::getDeclarationIfExists(M: &M, id: ID))
916 return !F->use_empty();
917 return false;
918 };
919 return containsIntrinsic(Intrinsic::instrprof_cover) ||
920 containsIntrinsic(Intrinsic::instrprof_increment) ||
921 containsIntrinsic(Intrinsic::instrprof_increment_step) ||
922 containsIntrinsic(Intrinsic::instrprof_timestamp) ||
923 containsIntrinsic(Intrinsic::instrprof_value_profile);
924}
925
926bool InstrLowerer::lower() {
927 bool MadeChange = false;
928 bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
929 if (NeedsRuntimeHook)
930 MadeChange = emitRuntimeHook();
931
932 if (!IsCS && isSamplingEnabled())
933 createProfileSamplingVar(M);
934
935 bool ContainsProfiling = containsProfilingIntrinsics(M);
936 GlobalVariable *CoverageNamesVar =
937 M.getNamedGlobal(Name: getCoverageUnusedNamesVarName());
938 // Improve compile time by avoiding linear scans when there is no work.
939 if (!ContainsProfiling && !CoverageNamesVar)
940 return MadeChange;
941
942 // We did not know how many value sites there would be inside
943 // the instrumented function. This is counting the number of instrumented
944 // target value sites to enter it as field in the profile data variable.
945 for (Function &F : M) {
946 InstrProfCntrInstBase *FirstProfInst = nullptr;
947 for (BasicBlock &BB : F) {
948 for (auto I = BB.begin(), E = BB.end(); I != E; I++) {
949 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Val&: I))
950 computeNumValueSiteCounts(Ins: Ind);
951 else {
952 if (FirstProfInst == nullptr &&
953 (isa<InstrProfIncrementInst>(Val: I) || isa<InstrProfCoverInst>(Val: I)))
954 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(Val&: I);
955 // If the MCDCBitmapParameters intrinsic seen, create the bitmaps.
956 if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(Val&: I))
957 static_cast<void>(getOrCreateRegionBitmaps(Inc: Params));
958 }
959 }
960 }
961
962 // Use a profile intrinsic to create the region counters and data variable.
963 // Also create the data variable based on the MCDCParams.
964 if (FirstProfInst != nullptr) {
965 static_cast<void>(getOrCreateRegionCounters(Inc: FirstProfInst));
966 }
967 }
968
969 if (EnableVTableValueProfiling)
970 for (GlobalVariable &GV : M.globals())
971 // Global variables with type metadata are virtual table variables.
972 if (GV.hasMetadata(KindID: LLVMContext::MD_type))
973 getOrCreateVTableProfData(GV: &GV);
974
975 for (Function &F : M)
976 MadeChange |= lowerIntrinsics(F: &F);
977
978 if (CoverageNamesVar) {
979 lowerCoverageData(CoverageNamesVar);
980 MadeChange = true;
981 }
982
983 if (!MadeChange)
984 return false;
985
986 emitVNodes();
987 emitNameData();
988 emitVTableNames();
989
990 // Emit runtime hook for the cases where the target does not unconditionally
991 // require pulling in profile runtime, and coverage is enabled on code that is
992 // not eliminated by the front-end, e.g. unused functions with internal
993 // linkage.
994 if (!NeedsRuntimeHook && ContainsProfiling)
995 emitRuntimeHook();
996
997 emitRegistration();
998 emitUses();
999 emitInitialization();
1000 return true;
1001}
1002
1003static FunctionCallee getOrInsertValueProfilingCall(
1004 Module &M, const TargetLibraryInfo &TLI,
1005 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
1006 LLVMContext &Ctx = M.getContext();
1007 auto *ReturnTy = Type::getVoidTy(C&: M.getContext());
1008
1009 AttributeList AL;
1010 if (auto AK = TLI.getExtAttrForI32Param(Signed: false))
1011 AL = AL.addParamAttribute(C&: M.getContext(), ArgNo: 2, Kind: AK);
1012
1013 assert((CallType == ValueProfilingCallType::Default ||
1014 CallType == ValueProfilingCallType::MemOp) &&
1015 "Must be Default or MemOp");
1016 Type *ParamTypes[] = {
1017#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
1018#include "llvm/ProfileData/InstrProfData.inc"
1019 };
1020 auto *ValueProfilingCallTy =
1021 FunctionType::get(Result: ReturnTy, Params: ArrayRef(ParamTypes), isVarArg: false);
1022 StringRef FuncName = CallType == ValueProfilingCallType::Default
1023 ? getInstrProfValueProfFuncName()
1024 : getInstrProfValueProfMemOpFuncName();
1025 return M.getOrInsertFunction(Name: FuncName, T: ValueProfilingCallTy, AttributeList: AL);
1026}
1027
1028void InstrLowerer::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
1029 GlobalVariable *Name = Ind->getName();
1030 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1031 uint64_t Index = Ind->getIndex()->getZExtValue();
1032 auto &PD = ProfileDataMap[Name];
1033 PD.NumValueSites[ValueKind] =
1034 std::max(a: PD.NumValueSites[ValueKind], b: (uint32_t)(Index + 1));
1035}
1036
1037void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
1038 // TODO: Value profiling heavily depends on the data section which is omitted
1039 // in lightweight mode. We need to move the value profile pointer to the
1040 // Counter struct to get this working.
1041 assert(
1042 !Opts.profile_correlate &&
1043 "Value profiling is not yet supported with lightweight instrumentation");
1044 GlobalVariable *Name = Ind->getName();
1045 auto It = ProfileDataMap.find(Val: Name);
1046 assert(It != ProfileDataMap.end() && It->second.DataVar &&
1047 "value profiling detected in function with no counter increment");
1048
1049 GlobalVariable *DataVar = It->second.DataVar;
1050 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1051 uint64_t Index = Ind->getIndex()->getZExtValue();
1052 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
1053 Index += It->second.NumValueSites[Kind];
1054
1055 IRBuilder<> Builder(Ind);
1056 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
1057 llvm::InstrProfValueKind::IPVK_MemOPSize);
1058 CallInst *Call = nullptr;
1059 auto *TLI = &GetTLI(*Ind->getFunction());
1060 auto *NormalizedDataVarPtr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1061 C: DataVar, Ty: PointerType::get(C&: M.getContext(), AddressSpace: 0));
1062
1063 // To support value profiling calls within Windows exception handlers, funclet
1064 // information contained within operand bundles needs to be copied over to
1065 // the library call. This is required for the IR to be processed by the
1066 // WinEHPrepare pass.
1067 SmallVector<OperandBundleDef, 1> OpBundles;
1068 Ind->getOperandBundlesAsDefs(Defs&: OpBundles);
1069 if (!IsMemOpSize) {
1070 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1071 Builder.getInt32(C: Index)};
1072 Call = Builder.CreateCall(Callee: getOrInsertValueProfilingCall(M, TLI: *TLI), Args,
1073 OpBundles);
1074 } else {
1075 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1076 Builder.getInt32(C: Index)};
1077 Call = Builder.CreateCall(
1078 Callee: getOrInsertValueProfilingCall(M, TLI: *TLI, CallType: ValueProfilingCallType::MemOp),
1079 Args, OpBundles);
1080 }
1081 if (auto AK = TLI->getExtAttrForI32Param(Signed: false))
1082 Call->addParamAttr(ArgNo: 2, Kind: AK);
1083 Ind->replaceAllUsesWith(V: Call);
1084 Ind->eraseFromParent();
1085}
1086
1087GlobalVariable *InstrLowerer::getOrCreateBiasVar(StringRef VarName) {
1088 GlobalVariable *Bias = M.getGlobalVariable(Name: VarName);
1089 if (Bias)
1090 return Bias;
1091
1092 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1093
1094 // Compiler must define this variable when runtime counter relocation
1095 // is being used. Runtime has a weak external reference that is used
1096 // to check whether that's the case or not.
1097 Bias = new GlobalVariable(M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
1098 Constant::getNullValue(Ty: Int64Ty), VarName);
1099 Bias->setVisibility(GlobalVariable::HiddenVisibility);
1100 // A definition that's weak (linkonce_odr) without being in a COMDAT
1101 // section wouldn't lead to link errors, but it would lead to a dead
1102 // data word from every TU but one. Putting it in COMDAT ensures there
1103 // will be exactly one data slot in the link.
1104 if (TT.supportsCOMDAT())
1105 Bias->setComdat(M.getOrInsertComdat(Name: VarName));
1106
1107 return Bias;
1108}
1109
1110Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
1111 auto *Counters = getOrCreateRegionCounters(Inc: I);
1112 IRBuilder<> Builder(I);
1113
1114 if (isa<InstrProfTimestampInst>(Val: I))
1115 Counters->setAlignment(Align(8));
1116
1117 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
1118 Ty: Counters->getValueType(), Ptr: Counters, Idx0: 0, Idx1: I->getIndex()->getZExtValue());
1119
1120 if (!isRuntimeCounterRelocationEnabled())
1121 return Addr;
1122
1123 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1124 Function *Fn = I->getParent()->getParent();
1125 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
1126 if (!BiasLI) {
1127 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1128 auto *Bias = getOrCreateBiasVar(VarName: getInstrProfCounterBiasVarName());
1129 BiasLI = EntryBuilder.CreateLoad(Ty: Int64Ty, Ptr: Bias, Name: "profc_bias");
1130 // Bias doesn't change after startup.
1131 BiasLI->setMetadata(KindID: LLVMContext::MD_invariant_load,
1132 Node: MDNode::get(Context&: M.getContext(), MDs: {}));
1133 }
1134 auto *Add = Builder.CreateAdd(LHS: Builder.CreatePtrToInt(V: Addr, DestTy: Int64Ty), RHS: BiasLI);
1135 return Builder.CreateIntToPtr(V: Add, DestTy: Addr->getType());
1136}
1137
1138Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
1139 auto *Bitmaps = getOrCreateRegionBitmaps(Inc: I);
1140 if (!isRuntimeCounterRelocationEnabled())
1141 return Bitmaps;
1142
1143 // Put BiasLI onto the entry block.
1144 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1145 Function *Fn = I->getFunction();
1146 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1147 auto *Bias = getOrCreateBiasVar(VarName: getInstrProfBitmapBiasVarName());
1148 auto *BiasLI = EntryBuilder.CreateLoad(Ty: Int64Ty, Ptr: Bias, Name: "profbm_bias");
1149 // Assume BiasLI invariant (in the function at least)
1150 BiasLI->setMetadata(KindID: LLVMContext::MD_invariant_load,
1151 Node: MDNode::get(Context&: M.getContext(), MDs: {}));
1152
1153 // Add Bias to Bitmaps and put it before the intrinsic.
1154 IRBuilder<> Builder(I);
1155 return Builder.CreatePtrAddWithSExtOrTrunc(Ptr: Bitmaps, Offset: BiasLI, Name: "profbm_addr");
1156}
1157
1158void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
1159 auto *Addr = getCounterAddress(I: CoverInstruction);
1160 IRBuilder<> Builder(CoverInstruction);
1161 // Optimizations can merge the counter update with a user access, and then
1162 // ThreadSanitizer ends up instrumenting the counter access - and can report
1163 // it as a data race. Making updates atomic cures this.
1164 Function *F = CoverInstruction->getFunction();
1165 bool MakeAtomic =
1166 F->hasFnAttribute(Kind: Attribute::SanitizeThread) && isAtomic(F: *F);
1167 if (Opts.conditional_counter_update) {
1168 Instruction *SplitBefore = CoverInstruction->getNextNode();
1169 auto &Ctx = CoverInstruction->getParent()->getContext();
1170 auto *Int8Ty = llvm::Type::getInt8Ty(C&: Ctx);
1171 auto *Load = Builder.CreateLoad(Ty: Int8Ty, Ptr: Addr, Name: "pgocount");
1172 Load->setNoSanitizeMetadata();
1173 if (MakeAtomic)
1174 Load->setAtomic(Ordering: AtomicOrdering::Monotonic);
1175 Value *Cmp = Builder.CreateIsNotNull(Arg: Load, Name: "pgocount.ifnonzero");
1176 Instruction *ThenBranch =
1177 SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore, Unreachable: false);
1178 Builder.SetInsertPoint(ThenBranch);
1179 }
1180
1181 // We store zero to represent that this block is covered.
1182 auto *Store = Builder.CreateStore(Val: Builder.getInt8(C: 0), Ptr: Addr);
1183 Store->setNoSanitizeMetadata();
1184 if (MakeAtomic)
1185 Store->setAtomic(Ordering: AtomicOrdering::Monotonic);
1186 CoverInstruction->eraseFromParent();
1187}
1188
1189void InstrLowerer::lowerTimestamp(
1190 InstrProfTimestampInst *TimestampInstruction) {
1191 assert(TimestampInstruction->getIndex()->isNullValue() &&
1192 "timestamp probes are always the first probe for a function");
1193 auto &Ctx = M.getContext();
1194 auto *TimestampAddr = getCounterAddress(I: TimestampInstruction);
1195 IRBuilder<> Builder(TimestampInstruction);
1196 auto *CalleeTy =
1197 FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: TimestampAddr->getType(), isVarArg: false);
1198 auto Callee = M.getOrInsertFunction(
1199 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), T: CalleeTy);
1200 Builder.CreateCall(Callee, Args: {TimestampAddr});
1201 TimestampInstruction->eraseFromParent();
1202}
1203
1204InstrLowerer::GPUPGOInvariants &
1205InstrLowerer::getOrCreateGPUInvariants(Function *F) {
1206 auto It = GPUInvariantsCache.find(Val: F);
1207 if (It != GPUInvariantsCache.end())
1208 return It->second;
1209
1210 LLVMContext &Context = M.getContext();
1211 auto *Int32Ty = Type::getInt32Ty(C&: Context);
1212
1213 BasicBlock &EntryBB = F->getEntryBlock();
1214 IRBuilder<> Builder(&*EntryBB.getFirstInsertionPt());
1215
1216 Value *Matched = ConstantInt::getTrue(Context);
1217 if (Opts.offload_pgo_sampling > 0) {
1218 FunctionCallee IsSampledFn =
1219 M.getOrInsertFunction(Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(
1220 CallImpl: RTLIB::impl___llvm_profile_sampling_gpu),
1221 RetTy: Int32Ty, Args: Int32Ty);
1222 Value *SampledInt = Builder.CreateCall(
1223 Callee: IsSampledFn, Args: {ConstantInt::get(Ty: Int32Ty, V: Opts.offload_pgo_sampling)},
1224 Name: "pgo.sampled");
1225 Matched = Builder.CreateICmpNE(LHS: SampledInt, RHS: ConstantInt::get(Ty: Int32Ty, V: 0),
1226 Name: "pgo.matched");
1227 }
1228
1229 auto &Inv = GPUInvariantsCache[F];
1230 Inv.Matched = Matched;
1231 return Inv;
1232}
1233
1234void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
1235 IRBuilder<> Builder(Inc);
1236 if (isGPUProfTarget(M)) {
1237 Function *F = Inc->getFunction();
1238 auto &Inv = getOrCreateGPUInvariants(F);
1239
1240 LLVMContext &Context = M.getContext();
1241 auto *Int64Ty = Type::getInt64Ty(C&: Context);
1242 auto *PtrTy = PointerType::getUnqual(C&: Context);
1243
1244 auto *Addr = getCounterAddress(I: Inc);
1245
1246 // Store the device wave/warp size into the profile data struct once per
1247 // function. AMDGPU folds llvm.amdgcn.wavefrontsize to the subtarget's
1248 // constant; other GPUs use their fixed warp size.
1249 if (!Inv.WaveSizeStored) {
1250 Inv.WaveSizeStored = true;
1251 GlobalVariable *NamePtr = Inc->getName();
1252 auto &PD = ProfileDataMap[NamePtr];
1253 if (PD.DataVar) {
1254 IRBuilder<> EntryBuilder(&*F->getEntryBlock().getFirstInsertionPt());
1255 Value *WaveSize16 = nullptr;
1256 // Look the intrinsic up by name so this target-agnostic pass does not
1257 // pull in IntrinsicsAMDGPU.h. AMDGPU folds the intrinsic to the
1258 // subtarget's wavefront size; other GPUs fall back to a 32-lane warp.
1259 if (TT.isAMDGPU()) {
1260 Intrinsic::ID WaveSizeID =
1261 Intrinsic::lookupIntrinsicID(Name: "llvm.amdgcn.wavefrontsize");
1262 if (WaveSizeID != Intrinsic::not_intrinsic) {
1263 Function *WaveSizeFn =
1264 Intrinsic::getOrInsertDeclaration(M: &M, id: WaveSizeID);
1265 Value *WaveSize = EntryBuilder.CreateCall(Callee: WaveSizeFn);
1266 WaveSize16 = EntryBuilder.CreateTrunc(
1267 V: WaveSize, DestTy: Type::getInt16Ty(C&: Context), Name: "wavesize.i16");
1268 }
1269 }
1270 if (!WaveSize16)
1271 WaveSize16 = ConstantInt::get(Ty: Type::getInt16Ty(C&: Context), V: 32);
1272 Value *WaveSizeAddr = EntryBuilder.CreateStructGEP(
1273 Ty: PD.DataVar->getValueType(), Ptr: PD.DataVar, Idx: 9, Name: "profd.wavesize");
1274 EntryBuilder.CreateStore(Val: WaveSize16, Ptr: WaveSizeAddr);
1275 }
1276 }
1277
1278 GlobalVariable *UniformCounters = getOrCreateUniformCounters(Inc);
1279 Value *UniformAddrArg = ConstantPointerNull::get(T: PtrTy);
1280 if (UniformCounters) {
1281 Value *UniformIndices[] = {Builder.getInt32(C: 0), Inc->getIndex()};
1282 Value *UniformAddr = Builder.CreateInBoundsGEP(
1283 Ty: UniformCounters->getValueType(), Ptr: UniformCounters, IdxList: UniformIndices,
1284 Name: "unifctr.addr");
1285 UniformAddrArg =
1286 Builder.CreatePointerBitCastOrAddrSpaceCast(V: UniformAddr, DestTy: PtrTy);
1287 }
1288 Value *CastAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Addr, DestTy: PtrTy);
1289 Value *StepI64 =
1290 Builder.CreateZExtOrTrunc(V: Inc->getStep(), DestTy: Int64Ty, Name: "step.i64");
1291
1292 auto *CalleeTy = FunctionType::get(Result: Type::getVoidTy(C&: Context),
1293 Params: {PtrTy, PtrTy, Int64Ty}, isVarArg: false);
1294 FunctionCallee Callee =
1295 M.getOrInsertFunction(Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(
1296 CallImpl: RTLIB::impl___llvm_profile_instrument_gpu),
1297 T: CalleeTy);
1298
1299 if (Opts.offload_pgo_sampling > 0) {
1300 BasicBlock *CurBB = Builder.GetInsertBlock();
1301 BasicBlock *ContBB =
1302 CurBB->splitBasicBlock(I: BasicBlock::iterator(Inc), BBName: "po_cont");
1303 BasicBlock *ThenBB = BasicBlock::Create(Context, Name: "po_then", Parent: F);
1304
1305 CurBB->getTerminator()->eraseFromParent();
1306 IRBuilder<> HeadBuilder(CurBB);
1307 HeadBuilder.CreateCondBr(Cond: Inv.Matched, True: ThenBB, False: ContBB);
1308
1309 IRBuilder<> ThenBuilder(ThenBB);
1310 ThenBuilder.CreateCall(Callee, Args: {CastAddr, UniformAddrArg, StepI64});
1311 ThenBuilder.CreateBr(Dest: ContBB);
1312 } else {
1313 Builder.CreateCall(Callee, Args: {CastAddr, UniformAddrArg, StepI64});
1314 }
1315 Inc->eraseFromParent();
1316 return;
1317 }
1318
1319 auto *Addr = getCounterAddress(I: Inc);
1320 // If promotion is enabled then delay generating atomic updates until
1321 // after promotion is done.
1322 if ((!isCounterPromotionEnabled() && isAtomic(F: *Inc->getFunction())) ||
1323 (Inc->getIndex()->isNullValue() && Opts.atomic_first_counter)) {
1324 Builder
1325 .CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: Inc->getStep(), Align: MaybeAlign(),
1326 Ordering: AtomicOrdering::Monotonic)
1327 ->setNoSanitizeMetadata();
1328 } else {
1329 Value *IncStep = Inc->getStep();
1330 auto *Load = Builder.CreateLoad(Ty: IncStep->getType(), Ptr: Addr, Name: "pgocount");
1331 Load->setNoSanitizeMetadata();
1332 auto *Count = Builder.CreateAdd(LHS: Load, RHS: Inc->getStep());
1333 auto *Store = Builder.CreateStore(Val: Count, Ptr: Addr);
1334 Store->setNoSanitizeMetadata();
1335 if (isCounterPromotionEnabled())
1336 PromotionCandidates.emplace_back(args: cast<Instruction>(Val: Load), args&: Store);
1337 }
1338 Inc->eraseFromParent();
1339}
1340
1341void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
1342 ConstantArray *Names =
1343 cast<ConstantArray>(Val: CoverageNamesVar->getInitializer());
1344 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
1345 Constant *NC = Names->getOperand(i_nocapture: I);
1346 Value *V = NC->stripPointerCasts();
1347 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
1348 GlobalVariable *Name = cast<GlobalVariable>(Val: V);
1349
1350 Name->setLinkage(GlobalValue::PrivateLinkage);
1351 ReferencedNames.push_back(x: Name);
1352 if (isa<ConstantExpr>(Val: NC))
1353 NC->dropAllReferences();
1354 }
1355 CoverageNamesVar->eraseFromParent();
1356}
1357
1358void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
1359 InstrProfMCDCTVBitmapUpdate *Update) {
1360 auto &Ctx = M.getContext();
1361 IRBuilder<> Builder(Update);
1362 auto *Int8Ty = Type::getInt8Ty(C&: Ctx);
1363 auto *Int32Ty = Type::getInt32Ty(C&: Ctx);
1364 auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
1365 auto *BitmapAddr = getBitmapAddress(I: Update);
1366
1367 // Load Temp Val + BitmapIdx.
1368 // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
1369 auto *Temp = Builder.CreateAdd(
1370 LHS: Builder.CreateLoad(Ty: Int32Ty, Ptr: MCDCCondBitmapAddr, Name: "mcdc.temp"),
1371 RHS: Update->getBitmapIndex());
1372
1373 // Calculate byte offset using div8.
1374 // %1 = lshr i32 %mcdc.temp, 3
1375 auto *BitmapByteOffset = Builder.CreateLShr(LHS: Temp, RHS: 0x3);
1376
1377 // Add byte offset to section base byte address.
1378 // %4 = getelementptr inbounds i8, ptr @__profbm_test, i32 %1
1379 auto *BitmapByteAddr = Builder.CreatePtrAddWithZExtOrTrunc(
1380 Ptr: BitmapAddr, Offset: BitmapByteOffset, Name: "", NW: GEPNoWrapFlags::inBounds());
1381
1382 // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8)
1383 // %5 = and i32 %mcdc.temp, 7
1384 // %6 = trunc i32 %5 to i8
1385 auto *BitToSet = Builder.CreateTrunc(V: Builder.CreateAnd(LHS: Temp, RHS: 0x7), DestTy: Int8Ty);
1386
1387 // Shift bit offset left to form a bitmap.
1388 // %7 = shl i8 1, %6
1389 auto *ShiftedVal = Builder.CreateShl(LHS: Builder.getInt8(C: 0x1), RHS: BitToSet);
1390
1391 // Load profile bitmap byte.
1392 // %mcdc.bits = load i8, ptr %4, align 1
1393 auto *Bitmap = Builder.CreateLoad(Ty: Int8Ty, Ptr: BitmapByteAddr, Name: "mcdc.bits");
1394 Bitmap->setNoSanitizeMetadata();
1395
1396 if (isAtomic(F: *Update->getFunction())) {
1397 if (Update->getFunction()->hasFnAttribute(Kind: Attribute::SanitizeThread))
1398 Bitmap->setAtomic(Ordering: AtomicOrdering::Monotonic);
1399
1400 // If ((Bitmap & Val) != Val), then execute atomic (Bitmap |= Val).
1401 // Note, just-loaded Bitmap might not be up-to-date. Use it just for
1402 // early testing.
1403 auto *Masked = Builder.CreateAnd(LHS: Bitmap, RHS: ShiftedVal);
1404 auto *ShouldStore = Builder.CreateICmpNE(LHS: Masked, RHS: ShiftedVal);
1405
1406 // Assume updating will be rare.
1407 auto *Unlikely = MDBuilder(Ctx).createUnlikelyBranchWeights();
1408 Instruction *ThenBranch =
1409 SplitBlockAndInsertIfThen(Cond: ShouldStore, SplitBefore: Update, Unreachable: false, BranchWeights: Unlikely);
1410
1411 // Execute if (unlikely(ShouldStore)).
1412 Builder.SetInsertPoint(ThenBranch);
1413 Builder
1414 .CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: BitmapByteAddr, Val: ShiftedVal,
1415 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic)
1416 ->setNoSanitizeMetadata();
1417 } else {
1418 // Perform logical OR of profile bitmap byte and shifted bit offset.
1419 // %8 = or i8 %mcdc.bits, %7
1420 auto *Result = Builder.CreateOr(LHS: Bitmap, RHS: ShiftedVal);
1421
1422 // Store the updated profile bitmap byte.
1423 // store i8 %8, ptr %3, align 1
1424 Builder.CreateStore(Val: Result, Ptr: BitmapByteAddr)->setNoSanitizeMetadata();
1425 }
1426
1427 Update->eraseFromParent();
1428}
1429
1430/// Get the name of a profiling variable for a particular function.
1431static std::string getVarName(const InstrumentationOptions &Opts,
1432 InstrProfInstBase *Inc, StringRef Prefix,
1433 bool &Renamed) {
1434 StringRef NamePrefix = getInstrProfNameVarPrefix();
1435 StringRef Name = Inc->getName()->getName().substr(Start: NamePrefix.size());
1436 Function *F = Inc->getParent()->getParent();
1437 Module *M = F->getParent();
1438 if (!Opts.hash_based_counter_split || !isIRPGOFlagSet(M) ||
1439 !canRenameComdatFunc(F: *F)) {
1440 Renamed = false;
1441 return (Prefix + Name).str();
1442 }
1443 Renamed = true;
1444 uint64_t FuncHash = Inc->getHash()->getZExtValue();
1445 SmallVector<char, 24> HashPostfix;
1446 if (Name.ends_with(Suffix: (Twine(".") + Twine(FuncHash)).toStringRef(Out&: HashPostfix)))
1447 return (Prefix + Name).str();
1448 return (Prefix + Name + "." + Twine(FuncHash)).str();
1449}
1450
1451static inline bool shouldRecordFunctionAddr(Function *F) {
1452 // Only record function addresses if IR PGO is enabled or if clang value
1453 // profiling is enabled. Recording function addresses greatly increases object
1454 // file size, because it prevents the inliner from deleting functions that
1455 // have been inlined everywhere.
1456 if (!profDataReferencedByCode(M: *F->getParent()))
1457 return false;
1458
1459 // Check the linkage
1460 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
1461 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
1462 !HasAvailableExternallyLinkage)
1463 return true;
1464
1465 // A function marked 'alwaysinline' with available_externally linkage can't
1466 // have its address taken. Doing so would create an undefined external ref to
1467 // the function, which would fail to link.
1468 if (HasAvailableExternallyLinkage &&
1469 F->hasFnAttribute(Kind: Attribute::AlwaysInline))
1470 return false;
1471
1472 // Prohibit function address recording if the function is both internal and
1473 // COMDAT. This avoids the profile data variable referencing internal symbols
1474 // in COMDAT.
1475 if (F->hasLocalLinkage() && F->hasComdat())
1476 return false;
1477
1478 // Check uses of this function for other than direct calls or invokes to it.
1479 // Inline virtual functions have linkeOnceODR linkage. When a key method
1480 // exists, the vtable will only be emitted in the TU where the key method
1481 // is defined. In a TU where vtable is not available, the function won't
1482 // be 'addresstaken'. If its address is not recorded here, the profile data
1483 // with missing address may be picked by the linker leading to missing
1484 // indirect call target info.
1485 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
1486}
1487
1488static inline bool shouldUsePublicSymbol(Function *Fn) {
1489 // It isn't legal to make an alias of this function at all
1490 if (Fn->isDeclarationForLinker())
1491 return true;
1492
1493 // Symbols with local linkage can just use the symbol directly without
1494 // introducing relocations
1495 if (Fn->hasLocalLinkage())
1496 return true;
1497
1498 // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some
1499 // unfavorable interaction between the new alias and the alias renaming done
1500 // in LowerTypeTests under ThinLTO. For comdat functions that would normally
1501 // be deduplicated, but the renaming scheme ends up preventing renaming, since
1502 // it creates unique names for each alias, resulting in duplicated symbols. In
1503 // the future, we should update the CFI related passes to migrate these
1504 // aliases to the same module as the jump-table they refer to will be defined.
1505 if (Fn->hasMetadata(KindID: LLVMContext::MD_type))
1506 return true;
1507
1508 // For comdat functions, an alias would need the same linkage as the original
1509 // function and hidden visibility. There is no point in adding an alias with
1510 // identical linkage an visibility to avoid introducing symbolic relocations.
1511 if (Fn->hasComdat() &&
1512 (Fn->getVisibility() == GlobalValue::VisibilityTypes::HiddenVisibility))
1513 return true;
1514
1515 // its OK to use an alias
1516 return false;
1517}
1518
1519static inline Constant *getFuncAddrForProfData(Function *Fn) {
1520 auto *Int8PtrTy = PointerType::getUnqual(C&: Fn->getContext());
1521 // Store a nullptr in __llvm_profd, if we shouldn't use a real address
1522 if (!shouldRecordFunctionAddr(F: Fn))
1523 return ConstantPointerNull::get(T: Int8PtrTy);
1524
1525 // If we can't use an alias, we must use the public symbol, even though this
1526 // may require a symbolic relocation.
1527 if (shouldUsePublicSymbol(Fn))
1528 return Fn;
1529
1530 // For GPU targets, weak functions cannot use private aliases because
1531 // LTO may pick a different TU's copy, leaving the alias undefined
1532 if (isGPUProfTarget(M: *Fn->getParent()) &&
1533 GlobalValue::isWeakForLinker(Linkage: Fn->getLinkage()))
1534 return Fn;
1535
1536 // When possible use a private alias to avoid symbolic relocations.
1537 auto *GA = GlobalAlias::create(Linkage: GlobalValue::LinkageTypes::PrivateLinkage,
1538 Name: Fn->getName() + ".local", Aliasee: Fn);
1539
1540 // When the instrumented function is a COMDAT function, we cannot use a
1541 // private alias. If we did, we would create reference to a local label in
1542 // this function's section. If this version of the function isn't selected by
1543 // the linker, then the metadata would introduce a reference to a discarded
1544 // section. So, for COMDAT functions, we need to adjust the linkage of the
1545 // alias. Using hidden visibility avoids a dynamic relocation and an entry in
1546 // the dynamic symbol table.
1547 //
1548 // Note that this handles COMDAT functions with visibility other than Hidden,
1549 // since that case is covered in shouldUsePublicSymbol()
1550 if (Fn->hasComdat()) {
1551 GA->setLinkage(Fn->getLinkage());
1552 GA->setVisibility(GlobalValue::VisibilityTypes::HiddenVisibility);
1553 }
1554
1555 // appendToCompilerUsed(*Fn->getParent(), {GA});
1556
1557 return GA;
1558}
1559
1560static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
1561 // NVPTX is an ELF target but PTX does not expose sections or linker symbols.
1562 if (TT.isNVPTX())
1563 return true;
1564
1565 // compiler-rt uses linker support to get data/counters/name start/end for
1566 // ELF, COFF, Mach-O, XCOFF, and Wasm.
1567 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
1568 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF() ||
1569 TT.isOSBinFormatWasm())
1570 return false;
1571
1572 return true;
1573}
1574
1575void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO,
1576 StringRef CounterGroupName) {
1577 // Place lowered global variables in a comdat group if the associated function
1578 // or global variable is a COMDAT. This will make sure that only one copy of
1579 // global variable (e.g. function counters) of the COMDAT function will be
1580 // emitted after linking.
1581 bool NeedComdat = needsComdatForCounter(GV: *GO, M);
1582 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
1583
1584 if (!UseComdat)
1585 return;
1586
1587 // Keep in mind that this pass may run before the inliner, so we need to
1588 // create a new comdat group (for counters, profiling data, etc). If we use
1589 // the comdat of the parent function, that will result in relocations against
1590 // discarded sections.
1591 //
1592 // If the data variable is referenced by code, non-counter variables (notably
1593 // profiling data) and counters have to be in different comdats for COFF
1594 // because the Visual C++ linker will report duplicate symbol errors if there
1595 // are multiple external symbols with the same name marked
1596 // IMAGE_COMDAT_SELECT_ASSOCIATIVE.
1597 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
1598 ? GV->getName()
1599 : CounterGroupName;
1600 Comdat *C = M.getOrInsertComdat(Name: GroupName);
1601
1602 if (!NeedComdat) {
1603 // Object file format must be ELF since `UseComdat && !NeedComdat` is true.
1604 //
1605 // For ELF, when not using COMDAT, put counters, data and values into a
1606 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
1607 // allows -z start-stop-gc to discard the entire group when the function is
1608 // discarded.
1609 C->setSelectionKind(Comdat::NoDeduplicate);
1610 }
1611 GV->setComdat(C);
1612 // COFF doesn't allow the comdat group leader to have private linkage, so
1613 // upgrade private linkage to internal linkage to produce a symbol table
1614 // entry.
1615 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
1616 GV->setLinkage(GlobalValue::InternalLinkage);
1617}
1618
1619static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {
1620 if (!profDataReferencedByCode(M: *GV->getParent()))
1621 return false;
1622
1623 if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&
1624 !GV->hasAvailableExternallyLinkage())
1625 return true;
1626
1627 // This avoids the profile data from referencing internal symbols in
1628 // COMDAT.
1629 if (GV->hasLocalLinkage() && GV->hasComdat())
1630 return false;
1631
1632 return true;
1633}
1634
1635// FIXME: Introduce an internal alias like what's done for functions to reduce
1636// the number of relocation entries.
1637static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
1638 // Store a nullptr in __profvt_ if a real address shouldn't be used.
1639 if (!shouldRecordVTableAddr(GV))
1640 return ConstantPointerNull::get(T: PointerType::getUnqual(C&: GV->getContext()));
1641
1642 return GV;
1643}
1644
1645void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) {
1646 assert(Opts.profile_correlate != ProfCorrelatorKind::DebugInfo &&
1647 "Value profiling is not supported with lightweight instrumentation");
1648 if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
1649 return;
1650
1651 // Skip llvm internal global variable or __prof variables.
1652 if (GV->getName().starts_with(Prefix: "llvm.") ||
1653 GV->getName().starts_with(Prefix: "__llvm") ||
1654 GV->getName().starts_with(Prefix: "__prof"))
1655 return;
1656
1657 // VTableProfData already created
1658 auto It = VTableDataMap.find(Val: GV);
1659 if (It != VTableDataMap.end() && It->second)
1660 return;
1661
1662 GlobalValue::LinkageTypes Linkage = GV->getLinkage();
1663 GlobalValue::VisibilityTypes Visibility = GV->getVisibility();
1664
1665 // This is to keep consistent with per-function profile data
1666 // for correctness.
1667 if (TT.isOSBinFormatXCOFF()) {
1668 Linkage = GlobalValue::InternalLinkage;
1669 Visibility = GlobalValue::DefaultVisibility;
1670 }
1671
1672 LLVMContext &Ctx = M.getContext();
1673 Type *DataTypes[] = {
1674#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
1675#include "llvm/ProfileData/InstrProfData.inc"
1676#undef INSTR_PROF_VTABLE_DATA
1677 };
1678
1679 auto *DataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes));
1680
1681 // Used by INSTR_PROF_VTABLE_DATA MACRO
1682 Constant *VTableAddr = getVTableAddrForProfData(GV);
1683 const std::string PGOVTableName = getIRPGOObjectName(GO: *GV);
1684 // Record the length of the vtable. This is needed since vtable pointers
1685 // loaded from C++ objects might be from the middle of a vtable definition.
1686 uint32_t VTableSizeVal = GV->getGlobalSize(DL: M.getDataLayout());
1687
1688 Constant *DataVals[] = {
1689#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
1690#include "llvm/ProfileData/InstrProfData.inc"
1691#undef INSTR_PROF_VTABLE_DATA
1692 };
1693
1694 auto *Data =
1695 new GlobalVariable(M, DataTy, /*constant=*/false, Linkage,
1696 ConstantStruct::get(T: DataTy, V: DataVals),
1697 getInstrProfVTableVarPrefix() + PGOVTableName);
1698
1699 Data->setVisibility(Visibility);
1700 Data->setSection(getInstrProfSectionName(IPSK: IPSK_vtab, OF: TT.getObjectFormat()));
1701 Data->setAlignment(Align(8));
1702
1703 maybeSetComdat(GV: Data, GO: GV, CounterGroupName: Data->getName());
1704
1705 VTableDataMap[GV] = Data;
1706
1707 ReferencedVTables.push_back(x: GV);
1708
1709 // VTable <Hash, Addr> is used by runtime but not referenced by other
1710 // sections. Conservatively mark it linker retained.
1711 UsedVars.push_back(x: Data);
1712}
1713
1714GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc,
1715 InstrProfSectKind IPSK) {
1716 GlobalVariable *NamePtr = Inc->getName();
1717
1718 // Match the linkage and visibility of the name global.
1719 Function *Fn = Inc->getParent()->getParent();
1720 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
1721 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1722
1723 // Use internal rather than private linkage so the counter variable shows up
1724 // in the symbol table when using debug info for correlation.
1725 if (Opts.profile_correlate == ProfCorrelatorKind::DebugInfo &&
1726 TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage)
1727 Linkage = GlobalValue::InternalLinkage;
1728
1729 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1730 // symbols in the same csect won't be discarded. When there are duplicate weak
1731 // symbols, we can NOT guarantee that the relocations get resolved to the
1732 // intended weak symbol, so we can not ensure the correctness of the relative
1733 // CounterPtr, so we have to use private linkage for counter and data symbols.
1734 if (TT.isOSBinFormatXCOFF()) {
1735 Linkage = GlobalValue::PrivateLinkage;
1736 Visibility = GlobalValue::DefaultVisibility;
1737 }
1738 // Move the name variable to the right section.
1739 bool Renamed;
1740 GlobalVariable *Ptr;
1741 StringRef VarPrefix;
1742 std::string VarName;
1743 if (IPSK == IPSK_cnts) {
1744 VarPrefix = getInstrProfCountersVarPrefix();
1745 VarName = getVarName(Opts, Inc, Prefix: VarPrefix, Renamed);
1746 InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Val: Inc);
1747 Ptr = createRegionCounters(Inc: CntrIncrement, Name: VarName, Linkage);
1748 } else if (IPSK == IPSK_bitmap) {
1749 VarPrefix = getInstrProfBitmapVarPrefix();
1750 VarName = getVarName(Opts, Inc, Prefix: VarPrefix, Renamed);
1751 InstrProfMCDCBitmapInstBase *BitmapUpdate =
1752 dyn_cast<InstrProfMCDCBitmapInstBase>(Val: Inc);
1753 Ptr = createRegionBitmaps(Inc: BitmapUpdate, Name: VarName, Linkage);
1754 } else {
1755 llvm_unreachable("Profile Section must be for Counters or Bitmaps");
1756 }
1757
1758 Ptr->setVisibility(Visibility);
1759 Ptr->setSection(getInstrProfSectionName(IPSK, OF: TT.getObjectFormat()));
1760 Ptr->setLinkage(Linkage);
1761 if (isGPUProfTarget(M) && !Ptr->hasComdat()) {
1762 Ptr->setComdat(M.getOrInsertComdat(Name: VarName));
1763 Ptr->setLinkage(GlobalValue::LinkOnceODRLinkage);
1764 Ptr->setVisibility(GlobalValue::ProtectedVisibility);
1765 } else {
1766 maybeSetComdat(GV: Ptr, GO: Fn, CounterGroupName: VarName);
1767 }
1768 return Ptr;
1769}
1770
1771GlobalVariable *
1772InstrLowerer::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
1773 StringRef Name,
1774 GlobalValue::LinkageTypes Linkage) {
1775 uint64_t NumBytes = Inc->getNumBitmapBytes();
1776 auto *BitmapTy = ArrayType::get(ElementType: Type::getInt8Ty(C&: M.getContext()), NumElements: NumBytes);
1777 auto GV = new GlobalVariable(M, BitmapTy, false, Linkage,
1778 Constant::getNullValue(Ty: BitmapTy), Name);
1779 GV->setAlignment(Align(1));
1780 return GV;
1781}
1782
1783GlobalVariable *
1784InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
1785 GlobalVariable *NamePtr = Inc->getName();
1786 auto &PD = ProfileDataMap[NamePtr];
1787 if (PD.RegionBitmaps)
1788 return PD.RegionBitmaps;
1789
1790 // If RegionBitmaps doesn't already exist, create it by first setting up
1791 // the corresponding profile section.
1792 auto *BitmapPtr = setupProfileSection(Inc, IPSK: IPSK_bitmap);
1793 PD.RegionBitmaps = BitmapPtr;
1794 PD.NumBitmapBytes = Inc->getNumBitmapBytes();
1795
1796 if (PD.NumBitmapBytes &&
1797 Opts.profile_correlate == ProfCorrelatorKind::DebugInfo) {
1798 LLVMContext &Ctx = M.getContext();
1799 Function *Fn = Inc->getParent()->getParent();
1800 if (auto *SP = Fn->getSubprogram()) {
1801 DIBuilder DB(M, true, SP->getUnit());
1802 Metadata *FunctionNameAnnotation[] = {
1803 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::FunctionNameAttributeName),
1804 MDString::get(Context&: Ctx, Str: getPGOFuncNameVarInitializer(NameVar: NamePtr)),
1805 };
1806 Metadata *NumBitmapBitsAnnotation[] = {
1807 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::NumBitmapBitsAttributeName),
1808 ConstantAsMetadata::get(C: Inc->getNumBitmapBits()),
1809 };
1810 auto Annotations = DB.getOrCreateArray(Elements: {
1811 MDNode::get(Context&: Ctx, MDs: FunctionNameAnnotation),
1812 MDNode::get(Context&: Ctx, MDs: NumBitmapBitsAnnotation),
1813 });
1814 auto *DICounter = DB.createGlobalVariableExpression(
1815 Context: SP, Name: BitmapPtr->getName(), /*LinkageName=*/StringRef(), File: SP->getFile(),
1816 /*LineNo=*/0, Ty: DB.createUnspecifiedType(Name: "Profile Bitmap Type"),
1817 IsLocalToUnit: BitmapPtr->hasLocalLinkage(), /*IsDefined=*/isDefined: true, /*Expr=*/nullptr,
1818 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1819 Annotations);
1820 BitmapPtr->addDebugInfo(GV: DICounter);
1821 DB.finalizeSubprogram(SP);
1822 DB.finalize();
1823 }
1824
1825 // Mark the bitmap variable as used so that it isn't optimized out.
1826 CompilerUsedVars.push_back(x: PD.RegionBitmaps);
1827 }
1828
1829 return PD.RegionBitmaps;
1830}
1831
1832GlobalVariable *
1833InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
1834 GlobalValue::LinkageTypes Linkage) {
1835 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1836 auto &Ctx = M.getContext();
1837 GlobalVariable *GV;
1838 if (isa<InstrProfCoverInst>(Val: Inc)) {
1839 auto *CounterTy = Type::getInt8Ty(C&: Ctx);
1840 auto *CounterArrTy = ArrayType::get(ElementType: CounterTy, NumElements: NumCounters);
1841 // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
1842 std::vector<Constant *> InitialValues(NumCounters,
1843 Constant::getAllOnesValue(Ty: CounterTy));
1844 GV = new GlobalVariable(M, CounterArrTy, false, Linkage,
1845 ConstantArray::get(T: CounterArrTy, V: InitialValues),
1846 Name);
1847 GV->setAlignment(Align(1));
1848 } else {
1849 auto *CounterTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NumCounters);
1850 GV = new GlobalVariable(M, CounterTy, false, Linkage,
1851 Constant::getNullValue(Ty: CounterTy), Name);
1852 GV->setAlignment(Align(8));
1853 }
1854 return GV;
1855}
1856
1857GlobalVariable *
1858InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
1859 GlobalVariable *NamePtr = Inc->getName();
1860 auto &PD = ProfileDataMap[NamePtr];
1861 if (PD.RegionCounters)
1862 return PD.RegionCounters;
1863
1864 // If RegionCounters doesn't already exist, create it by first setting up
1865 // the corresponding profile section.
1866 auto *CounterPtr = setupProfileSection(Inc, IPSK: IPSK_cnts);
1867 PD.RegionCounters = CounterPtr;
1868
1869 if (Opts.profile_correlate == ProfCorrelatorKind::DebugInfo) {
1870 LLVMContext &Ctx = M.getContext();
1871 Function *Fn = Inc->getParent()->getParent();
1872 if (auto *SP = Fn->getSubprogram()) {
1873 DIBuilder DB(M, true, SP->getUnit());
1874 Metadata *FunctionNameAnnotation[] = {
1875 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::FunctionNameAttributeName),
1876 MDString::get(Context&: Ctx, Str: getPGOFuncNameVarInitializer(NameVar: NamePtr)),
1877 };
1878 Metadata *CFGHashAnnotation[] = {
1879 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::CFGHashAttributeName),
1880 ConstantAsMetadata::get(C: Inc->getHash()),
1881 };
1882 Metadata *NumCountersAnnotation[] = {
1883 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::NumCountersAttributeName),
1884 ConstantAsMetadata::get(C: Inc->getNumCounters()),
1885 };
1886 auto Annotations = DB.getOrCreateArray(Elements: {
1887 MDNode::get(Context&: Ctx, MDs: FunctionNameAnnotation),
1888 MDNode::get(Context&: Ctx, MDs: CFGHashAnnotation),
1889 MDNode::get(Context&: Ctx, MDs: NumCountersAnnotation),
1890 });
1891 auto *DICounter = DB.createGlobalVariableExpression(
1892 Context: SP, Name: CounterPtr->getName(), /*LinkageName=*/StringRef(), File: SP->getFile(),
1893 /*LineNo=*/0, Ty: DB.createUnspecifiedType(Name: "Profile Data Type"),
1894 IsLocalToUnit: CounterPtr->hasLocalLinkage(), /*IsDefined=*/isDefined: true, /*Expr=*/nullptr,
1895 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1896 Annotations);
1897 CounterPtr->addDebugInfo(GV: DICounter);
1898 DB.finalizeSubprogram(SP);
1899 DB.finalize();
1900 }
1901
1902 // Mark the counter variable as used so that it isn't optimized out.
1903 CompilerUsedVars.push_back(x: PD.RegionCounters);
1904 }
1905
1906 // Create uniform counters before the data variable so that
1907 // UniformCounterPtr can reference them in createDataVariable().
1908 getOrCreateUniformCounters(Inc);
1909
1910 // Create the data variable (if it doesn't already exist).
1911 createDataVariable(Inc);
1912
1913 return PD.RegionCounters;
1914}
1915
1916GlobalVariable *
1917InstrLowerer::getOrCreateUniformCounters(InstrProfCntrInstBase *Inc) {
1918 // Uniform counters are only meaningful for GPU profile targets.
1919 if (!isGPUProfTarget(M))
1920 return nullptr;
1921
1922 GlobalVariable *NamePtr = Inc->getName();
1923 auto &PD = ProfileDataMap[NamePtr];
1924 if (PD.UniformCounters)
1925 return PD.UniformCounters;
1926
1927 assert(PD.RegionCounters && "region counters must be created first");
1928
1929 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1930
1931 LLVMContext &Ctx = M.getContext();
1932 ArrayType *CounterTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NumCounters);
1933
1934 bool Renamed;
1935 std::string VarName = getVarName(Opts, Inc, Prefix: "__llvm_prf_unifcnt_", Renamed);
1936
1937 auto *GV = new GlobalVariable(M, CounterTy, false, NamePtr->getLinkage(),
1938 Constant::getNullValue(Ty: CounterTy), VarName);
1939 GV->setAlignment(Align(8));
1940
1941 GV->setSection(getInstrProfSectionName(IPSK: IPSK_ucnts, OF: TT.getObjectFormat()));
1942
1943 GV->setComdat(M.getOrInsertComdat(Name: VarName));
1944 GV->setLinkage(GlobalValue::LinkOnceODRLinkage);
1945 GV->setVisibility(GlobalValue::ProtectedVisibility);
1946
1947 PD.UniformCounters = GV;
1948 CompilerUsedVars.push_back(x: GV);
1949
1950 return PD.UniformCounters;
1951}
1952
1953void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
1954 // When debug information is correlated to profile data, a data variable
1955 // is not needed.
1956 if (Opts.profile_correlate == ProfCorrelatorKind::DebugInfo)
1957 return;
1958
1959 GlobalVariable *NamePtr = Inc->getName();
1960 auto &PD = ProfileDataMap[NamePtr];
1961
1962 // Return if data variable was already created.
1963 if (PD.DataVar)
1964 return;
1965
1966 LLVMContext &Ctx = M.getContext();
1967
1968 Function *Fn = Inc->getParent()->getParent();
1969 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
1970 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1971
1972 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1973 // symbols in the same csect won't be discarded. When there are duplicate weak
1974 // symbols, we can NOT guarantee that the relocations get resolved to the
1975 // intended weak symbol, so we can not ensure the correctness of the relative
1976 // CounterPtr, so we have to use private linkage for counter and data symbols.
1977 if (TT.isOSBinFormatXCOFF()) {
1978 Linkage = GlobalValue::PrivateLinkage;
1979 Visibility = GlobalValue::DefaultVisibility;
1980 }
1981
1982 bool NeedComdat = needsComdatForCounter(GV: *Fn, M);
1983 bool Renamed;
1984
1985 // The Data Variable section is anchored to profile counters.
1986 std::string CntsVarName =
1987 getVarName(Opts, Inc, Prefix: getInstrProfCountersVarPrefix(), Renamed);
1988 std::string DataVarName =
1989 getVarName(Opts, Inc, Prefix: getInstrProfDataVarPrefix(), Renamed);
1990
1991 auto *Int8PtrTy = PointerType::getUnqual(C&: Ctx);
1992 // Allocate statically the array of pointers to value profile nodes for
1993 // the current function.
1994 Constant *ValuesPtrExpr = ConstantPointerNull::get(T: Int8PtrTy);
1995 uint64_t NS = 0;
1996 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1997 NS += PD.NumValueSites[Kind];
1998 if (NS > 0 && Opts.vp_static_alloc &&
1999 !needsRuntimeRegistrationOfSectionRange(TT)) {
2000 ArrayType *ValuesTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NS);
2001 auto *ValuesVar = new GlobalVariable(
2002 M, ValuesTy, false, Linkage, Constant::getNullValue(Ty: ValuesTy),
2003 getVarName(Opts, Inc, Prefix: getInstrProfValuesVarPrefix(), Renamed));
2004 ValuesVar->setVisibility(Visibility);
2005 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *ValuesVar);
2006 ValuesVar->setSection(
2007 getInstrProfSectionName(IPSK: IPSK_vals, OF: TT.getObjectFormat()));
2008 ValuesVar->setAlignment(Align(8));
2009 maybeSetComdat(GV: ValuesVar, GO: Fn, CounterGroupName: CntsVarName);
2010 ValuesPtrExpr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
2011 C: ValuesVar, Ty: PointerType::get(C&: Fn->getContext(), AddressSpace: 0));
2012 }
2013
2014 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
2015
2016 Constant *CounterPtr = PD.RegionCounters;
2017 Constant *UniformCounterPtr = PD.UniformCounters;
2018
2019 uint64_t NumBitmapBytes = PD.NumBitmapBytes;
2020
2021 // Create data variable.
2022 auto *IntPtrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
2023 auto *Int16Ty = Type::getInt16Ty(C&: Ctx);
2024 auto *Int16ArrayTy = ArrayType::get(ElementType: Int16Ty, NumElements: IPVK_Last + 1);
2025 auto *DataTy = getProfileDataTy();
2026
2027 Constant *FunctionAddr = getFuncAddrForProfData(Fn);
2028
2029 Constant *Int16ArrayVals[IPVK_Last + 1];
2030 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
2031 Int16ArrayVals[Kind] = ConstantInt::get(Ty: Int16Ty, V: PD.NumValueSites[Kind]);
2032
2033 uint16_t OffloadDeviceWaveSizeVal = 0;
2034
2035 if (isGPUProfTarget(M)) {
2036 // For GPU targets, weak functions need weak linkage for their profile data
2037 // aliases to allow linker deduplication across TUs
2038 if (GlobalValue::isWeakForLinker(Linkage: Fn->getLinkage()))
2039 Linkage = Fn->getLinkage();
2040 else
2041 Linkage = GlobalValue::ExternalLinkage;
2042 Visibility = GlobalValue::ProtectedVisibility;
2043 }
2044 // If the data variable is not referenced by code (if we don't emit
2045 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
2046 // data variable live under linker GC, the data variable can be private. This
2047 // optimization applies to ELF.
2048 //
2049 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
2050 // to be false.
2051 //
2052 // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
2053 // that other copies must have the same CFG and cannot have value profiling.
2054 // If no hash suffix, other profd copies may be referenced by code.
2055 if (!isGPUProfTarget(M) && NS == 0 &&
2056 !(DataReferencedByCode && NeedComdat && !Renamed) &&
2057 (TT.isOSBinFormatELF() ||
2058 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
2059 Linkage = GlobalValue::PrivateLinkage;
2060 Visibility = GlobalValue::DefaultVisibility;
2061 }
2062 // GPU-target ELF objects are always ET_DYN, so non-local symbols with
2063 // default visibility are preemptible. The CounterPtr label difference
2064 // emits a REL32 relocation that lld rejects against preemptible targets.
2065 if (TT.isGPU() && TT.isOSBinFormatELF() &&
2066 !GlobalValue::isLocalLinkage(Linkage))
2067 Visibility = GlobalValue::ProtectedVisibility;
2068 auto *Data =
2069 new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName);
2070
2071 Constant *RelativeCounterPtr;
2072 Constant *RelativeUniformCounterPtr = ConstantInt::get(Ty: IntPtrTy, V: 0);
2073 GlobalVariable *BitmapPtr = PD.RegionBitmaps;
2074 Constant *RelativeBitmapPtr = ConstantInt::get(Ty: IntPtrTy, V: 0);
2075 InstrProfSectKind DataSectionKind;
2076 // With binary profile correlation, profile data is not loaded into memory.
2077 // profile data must reference profile counter with an absolute relocation.
2078 if (Opts.profile_correlate == ProfCorrelatorKind::Binary) {
2079 DataSectionKind = IPSK_covdata;
2080 RelativeCounterPtr = ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy);
2081 if (BitmapPtr != nullptr)
2082 RelativeBitmapPtr = ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy);
2083 if (UniformCounterPtr != nullptr)
2084 RelativeUniformCounterPtr =
2085 ConstantExpr::getPtrToInt(C: UniformCounterPtr, Ty: IntPtrTy);
2086 } else if (TT.isNVPTX()) {
2087 // The NVPTX target cannot handle self-referencing constant expressions in
2088 // global initializers at all. Use absolute pointers and have the runtime
2089 // registration convert them to relative offsets.
2090 DataSectionKind = IPSK_data;
2091 RelativeCounterPtr = ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy);
2092 } else {
2093 // Reference the counter variable with a label difference (link-time
2094 // constant).
2095 DataSectionKind = IPSK_data;
2096 RelativeCounterPtr =
2097 ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy),
2098 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2099 if (BitmapPtr != nullptr)
2100 RelativeBitmapPtr =
2101 ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy),
2102 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2103 if (UniformCounterPtr != nullptr)
2104 RelativeUniformCounterPtr = ConstantExpr::getSub(
2105 C1: ConstantExpr::getPtrToInt(C: UniformCounterPtr, Ty: IntPtrTy),
2106 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2107 }
2108
2109 Constant *DataVals[] = {
2110#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
2111#include "llvm/ProfileData/InstrProfData.inc"
2112 };
2113 Data->setInitializer(ConstantStruct::get(T: DataTy, V: DataVals));
2114
2115 Data->setVisibility(Visibility);
2116 Data->setSection(
2117 getInstrProfSectionName(IPSK: DataSectionKind, OF: TT.getObjectFormat()));
2118 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
2119 if (isGPUProfTarget(M) && !Data->hasComdat()) {
2120 Data->setComdat(M.getOrInsertComdat(Name: CntsVarName));
2121 Data->setLinkage(GlobalValue::LinkOnceODRLinkage);
2122 } else {
2123 maybeSetComdat(GV: Data, GO: Fn, CounterGroupName: CntsVarName);
2124 }
2125
2126 PD.DataVar = Data;
2127
2128 // Mark the data variable as used so that it isn't stripped out.
2129 CompilerUsedVars.push_back(x: Data);
2130 // Now that the linkage set by the FE has been passed to the data and counter
2131 // variables, reset Name variable's linkage and visibility to private so that
2132 // it can be removed later by the compiler.
2133 NamePtr->setLinkage(GlobalValue::PrivateLinkage);
2134 // Collect the referenced names to be used by emitNameData.
2135 ReferencedNames.push_back(x: NamePtr);
2136}
2137
2138void InstrLowerer::emitVNodes() {
2139 if (!Opts.vp_static_alloc)
2140 return;
2141
2142 // For now only support this on platforms that do
2143 // not require runtime registration to discover
2144 // named section start/end.
2145 if (needsRuntimeRegistrationOfSectionRange(TT))
2146 return;
2147
2148 size_t TotalNS = 0;
2149 for (auto &PD : ProfileDataMap) {
2150 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
2151 TotalNS += PD.second.NumValueSites[Kind];
2152 }
2153
2154 if (!TotalNS)
2155 return;
2156
2157 uint64_t NumCounters = TotalNS * Opts.vp_counters_per_site;
2158// Heuristic for small programs with very few total value sites.
2159// The default value of vp-counters-per-site is chosen based on
2160// the observation that large apps usually have a low percentage
2161// of value sites that actually have any profile data, and thus
2162// the average number of counters per site is low. For small
2163// apps with very few sites, this may not be true. Bump up the
2164// number of counters in this case.
2165#define INSTR_PROF_MIN_VAL_COUNTS 10
2166 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
2167 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, b: (int)NumCounters * 2);
2168
2169 auto &Ctx = M.getContext();
2170 Type *VNodeTypes[] = {
2171#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
2172#include "llvm/ProfileData/InstrProfData.inc"
2173 };
2174 auto *VNodeTy = StructType::get(Context&: Ctx, Elements: ArrayRef(VNodeTypes));
2175
2176 ArrayType *VNodesTy = ArrayType::get(ElementType: VNodeTy, NumElements: NumCounters);
2177 auto *VNodesVar = new GlobalVariable(
2178 M, VNodesTy, false, GlobalValue::PrivateLinkage,
2179 Constant::getNullValue(Ty: VNodesTy), getInstrProfVNodesVarName());
2180 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *VNodesVar);
2181 VNodesVar->setSection(
2182 getInstrProfSectionName(IPSK: IPSK_vnodes, OF: TT.getObjectFormat()));
2183 VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(Ty: VNodesTy));
2184 // VNodesVar is used by runtime but not referenced via relocation by other
2185 // sections. Conservatively make it linker retained.
2186 UsedVars.push_back(x: VNodesVar);
2187}
2188
2189// Build the per-TU device-PGO sections struct: section start/stop bounds for
2190// names/counters/data/uniform-counters plus the raw version. Returns null if it
2191// already exists.
2192static GlobalVariable *emitGPUOffloadSectionsStruct(Module &M,
2193 StringRef CUIDPostfix) {
2194 std::string Name = ("__llvm_profile_sections" + CUIDPostfix).str();
2195 if (M.getNamedValue(Name))
2196 return nullptr;
2197
2198 LLVMContext &Ctx = M.getContext();
2199 unsigned AS = M.getDataLayout().getDefaultGlobalsAddressSpace();
2200 auto Extern = [&](StringRef Sym, Type *Ty, bool IsConst,
2201 GlobalValue::VisibilityTypes Vis) {
2202 GlobalVariable *GV = M.getNamedGlobal(Name: Sym);
2203 if (!GV) {
2204 GV = new GlobalVariable(M, Ty, IsConst, GlobalValue::ExternalLinkage,
2205 nullptr, Sym, nullptr,
2206 GlobalValue::NotThreadLocal, AS);
2207 GV->setVisibility(Vis);
2208 }
2209 return GV;
2210 };
2211 // Section bounds are hidden i8 markers; raw_version is an i64 constant.
2212 auto *I8 = Type::getInt8Ty(C&: Ctx);
2213 auto Hidden = GlobalValue::HiddenVisibility;
2214 Constant *Fields[] = {Extern("__start___llvm_prf_names", I8, false, Hidden),
2215 Extern("__stop___llvm_prf_names", I8, false, Hidden),
2216 Extern("__start___llvm_prf_cnts", I8, false, Hidden),
2217 Extern("__stop___llvm_prf_cnts", I8, false, Hidden),
2218 Extern("__start___llvm_prf_data", I8, false, Hidden),
2219 Extern("__stop___llvm_prf_data", I8, false, Hidden),
2220 Extern("__start___llvm_prf_ucnts", I8, false, Hidden),
2221 Extern("__stop___llvm_prf_ucnts", I8, false, Hidden),
2222 Extern("__llvm_profile_raw_version",
2223 Type::getInt64Ty(C&: Ctx), true,
2224 GlobalValue::DefaultVisibility)};
2225 auto *PtrTy = PointerType::get(C&: Ctx, AddressSpace: AS);
2226 auto *STy = StructType::get(
2227 Context&: Ctx, Elements: {PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy});
2228 auto *GV = new GlobalVariable(M, STy, /*isConstant=*/true,
2229 GlobalValue::ExternalLinkage,
2230 ConstantStruct::get(T: STy, V: Fields), Name, nullptr,
2231 GlobalValue::NotThreadLocal, AS);
2232 GV->setVisibility(GlobalValue::ProtectedVisibility);
2233 return GV;
2234}
2235
2236void InstrLowerer::emitNameData() {
2237 if (ReferencedNames.empty())
2238 return;
2239
2240 std::string CompressedNameStr;
2241 if (Error E = collectPGOFuncNameStrings(NameVars: ReferencedNames, Result&: CompressedNameStr,
2242 doCompression: DoInstrProfNameCompression)) {
2243 report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false);
2244 }
2245
2246 auto &Ctx = M.getContext();
2247 auto *NamesVal =
2248 ConstantDataArray::getString(Context&: Ctx, Initializer: StringRef(CompressedNameStr), AddNull: false);
2249 std::string NamesVarName = std::string(getInstrProfNamesVarName());
2250 GlobalValue::LinkageTypes NamesLinkage = GlobalValue::PrivateLinkage;
2251 GlobalValue::VisibilityTypes NamesVisibility = GlobalValue::DefaultVisibility;
2252 std::string GPUCUIDPostfix;
2253 if (isGPUProfTarget(M)) {
2254 if (auto *GV = M.getNamedGlobal(Name: getInstrProfNamesVarPostfixVarName())) {
2255 if (auto *Init =
2256 dyn_cast_or_null<ConstantDataArray>(Val: GV->getInitializer())) {
2257 if (Init->isCString()) {
2258 GPUCUIDPostfix = Init->getAsCString().str();
2259 NamesVarName += GPUCUIDPostfix;
2260 NamesLinkage = GlobalValue::ExternalLinkage;
2261 NamesVisibility = GlobalValue::ProtectedVisibility;
2262 removeFromUsedLists(
2263 M, ShouldRemove: [GV](Constant *C) { return C->stripPointerCasts() == GV; });
2264 GV->eraseFromParent();
2265 }
2266 }
2267 }
2268 }
2269 NamesVar = new GlobalVariable(M, NamesVal->getType(), true, NamesLinkage,
2270 NamesVal, NamesVarName);
2271 NamesVar->setVisibility(NamesVisibility);
2272
2273 NamesSize = CompressedNameStr.size();
2274 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *NamesVar);
2275 std::string NamesSectionName =
2276 Opts.profile_correlate == ProfCorrelatorKind::Binary
2277 ? getInstrProfSectionName(IPSK: IPSK_covname, OF: TT.getObjectFormat())
2278 : getInstrProfSectionName(IPSK: IPSK_name, OF: TT.getObjectFormat());
2279 NamesVar->setSection(NamesSectionName);
2280 // On COFF, it's important to reduce the alignment down to 1 to prevent the
2281 // linker from inserting padding before the start of the names section or
2282 // between names entries.
2283 NamesVar->setAlignment(Align(1));
2284 // NamesVar is used by runtime but not referenced via relocation by other
2285 // sections. Conservatively make it linker retained.
2286 UsedVars.push_back(x: NamesVar);
2287
2288 for (auto *NamePtr : ReferencedNames)
2289 NamePtr->eraseFromParent();
2290
2291 // Emit the device sections struct only when this TU produced profile data, so
2292 // its section start/stop references are backed by a real section.
2293 bool HasData = llvm::any_of(Range&: ProfileDataMap,
2294 P: [](const auto &KV) { return KV.second.DataVar; });
2295 if (!GPUCUIDPostfix.empty() && HasData)
2296 if (GlobalVariable *GV = emitGPUOffloadSectionsStruct(M, CUIDPostfix: GPUCUIDPostfix))
2297 CompilerUsedVars.push_back(x: GV);
2298}
2299
2300void InstrLowerer::emitVTableNames() {
2301 if (!EnableVTableValueProfiling || ReferencedVTables.empty())
2302 return;
2303
2304 // Collect the PGO names of referenced vtables and compress them.
2305 std::string CompressedVTableNames;
2306 if (Error E = collectVTableStrings(VTables: ReferencedVTables, Result&: CompressedVTableNames,
2307 doCompression: DoInstrProfNameCompression)) {
2308 report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false);
2309 }
2310
2311 auto &Ctx = M.getContext();
2312 auto *VTableNamesVal = ConstantDataArray::getString(
2313 Context&: Ctx, Initializer: StringRef(CompressedVTableNames), AddNull: false /* AddNull */);
2314 GlobalVariable *VTableNamesVar =
2315 new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */,
2316 GlobalValue::PrivateLinkage, VTableNamesVal,
2317 getInstrProfVTableNamesVarName());
2318 VTableNamesVar->setSection(
2319 getInstrProfSectionName(IPSK: IPSK_vname, OF: TT.getObjectFormat()));
2320 VTableNamesVar->setAlignment(Align(1));
2321 // Make VTableNames linker retained.
2322 UsedVars.push_back(x: VTableNamesVar);
2323}
2324
2325void InstrLowerer::emitRegistration() {
2326 if (!needsRuntimeRegistrationOfSectionRange(TT))
2327 return;
2328
2329 // Construct the function.
2330 auto *VoidTy = Type::getVoidTy(C&: M.getContext());
2331 auto *VoidPtrTy = PointerType::getUnqual(C&: M.getContext());
2332 auto *Int64Ty = Type::getInt64Ty(C&: M.getContext());
2333 auto *RegisterFTy = FunctionType::get(Result: VoidTy, isVarArg: false);
2334 auto *RegisterF = Function::Create(Ty: RegisterFTy, Linkage: GlobalValue::InternalLinkage,
2335 N: getInstrProfRegFuncsName(), M);
2336 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2337 if (Options.NoRedZone)
2338 RegisterF->addFnAttr(Kind: Attribute::NoRedZone);
2339
2340 auto *RuntimeRegisterTy = FunctionType::get(Result: VoidTy, Params: VoidPtrTy, isVarArg: false);
2341 auto *RuntimeRegisterF =
2342 Function::Create(Ty: RuntimeRegisterTy, Linkage: GlobalVariable::ExternalLinkage,
2343 N: getInstrProfRegFuncName(), M);
2344
2345 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: RegisterF));
2346 for (Value *Data : CompilerUsedVars)
2347 if (!isa<Function>(Val: Data))
2348 // Check for addrspace cast when profiling GPU
2349 IRB.CreateCall(Callee: RuntimeRegisterF,
2350 Args: IRB.CreatePointerBitCastOrAddrSpaceCast(V: Data, DestTy: VoidPtrTy));
2351 for (Value *Data : UsedVars)
2352 if (Data != NamesVar && !isa<Function>(Val: Data))
2353 IRB.CreateCall(Callee: RuntimeRegisterF,
2354 Args: IRB.CreatePointerBitCastOrAddrSpaceCast(V: Data, DestTy: VoidPtrTy));
2355
2356 if (NamesVar) {
2357 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
2358 auto *NamesRegisterTy =
2359 FunctionType::get(Result: VoidTy, Params: ArrayRef(ParamTypes), isVarArg: false);
2360 auto *NamesRegisterF =
2361 Function::Create(Ty: NamesRegisterTy, Linkage: GlobalVariable::ExternalLinkage,
2362 N: getInstrProfNamesRegFuncName(), M);
2363 IRB.CreateCall(Callee: NamesRegisterF, Args: {IRB.CreatePointerBitCastOrAddrSpaceCast(
2364 V: NamesVar, DestTy: VoidPtrTy),
2365 IRB.getInt64(C: NamesSize)});
2366 }
2367
2368 IRB.CreateRetVoid();
2369}
2370
2371bool InstrLowerer::emitRuntimeHook() {
2372 // GPU profiling data is read directly by the host offload runtime. We do not
2373 // need the standard runtime hook.
2374 if (TT.isGPU())
2375 return false;
2376
2377 // We expect the linker to be invoked with -u<hook_var> flag for Linux
2378 // in which case there is no need to emit the external variable.
2379 if (TT.isOSLinux() || TT.isOSAIX())
2380 return false;
2381
2382 // If the module's provided its own runtime, we don't need to do anything.
2383 if (M.getGlobalVariable(Name: getInstrProfRuntimeHookVarName()))
2384 return false;
2385
2386 // Declare an external variable that will pull in the runtime initialization.
2387 auto *Int32Ty = Type::getInt32Ty(C&: M.getContext());
2388 auto *Var =
2389 new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
2390 nullptr, getInstrProfRuntimeHookVarName());
2391 Var->setVisibility(GlobalValue::HiddenVisibility);
2392
2393 if (TT.isOSBinFormatELF() && !TT.isPS()) {
2394 // Mark the user variable as used so that it isn't stripped out.
2395 CompilerUsedVars.push_back(x: Var);
2396 } else {
2397 // Make a function that uses it.
2398 auto *User = Function::Create(Ty: FunctionType::get(Result: Int32Ty, isVarArg: false),
2399 Linkage: GlobalValue::LinkOnceODRLinkage,
2400 N: getInstrProfRuntimeHookVarUseFuncName(), M);
2401 User->addFnAttr(Kind: Attribute::NoInline);
2402 if (Options.NoRedZone)
2403 User->addFnAttr(Kind: Attribute::NoRedZone);
2404 User->setVisibility(GlobalValue::HiddenVisibility);
2405 if (TT.supportsCOMDAT())
2406 User->setComdat(M.getOrInsertComdat(Name: User->getName()));
2407 // Explicitly mark this function as cold since it is never called.
2408 User->setEntryCount(Count: 0);
2409
2410 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: User));
2411 auto *Load = IRB.CreateLoad(Ty: Int32Ty, Ptr: Var);
2412 IRB.CreateRet(V: Load);
2413
2414 // Mark the function as used so that it isn't stripped out.
2415 CompilerUsedVars.push_back(x: User);
2416 }
2417 return true;
2418}
2419
2420void InstrLowerer::emitUses() {
2421 // The metadata sections are parallel arrays. Optimizers (e.g.
2422 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
2423 // we conservatively retain all unconditionally in the compiler.
2424 //
2425 // On ELF and Mach-O, the linker can guarantee the associated sections will be
2426 // retained or discarded as a unit, so llvm.compiler.used is sufficient.
2427 // Similarly on COFF, if prof data is not referenced by code we use one comdat
2428 // and ensure this GC property as well. Otherwise, we have to conservatively
2429 // make all of the sections retained by the linker.
2430 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
2431 (TT.isOSBinFormatCOFF() && !DataReferencedByCode))
2432 appendToCompilerUsed(M, Values: CompilerUsedVars);
2433 else
2434 appendToUsed(M, Values: CompilerUsedVars);
2435
2436 // We do not add proper references from used metadata sections to NamesVar and
2437 // VNodesVar, so we have to be conservative and place them in llvm.used
2438 // regardless of the target,
2439 appendToUsed(M, Values: UsedVars);
2440}
2441
2442void InstrLowerer::emitInitialization() {
2443 // Create ProfileFileName variable. Don't don't this for the
2444 // context-sensitive instrumentation lowering: This lowering is after
2445 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
2446 // have already create the variable before LTO/ThinLTO linking.
2447 if (!IsCS)
2448 createProfileFileNameVar(M, InstrProfileOutput: Options.InstrProfileOutput);
2449 Function *RegisterF = M.getFunction(Name: getInstrProfRegFuncsName());
2450 if (!RegisterF)
2451 return;
2452
2453 // Create the initialization function.
2454 auto *VoidTy = Type::getVoidTy(C&: M.getContext());
2455 auto *F = Function::Create(Ty: FunctionType::get(Result: VoidTy, isVarArg: false),
2456 Linkage: GlobalValue::InternalLinkage,
2457 N: getInstrProfInitFuncName(), M);
2458 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2459 F->addFnAttr(Kind: Attribute::NoInline);
2460 if (Options.NoRedZone)
2461 F->addFnAttr(Kind: Attribute::NoRedZone);
2462
2463 // Add the basic block and the necessary calls.
2464 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: F));
2465 IRB.CreateCall(Callee: RegisterF, Args: {});
2466 IRB.CreateRetVoid();
2467
2468 appendToGlobalCtors(M, F, Priority: 0);
2469}
2470
2471namespace llvm {
2472// Create the variable for profile sampling.
2473void createProfileSamplingVar(Module &M) {
2474 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
2475 IntegerType *SamplingVarTy;
2476 Constant *ValueZero;
2477 const InstrumentationOptions &Opts = getInstrumentationOptions(M);
2478 if (getSampledInstrumentationConfig(Opts).UseShort) {
2479 SamplingVarTy = Type::getInt16Ty(C&: M.getContext());
2480 ValueZero = Constant::getIntegerValue(Ty: SamplingVarTy, V: APInt(16, 0));
2481 } else {
2482 SamplingVarTy = Type::getInt32Ty(C&: M.getContext());
2483 ValueZero = Constant::getIntegerValue(Ty: SamplingVarTy, V: APInt(32, 0));
2484 }
2485 auto SamplingVar = new GlobalVariable(
2486 M, SamplingVarTy, false, GlobalValue::WeakAnyLinkage, ValueZero, VarName);
2487 SamplingVar->setVisibility(GlobalValue::DefaultVisibility);
2488 SamplingVar->setThreadLocal(true);
2489 Triple TT(M.getTargetTriple());
2490 if (TT.supportsCOMDAT()) {
2491 SamplingVar->setLinkage(GlobalValue::ExternalLinkage);
2492 SamplingVar->setComdat(M.getOrInsertComdat(Name: VarName));
2493 }
2494 appendToCompilerUsed(M, Values: SamplingVar);
2495}
2496} // namespace llvm
2497
2498// For GPU targets: Allocate contiguous arrays for all profile data.
2499// This solves the linker reordering problem by using ONE symbol per section
2500// type, so there's nothing for the linker to reorder.
2501StructType *InstrLowerer::getProfileDataTy() {
2502 if (ProfileDataTy)
2503 return ProfileDataTy;
2504
2505 auto &Ctx = M.getContext();
2506 auto *IntPtrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
2507 auto *Int16Ty = Type::getInt16Ty(C&: Ctx);
2508 auto *Int16ArrayTy = ArrayType::get(ElementType: Int16Ty, NumElements: IPVK_Last + 1);
2509 Type *DataTypes[] = {
2510#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
2511#include "llvm/ProfileData/InstrProfData.inc"
2512 };
2513 ProfileDataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes));
2514 return ProfileDataTy;
2515}
2516