1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by an instrumentor.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
16#include "InstrumentationOptions.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/ADT/Twine.h"
22#include "llvm/Analysis/BlockFrequencyInfo.h"
23#include "llvm/Analysis/BranchProbabilityInfo.h"
24#include "llvm/Analysis/CFG.h"
25#include "llvm/Analysis/LoopInfo.h"
26#include "llvm/Analysis/TargetLibraryInfo.h"
27#include "llvm/IR/Attributes.h"
28#include "llvm/IR/BasicBlock.h"
29#include "llvm/IR/CFG.h"
30#include "llvm/IR/Constant.h"
31#include "llvm/IR/Constants.h"
32#include "llvm/IR/CycleInfo.h"
33#include "llvm/IR/DIBuilder.h"
34#include "llvm/IR/DerivedTypes.h"
35#include "llvm/IR/DiagnosticInfo.h"
36#include "llvm/IR/Function.h"
37#include "llvm/IR/GlobalAlias.h"
38#include "llvm/IR/GlobalValue.h"
39#include "llvm/IR/GlobalVariable.h"
40#include "llvm/IR/IRBuilder.h"
41#include "llvm/IR/InstIterator.h"
42#include "llvm/IR/Instruction.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/Intrinsics.h"
46#include "llvm/IR/MDBuilder.h"
47#include "llvm/IR/Module.h"
48#include "llvm/IR/RuntimeLibcalls.h"
49#include "llvm/IR/Type.h"
50#include "llvm/Pass.h"
51#include "llvm/ProfileData/InstrProf.h"
52#include "llvm/ProfileData/InstrProfCorrelator.h"
53#include "llvm/Support/Casting.h"
54#include "llvm/Support/CommandLine.h"
55#include "llvm/Support/Compiler.h"
56#include "llvm/Support/Error.h"
57#include "llvm/Support/ErrorHandling.h"
58#include "llvm/TargetParser/Triple.h"
59#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
60#include "llvm/Transforms/Utils/BasicBlockUtils.h"
61#include "llvm/Transforms/Utils/Instrumentation.h"
62#include "llvm/Transforms/Utils/ModuleUtils.h"
63#include "llvm/Transforms/Utils/SSAUpdater.h"
64#include <algorithm>
65#include <cassert>
66#include <cstdint>
67#include <string>
68
69using namespace llvm;
70
71#define DEBUG_TYPE "instrprof"
72
73namespace llvm {
74// Command line option to enable vtable value profiling. Defined in
75// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
76extern cl::opt<bool> EnableVTableValueProfiling;
77} // namespace llvm
78
79bool llvm::isProfileCorrelationEnabled(
80 const cl::LibraryOptionsContext &LibraryOptions) {
81 return LibraryOptions.get<InstrumentationOptions>()
82 .profile_correlate.has_value();
83}
84
85namespace {
86
87struct SampledInstrumentationConfig {
88 unsigned BurstDuration;
89 unsigned Period;
90 bool UseShort;
91 bool IsSimpleSampling;
92 bool IsFastSampling;
93};
94
95static SampledInstrumentationConfig
96getSampledInstrumentationConfig(const InstrumentationOptions &Opts) {
97 SampledInstrumentationConfig config;
98 config.BurstDuration = Opts.sampled_instr_burst_duration;
99 config.Period = Opts.sampled_instr_period;
100 if (config.BurstDuration > config.Period)
101 report_fatal_error(
102 reason: "SampledBurstDuration must be less than or equal to SampledPeriod");
103 if (config.Period == 0 || config.BurstDuration == 0)
104 report_fatal_error(
105 reason: "SampledPeriod and SampledBurstDuration must be greater than 0");
106 config.IsSimpleSampling = (config.BurstDuration == 1);
107 // If (BurstDuration == 1 && Period == 65536), generate the simple sampling
108 // style code.
109 config.IsFastSampling =
110 (!config.IsSimpleSampling && config.Period == USHRT_MAX + 1);
111 config.UseShort = (config.Period <= USHRT_MAX) || config.IsFastSampling;
112 return config;
113}
114
115using LoadStorePair = std::pair<Instruction *, Instruction *>;
116
117static void makeAtomic(Instruction *Load, Instruction *Store) {
118 auto *Addition = dyn_cast<BinaryOperator>(Val: Store->getOperand(i: 0));
119 assert(Addition && Addition->getOpcode() == Instruction::BinaryOps::Add);
120 auto *Addend = Addition->getOperand(i_nocapture: 1);
121
122 IRBuilder<> Builder(Load);
123 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Store->getOperand(i: 1), Val: Addend,
124 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
125 Store->eraseFromParent();
126 Addition->eraseFromParent();
127 Load->eraseFromParent();
128}
129
130static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {
131 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(Val: M.getModuleFlag(Key: Flag));
132 if (!MD)
133 return 0;
134
135 // If the flag is a ConstantAsMetadata, it should be an integer representable
136 // in 64-bits.
137 return cast<ConstantInt>(Val: MD->getValue())->getZExtValue();
138}
139
140static bool enablesValueProfiling(const Module &M) {
141 return isIRPGOFlagSet(M: &M) ||
142 getIntModuleFlagOrZero(M, Flag: "EnableValueProfiling") != 0;
143}
144
145// Conservatively returns true if value profiling is enabled.
146static bool profDataReferencedByCode(const Module &M) {
147 return enablesValueProfiling(M);
148}
149
150class InstrLowerer final {
151public:
152 InstrLowerer(const InstrumentationOptions &Opts, Module &M,
153 const InstrProfOptions &Options,
154 std::function<const TargetLibraryInfo &(Function &F)> GetTLI,
155 bool IsCS)
156 : Opts(Opts), M(M), Options(Options), TT(M.getTargetTriple()), IsCS(IsCS),
157 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {}
158
159 bool lower();
160
161private:
162 const InstrumentationOptions &Opts;
163 Module &M;
164 const InstrProfOptions Options;
165 const Triple TT;
166 // Is this lowering for the context-sensitive instrumentation.
167 const bool IsCS;
168
169 std::function<const TargetLibraryInfo &(Function &F)> GetTLI;
170
171 const bool DataReferencedByCode;
172
173 struct PerFunctionProfileData {
174 uint32_t NumValueSites[IPVK_Last + 1] = {};
175 GlobalVariable *RegionCounters = nullptr;
176 GlobalVariable *UniformCounters =
177 nullptr; // Per-block uniform-entry counters
178 GlobalVariable *DataVar = nullptr;
179 GlobalVariable *RegionBitmaps = nullptr;
180 uint32_t NumBitmapBytes = 0;
181
182 PerFunctionProfileData() = default;
183 };
184 DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;
185 // Key is virtual table variable, value is 'VTableProfData' in the form of
186 // GlobalVariable.
187 DenseMap<GlobalVariable *, GlobalVariable *> VTableDataMap;
188 /// If runtime relocation is enabled, this maps functions to the load
189 /// instruction that produces the profile relocation bias.
190 DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;
191 std::vector<GlobalValue *> CompilerUsedVars;
192 std::vector<GlobalValue *> UsedVars;
193 std::vector<GlobalVariable *> ReferencedNames;
194 // The list of virtual table variables of which the VTableProfData is
195 // collected.
196 std::vector<GlobalVariable *> ReferencedVTables;
197 GlobalVariable *NamesVar = nullptr;
198 size_t NamesSize = 0;
199
200 StructType *ProfileDataTy = nullptr;
201
202 // vector of counter load/store pairs to be register promoted.
203 std::vector<LoadStorePair> PromotionCandidates;
204
205 int64_t TotalCountersPromoted = 0;
206
207 // Per-function cache of invariant values for GPU PGO instrumentation.
208 // Computed once at the function entry and reused across all instrumentation
209 // points to avoid redundant IR and help the optimizer.
210 struct GPUPGOInvariants {
211 Value *Matched = nullptr;
212 bool WaveSizeStored = false;
213 };
214 DenseMap<Function *, GPUPGOInvariants> GPUInvariantsCache;
215
216 /// Emit invariant PGO values at the function entry block and cache them.
217 GPUPGOInvariants &getOrCreateGPUInvariants(Function *F);
218
219 /// Lower instrumentation intrinsics in the function. Returns true if there
220 /// any lowering.
221 bool lowerIntrinsics(Function *F);
222
223 /// Register-promote counter loads and stores in loops.
224 void promoteCounterLoadStores(Function *F);
225
226 /// Returns true if relocating counters at runtime is enabled.
227 bool isRuntimeCounterRelocationEnabled() const;
228
229 /// Returns true if profile counter update register promotion is enabled.
230 bool isCounterPromotionEnabled() const;
231
232 /// Returns true if profile counter updates should be atomic.
233 bool isAtomic() const;
234
235 /// Return true if profile sampling is enabled.
236 bool isSamplingEnabled() const;
237
238 /// Count the number of instrumented value sites for the function.
239 void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
240
241 /// Replace instrprof.value.profile with a call to runtime library.
242 void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
243
244 /// Replace instrprof.cover with a store instruction to the coverage byte.
245 void lowerCover(InstrProfCoverInst *Inc);
246
247 /// Replace instrprof.timestamp with a call to
248 /// INSTR_PROF_PROFILE_SET_TIMESTAMP.
249 void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction);
250
251 /// Replace instrprof.increment with an increment of the appropriate value.
252 void lowerIncrement(InstrProfIncrementInst *Inc);
253
254 /// Force emitting of name vars for unused functions.
255 void lowerCoverageData(GlobalVariable *CoverageNamesVar);
256
257 /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction
258 /// using the index represented by the a temp value into a bitmap.
259 void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins);
260
261 /// Get the Bias value for data to access mmap-ed area.
262 /// Create it if it hasn't been seen.
263 GlobalVariable *getOrCreateBiasVar(StringRef VarName);
264
265 /// Compute the address of the counter value that this profiling instruction
266 /// acts on.
267 Value *getCounterAddress(InstrProfCntrInstBase *I);
268
269 /// Lower the incremental instructions under profile sampling predicates.
270 void doSampling(Instruction *I);
271
272 /// Get the region counters for an increment, creating them if necessary.
273 ///
274 /// If the counter array doesn't yet exist, the profile data variables
275 /// referring to them will also be created.
276 GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc);
277
278 /// Get the uniform entry counters for GPU divergence tracking.
279 /// These counters track how often blocks are entered with all lanes active.
280 GlobalVariable *getOrCreateUniformCounters(InstrProfCntrInstBase *Inc);
281
282 /// Create the region counters.
283 GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc,
284 StringRef Name,
285 GlobalValue::LinkageTypes Linkage);
286
287 /// Compute the address of the test vector bitmap that this profiling
288 /// instruction acts on.
289 Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I);
290
291 /// Get the region bitmaps for an increment, creating them if necessary.
292 ///
293 /// If the bitmap array doesn't yet exist, the profile data variables
294 /// referring to them will also be created.
295 GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc);
296
297 /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with
298 /// an MC/DC Decision region. The number of bytes required is indicated by
299 /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called
300 /// as part of setupProfileSection() and is conceptually very similar to
301 /// what is done for profile data counters in createRegionCounters().
302 GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
303 StringRef Name,
304 GlobalValue::LinkageTypes Linkage);
305
306 /// Set Comdat property of GV, if required.
307 void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName);
308
309 /// Setup the sections into which counters and bitmaps are allocated.
310 GlobalVariable *setupProfileSection(InstrProfInstBase *Inc,
311 InstrProfSectKind IPSK);
312
313 /// Create INSTR_PROF_DATA variable for counters and bitmaps.
314 void createDataVariable(InstrProfCntrInstBase *Inc);
315
316 /// Get the counters for virtual table values, creating them if necessary.
317 void getOrCreateVTableProfData(GlobalVariable *GV);
318
319 /// Emit the section with compressed function names.
320 void emitNameData();
321
322 /// Emit the section with compressed vtable names.
323 void emitVTableNames();
324
325 /// Emit value nodes section for value profiling.
326 void emitVNodes();
327
328 /// Emit runtime registration functions for each profile data variable.
329 void emitRegistration();
330
331 /// Emit the necessary plumbing to pull in the runtime initialization.
332 /// Returns true if a change was made.
333 bool emitRuntimeHook();
334
335 /// Add uses of our data variables and runtime hook.
336 void emitUses();
337
338 /// Create a static initializer for our data, on platforms that need it,
339 /// and for any profile output file that was specified.
340 void emitInitialization();
341
342 /// Return the __llvm_profile_data struct type.
343 StructType *getProfileDataTy();
344};
345
346///
347/// A helper class to promote one counter RMW operation in the loop
348/// into register update.
349///
350/// RWM update for the counter will be sinked out of the loop after
351/// the transformation.
352///
353class PGOCounterPromoterHelper : public LoadAndStorePromoter {
354public:
355 PGOCounterPromoterHelper(
356 const InstrumentationOptions &Opts, Instruction *L, Instruction *S,
357 SSAUpdater &SSA, Value *Init, BasicBlock *PH,
358 ArrayRef<BasicBlock *> ExitBlocks, ArrayRef<Instruction *> InsertPts,
359 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
360 LoopInfo &LI, bool IsAtomic)
361 : LoadAndStorePromoter({L, S}, SSA), Opts(Opts), Store(S),
362 ExitBlocks(ExitBlocks), InsertPts(InsertPts),
363 LoopToCandidates(LoopToCands), LI(LI), IsAtomic(IsAtomic) {
364 assert(isa<LoadInst>(L));
365 assert(isa<StoreInst>(S));
366 SSA.AddAvailableValue(BB: PH, V: Init);
367 }
368
369 void doExtraRewritesBeforeFinalDeletion() override {
370 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
371 BasicBlock *ExitBlock = ExitBlocks[i];
372 Instruction *InsertPos = InsertPts[i];
373 // Get LiveIn value into the ExitBlock. If there are multiple
374 // predecessors, the value is defined by a PHI node in this
375 // block.
376 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(BB: ExitBlock);
377 Value *Addr = cast<StoreInst>(Val: Store)->getPointerOperand();
378 Type *Ty = LiveInValue->getType();
379 IRBuilder<> Builder(InsertPos);
380 if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Val: Addr)) {
381 // If isRuntimeCounterRelocationEnabled() is true then the address of
382 // the store instruction is computed with two instructions in
383 // InstrProfiling::getCounterAddress(). We need to copy those
384 // instructions to this block to compute Addr correctly.
385 // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
386 // %Addr = inttoptr i64 %BiasAdd to i64*
387 auto *OrigBiasInst = dyn_cast<BinaryOperator>(Val: AddrInst->getOperand(i_nocapture: 0));
388 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
389 Value *BiasInst = Builder.Insert(I: OrigBiasInst->clone());
390 Addr = Builder.CreateIntToPtr(V: BiasInst,
391 DestTy: PointerType::getUnqual(C&: Ty->getContext()));
392 }
393 auto *TargetLoop =
394 Opts.iterative_counter_promotion ? LI.getLoopFor(BB: ExitBlock) : nullptr;
395 // Generate the relaxed atomic RMW if we've asked for it and no more
396 // promotion is possible.
397 if ((IsAtomic && !TargetLoop) || Opts.atomic_counter_update_promoted)
398 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: LiveInValue,
399 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
400 else {
401 LoadInst *OldVal = Builder.CreateLoad(Ty, Ptr: Addr, Name: "pgocount.promoted");
402 auto *NewVal = Builder.CreateAdd(LHS: OldVal, RHS: LiveInValue);
403 auto *NewStore = Builder.CreateStore(Val: NewVal, Ptr: Addr);
404
405 // Now update the parent loop's candidate list:
406 if (TargetLoop)
407 LoopToCandidates[TargetLoop].emplace_back(Args&: OldVal, Args&: NewStore);
408 }
409 }
410 }
411
412private:
413 const InstrumentationOptions &Opts;
414 Instruction *Store;
415 ArrayRef<BasicBlock *> ExitBlocks;
416 ArrayRef<Instruction *> InsertPts;
417 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
418 LoopInfo &LI;
419 const bool IsAtomic;
420};
421
422/// A helper class to do register promotion for all profile counter
423/// updates in a loop.
424///
425class PGOCounterPromoter {
426public:
427 PGOCounterPromoter(
428 const InstrumentationOptions &Opts,
429 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
430 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI, bool IsAtomic)
431 : Opts(Opts), LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI),
432 IsAtomic(IsAtomic) {
433
434 // Skip collection of ExitBlocks and InsertPts for loops that will not be
435 // able to have counters promoted.
436 SmallVector<BasicBlock *, 8> LoopExitBlocks;
437 SmallPtrSet<BasicBlock *, 8> BlockSet;
438
439 L.getExitBlocks(ExitBlocks&: LoopExitBlocks);
440 if (!isPromotionPossible(LP: &L, LoopExitBlocks))
441 return;
442
443 for (BasicBlock *ExitBlock : LoopExitBlocks) {
444 if (BlockSet.insert(Ptr: ExitBlock).second &&
445 llvm::none_of(Range: predecessors(BB: ExitBlock), P: [&](const BasicBlock *Pred) {
446 return llvm::isPresplitCoroSuspendExitEdge(Src: *Pred, Dest: *ExitBlock);
447 })) {
448 ExitBlocks.push_back(Elt: ExitBlock);
449 InsertPts.push_back(Elt: &*ExitBlock->getFirstInsertionPt());
450 }
451 }
452 }
453
454 bool run(int64_t *NumPromoted) {
455 // Move L's candidates out of LoopToCandidates before promoting them, as
456 // promoting a counter to an enclosing loop may insert a new key into
457 // LoopToCandidates and trigger DenseMap::grow().
458 auto &OrigCandidates = LoopToCandidates[&L];
459 SmallVector<LoadStorePair, 8> Candidates = std::move(OrigCandidates);
460 OrigCandidates.clear();
461 bool RC = promoteCandidates(Candidates, NumPromoted);
462 assert(LoopToCandidates[&L].empty() &&
463 "Did not expect new candidates to be added to current loop");
464 // In certain case, e.g. with -fprofile-update=atomic, we want to generate
465 // atomic updates of the PGO counters, but also perform promotion of these
466 // updates out of loops to reduce train time. The strategy is:
467 // 1) generate non-atomic load-increment-store sequence of instructions
468 // during lowerIntrinsics phase,
469 // 2) perform the promotion (in promoteCandidates function), then
470 // 3) convert all (promoted and unpromotable) updates to atomicRMW.
471 // This requires that promoted candidates are set to nullptr in the
472 // Candidates array by the promoteCandidates() function.
473 if (IsAtomic)
474 for (auto &Cand : Candidates)
475 if (Cand.first != nullptr && Cand.second != nullptr)
476 makeAtomic(Load: Cand.first, Store: Cand.second);
477 return RC;
478 }
479
480private:
481 bool promoteCandidates(SmallVectorImpl<LoadStorePair> &Candidates,
482 int64_t *NumPromoted) {
483 // Skip 'infinite' loops:
484 if (ExitBlocks.size() == 0)
485 return false;
486
487 // Skip if any of the ExitBlocks contains a ret instruction.
488 // This is to prevent dumping of incomplete profile -- if the
489 // the loop is a long running loop and dump is called in the middle
490 // of the loop, the result profile is incomplete.
491 // FIXME: add other heuristics to detect long running loops.
492 if (Opts.skip_ret_exit_block) {
493 for (auto *BB : ExitBlocks)
494 if (isa<ReturnInst>(Val: BB->getTerminator()))
495 return false;
496 }
497
498 unsigned MaxProm = getMaxNumOfPromotionsInLoop(LP: &L);
499 if (MaxProm == 0)
500 return false;
501
502 unsigned Promoted = 0;
503 for (auto &Cand : Candidates) {
504 SmallVector<PHINode *, 4> NewPHIs;
505 SSAUpdater SSA(&NewPHIs);
506 Value *InitVal = ConstantInt::get(Ty: Cand.first->getType(), V: 0);
507
508 // If BFI is set, we will use it to guide the promotions.
509 if (BFI) {
510 auto *BB = Cand.first->getParent();
511 auto InstrCount = BFI->getBlockProfileCount(BB);
512 if (!InstrCount)
513 continue;
514 auto PreheaderCount = BFI->getBlockProfileCount(BB: L.getLoopPreheader());
515 // If the average loop trip count is not greater than 1.5, we skip
516 // promotion.
517 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
518 continue;
519 }
520
521 PGOCounterPromoterHelper Promoter(
522 Opts, Cand.first, Cand.second, SSA, InitVal, L.getLoopPreheader(),
523 ExitBlocks, InsertPts, LoopToCandidates, LI, IsAtomic);
524 Promoter.run(Insts: SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
525
526 Cand = {nullptr, nullptr};
527
528 Promoted++;
529 if (Promoted >= MaxProm)
530 break;
531
532 (*NumPromoted)++;
533 if (Opts.max_counter_promotions != -1 &&
534 *NumPromoted >= Opts.max_counter_promotions)
535 break;
536 }
537
538 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
539 << L.getLoopDepth() << ")\n");
540 return Promoted != 0;
541 }
542
543private:
544 bool allowSpeculativeCounterPromotion(Loop *LP) {
545 SmallVector<BasicBlock *, 8> ExitingBlocks;
546 L.getExitingBlocks(ExitingBlocks);
547 // Not considierered speculative.
548 if (ExitingBlocks.size() == 1)
549 return true;
550 if (ExitingBlocks.size() > Opts.speculative_counter_promotion_max_exiting)
551 return false;
552 return true;
553 }
554
555 // Check whether the loop satisfies the basic conditions needed to perform
556 // Counter Promotions.
557 bool
558 isPromotionPossible(Loop *LP,
559 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
560 // We can't insert into a catchswitch.
561 if (llvm::any_of(Range: LoopExitBlocks, P: [](BasicBlock *Exit) {
562 return isa<CatchSwitchInst>(Val: Exit->getTerminator());
563 }))
564 return false;
565
566 if (!LP->hasDedicatedExits())
567 return false;
568
569 BasicBlock *PH = LP->getLoopPreheader();
570 if (!PH)
571 return false;
572
573 return true;
574 }
575
576 // Returns the max number of Counter Promotions for LP.
577 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
578 SmallVector<BasicBlock *, 8> LoopExitBlocks;
579 LP->getExitBlocks(ExitBlocks&: LoopExitBlocks);
580 if (!isPromotionPossible(LP, LoopExitBlocks))
581 return 0;
582
583 SmallVector<BasicBlock *, 8> ExitingBlocks;
584 LP->getExitingBlocks(ExitingBlocks);
585
586 // If BFI is set, we do more aggressive promotions based on BFI.
587 if (BFI)
588 return (unsigned)-1;
589
590 // Not considierered speculative.
591 if (ExitingBlocks.size() == 1)
592 return Opts.max_counter_promotions_per_loop;
593
594 if (ExitingBlocks.size() > Opts.speculative_counter_promotion_max_exiting)
595 return 0;
596
597 // Whether the target block is in a loop does not matter:
598 if (Opts.speculative_counter_promotion_to_loop)
599 return Opts.max_counter_promotions_per_loop;
600
601 // Now check the target block:
602 unsigned MaxProm = Opts.max_counter_promotions_per_loop;
603 for (auto *TargetBlock : LoopExitBlocks) {
604 auto *TargetLoop = LI.getLoopFor(BB: TargetBlock);
605 if (!TargetLoop)
606 continue;
607 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(LP: TargetLoop);
608 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
609 MaxProm =
610 std::min(a: MaxProm, b: std::max(a: MaxPromForTarget, b: PendingCandsInTarget) -
611 PendingCandsInTarget);
612 }
613 return MaxProm;
614 }
615
616 const InstrumentationOptions &Opts;
617 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
618 SmallVector<BasicBlock *, 8> ExitBlocks;
619 SmallVector<Instruction *, 8> InsertPts;
620 Loop &L;
621 LoopInfo &LI;
622 BlockFrequencyInfo *BFI;
623 const bool IsAtomic; // Whether to convert counter updates to atomics.
624};
625
626enum class ValueProfilingCallType {
627 // Individual values are tracked. Currently used for indiret call target
628 // profiling.
629 Default,
630
631 // MemOp: the memop size value profiling.
632 MemOp
633};
634
635} // end anonymous namespace
636
637PreservedAnalyses InstrProfilingLoweringPass::run(Module &M,
638 ModuleAnalysisManager &AM) {
639 FunctionAnalysisManager &FAM =
640 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
641 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
642 return FAM.getResult<TargetLibraryAnalysis>(IR&: F);
643 };
644 const InstrumentationOptions &Opts =
645 M.getContext().getLibraryOptions().get<InstrumentationOptions>();
646 InstrLowerer Lowerer(Opts, M, Options, GetTLI, IsCS);
647 if (!Lowerer.lower())
648 return PreservedAnalyses::all();
649
650 return PreservedAnalyses::none();
651}
652
653//
654// Perform instrumentation sampling.
655//
656// There are 3 favors of sampling:
657// (1) Full burst sampling: We transform:
658// Increment_Instruction;
659// to:
660// if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
661// Increment_Instruction;
662// }
663// __llvm_profile_sampling__ += 1;
664// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
665// __llvm_profile_sampling__ = 0;
666// }
667//
668// "__llvm_profile_sampling__" is a thread-local global shared by all PGO
669// counters (value-instrumentation and edge instrumentation).
670//
671// (2) Fast burst sampling:
672// "__llvm_profile_sampling__" variable is an unsigned type, meaning it will
673// wrap around to zero when overflows. In this case, the second check is
674// unnecessary, so we won't generate check2 when the SampledInstrPeriod is
675// set to 65536 (64K). The code after:
676// if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {
677// Increment_Instruction;
678// }
679// __llvm_profile_sampling__ += 1;
680//
681// (3) Simple sampling:
682// When SampledInstrBurstDuration is set to 1, we do a simple sampling:
683// __llvm_profile_sampling__ += 1;
684// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
685// __llvm_profile_sampling__ = 0;
686// Increment_Instruction;
687// }
688//
689// Note that, the code snippet after the transformation can still be counter
690// promoted. However, with sampling enabled, counter updates are expected to
691// be infrequent, making the benefits of counter promotion negligible.
692// Moreover, counter promotion can potentially cause issues in server
693// applications, particularly when the counters are dumped without a clean
694// exit. To mitigate this risk, counter promotion is disabled by default when
695// sampling is enabled. This behavior can be overridden using the internal
696// option.
697void InstrLowerer::doSampling(Instruction *I) {
698 if (!isSamplingEnabled())
699 return;
700
701 SampledInstrumentationConfig config = getSampledInstrumentationConfig(Opts);
702 auto GetConstant = [&config](IRBuilder<> &Builder, uint32_t C) {
703 if (config.UseShort)
704 return Builder.getInt16(C);
705 else
706 return Builder.getInt32(C);
707 };
708
709 IntegerType *SamplingVarTy;
710 if (config.UseShort)
711 SamplingVarTy = Type::getInt16Ty(C&: M.getContext());
712 else
713 SamplingVarTy = Type::getInt32Ty(C&: M.getContext());
714 auto *SamplingVar =
715 M.getGlobalVariable(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
716 assert(SamplingVar && "SamplingVar not set properly");
717
718 // Create the condition for checking the burst duration.
719 Instruction *SamplingVarIncr;
720 Value *NewSamplingVarVal;
721 MDBuilder MDB(I->getContext());
722 MDNode *BranchWeight;
723 IRBuilder<> CondBuilder(I);
724 auto *LoadSamplingVar = CondBuilder.CreateLoad(Ty: SamplingVarTy, Ptr: SamplingVar);
725 if (config.IsSimpleSampling) {
726 // For the simple sampling, just create the load and increments.
727 IRBuilder<> IncBuilder(I);
728 NewSamplingVarVal =
729 IncBuilder.CreateAdd(LHS: LoadSamplingVar, RHS: GetConstant(IncBuilder, 1));
730 SamplingVarIncr = IncBuilder.CreateStore(Val: NewSamplingVarVal, Ptr: SamplingVar);
731 } else {
732 // For the burst-sampling, create the conditional update.
733 auto *DurationCond = CondBuilder.CreateICmpULE(
734 LHS: LoadSamplingVar, RHS: GetConstant(CondBuilder, config.BurstDuration - 1));
735 BranchWeight = MDB.createBranchWeights(
736 TrueWeight: config.BurstDuration, FalseWeight: config.Period - config.BurstDuration);
737 Instruction *ThenTerm = SplitBlockAndInsertIfThen(
738 Cond: DurationCond, SplitBefore: I, /* Unreachable */ false, BranchWeights: BranchWeight);
739 IRBuilder<> IncBuilder(I);
740 NewSamplingVarVal =
741 IncBuilder.CreateAdd(LHS: LoadSamplingVar, RHS: GetConstant(IncBuilder, 1));
742 SamplingVarIncr = IncBuilder.CreateStore(Val: NewSamplingVarVal, Ptr: SamplingVar);
743 I->moveBefore(InsertPos: ThenTerm->getIterator());
744 }
745
746 if (config.IsFastSampling)
747 return;
748
749 // Create the condition for checking the period.
750 Instruction *ThenTerm, *ElseTerm;
751 IRBuilder<> PeriodCondBuilder(SamplingVarIncr);
752 auto *PeriodCond = PeriodCondBuilder.CreateICmpUGE(
753 LHS: NewSamplingVarVal, RHS: GetConstant(PeriodCondBuilder, config.Period));
754 BranchWeight = MDB.createBranchWeights(TrueWeight: 1, FalseWeight: config.Period - 1);
755 SplitBlockAndInsertIfThenElse(Cond: PeriodCond, SplitBefore: SamplingVarIncr, ThenTerm: &ThenTerm,
756 ElseTerm: &ElseTerm, BranchWeights: BranchWeight);
757
758 // For the simple sampling, the counter update happens in sampling var reset.
759 if (config.IsSimpleSampling)
760 I->moveBefore(InsertPos: ThenTerm->getIterator());
761
762 IRBuilder<> ResetBuilder(ThenTerm);
763 ResetBuilder.CreateStore(Val: GetConstant(ResetBuilder, 0), Ptr: SamplingVar);
764 SamplingVarIncr->moveBefore(InsertPos: ElseTerm->getIterator());
765}
766
767bool InstrLowerer::lowerIntrinsics(Function *F) {
768 bool MadeChange = false;
769 PromotionCandidates.clear();
770 SmallVector<InstrProfInstBase *, 8> InstrProfInsts;
771
772 // To ensure compatibility with sampling, we save the intrinsics into
773 // a buffer to prevent potential breakage of the iterator (as the
774 // intrinsics will be moved to a different BB).
775 for (BasicBlock &BB : *F) {
776 for (Instruction &Instr : llvm::make_early_inc_range(Range&: BB)) {
777 if (auto *IP = dyn_cast<InstrProfInstBase>(Val: &Instr))
778 InstrProfInsts.push_back(Elt: IP);
779 }
780 }
781
782 for (auto *Instr : InstrProfInsts) {
783 doSampling(I: Instr);
784 if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(Val: Instr)) {
785 lowerIncrement(Inc: IPIS);
786 MadeChange = true;
787 } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(Val: Instr)) {
788 lowerIncrement(Inc: IPI);
789 MadeChange = true;
790 } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(Val: Instr)) {
791 lowerTimestamp(TimestampInstruction: IPC);
792 MadeChange = true;
793 } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(Val: Instr)) {
794 lowerCover(Inc: IPC);
795 MadeChange = true;
796 } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(Val: Instr)) {
797 lowerValueProfileInst(Ins: IPVP);
798 MadeChange = true;
799 } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(Val: Instr)) {
800 IPMP->eraseFromParent();
801 MadeChange = true;
802 } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(Val: Instr)) {
803 lowerMCDCTestVectorBitmapUpdate(Ins: IPBU);
804 MadeChange = true;
805 }
806 }
807
808 if (!MadeChange)
809 return false;
810
811 promoteCounterLoadStores(F);
812 return true;
813}
814
815bool InstrLowerer::isRuntimeCounterRelocationEnabled() const {
816 // Mach-O don't support weak external references.
817 if (TT.isOSBinFormatMachO())
818 return false;
819
820 // Fuchsia uses runtime counter relocation by default.
821 return valueOr(X: Opts.runtime_counter_relocation, Default: TT.isOSFuchsia());
822}
823
824bool InstrLowerer::isSamplingEnabled() const {
825 return valueOr(X: Opts.sampled_instrumentation, Default: Options.Sampling);
826}
827
828bool InstrLowerer::isCounterPromotionEnabled() const {
829 return valueOr(X: Opts.do_counter_promotion, Default: Options.DoCounterPromotion);
830}
831
832bool InstrLowerer::isAtomic() const {
833 return Options.Atomic || Opts.instrprof_atomic_counter_update_all;
834}
835
836static void doAtomicCheck(Function *F) {
837 for (const llvm::Instruction &I : llvm::instructions(F)) {
838 const Value *Addr = nullptr;
839 if (const LoadInst *LI = dyn_cast<LoadInst>(Val: &I))
840 Addr = LI->getOperand(i_nocapture: 0);
841 else if (const StoreInst *LI = dyn_cast<StoreInst>(Val: &I))
842 Addr = LI->getOperand(i_nocapture: 1);
843
844 if (Addr && Addr->stripInBoundsOffsets()->getName().starts_with(
845 Prefix: getInstrProfCountersVarPrefix())) {
846 LLVM_DEBUG(dbgs() << "Missed candidate: "; I.dump());
847 report_fatal_error(reason: "Candidate load/store not converted to atomic");
848 }
849 }
850}
851
852void InstrLowerer::promoteCounterLoadStores(Function *F) {
853 if (!isCounterPromotionEnabled())
854 return;
855
856 CycleInfo CI;
857 CI.compute(F&: *F);
858 LoopInfo LI;
859 LI.analyze(F);
860 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
861
862 std::unique_ptr<BlockFrequencyInfo> BFI;
863 if (Options.UseBFIInPromotion) {
864 std::unique_ptr<BranchProbabilityInfo> BPI;
865 BPI.reset(p: new BranchProbabilityInfo(*F, CI, &GetTLI(*F)));
866 BFI.reset(p: new BlockFrequencyInfo(*F, *BPI, CI));
867 }
868
869 for (const auto &LoadStore : PromotionCandidates) {
870 auto *CounterLoad = LoadStore.first;
871 auto *CounterStore = LoadStore.second;
872 BasicBlock *BB = CounterLoad->getParent();
873 Loop *ParentLoop = LI.getLoopFor(BB);
874 if (!ParentLoop) {
875 if (isAtomic())
876 makeAtomic(Load: CounterLoad, Store: CounterStore);
877 continue;
878 }
879 LoopPromotionCandidates[ParentLoop].emplace_back(Args&: CounterLoad, Args&: CounterStore);
880 }
881
882 SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
883
884 // Do a post-order traversal of the loops so that counter updates can be
885 // iteratively hoisted outside the loop nest.
886 for (auto *Loop : llvm::reverse(C&: Loops)) {
887 PGOCounterPromoter Promoter(Opts, LoopPromotionCandidates, *Loop, LI,
888 BFI.get(), isAtomic());
889 Promoter.run(NumPromoted: &TotalCountersPromoted);
890 }
891
892 if (isAtomic() && Opts.verify_atomic_counter_promoted)
893 doAtomicCheck(F);
894}
895
896static bool needsRuntimeHookUnconditionally(const Triple &TT) {
897 // On Fuchsia, we only need runtime hook if any counters are present.
898 if (TT.isOSFuchsia())
899 return false;
900
901 return true;
902}
903
904/// Check if the module contains uses of any profiling intrinsics.
905static bool containsProfilingIntrinsics(Module &M) {
906 auto containsIntrinsic = [&](int ID) {
907 if (auto *F = Intrinsic::getDeclarationIfExists(M: &M, id: ID))
908 return !F->use_empty();
909 return false;
910 };
911 return containsIntrinsic(Intrinsic::instrprof_cover) ||
912 containsIntrinsic(Intrinsic::instrprof_increment) ||
913 containsIntrinsic(Intrinsic::instrprof_increment_step) ||
914 containsIntrinsic(Intrinsic::instrprof_timestamp) ||
915 containsIntrinsic(Intrinsic::instrprof_value_profile);
916}
917
918bool InstrLowerer::lower() {
919 bool MadeChange = false;
920 bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
921 if (NeedsRuntimeHook)
922 MadeChange = emitRuntimeHook();
923
924 if (!IsCS && isSamplingEnabled())
925 createProfileSamplingVar(M);
926
927 bool ContainsProfiling = containsProfilingIntrinsics(M);
928 GlobalVariable *CoverageNamesVar =
929 M.getNamedGlobal(Name: getCoverageUnusedNamesVarName());
930 // Improve compile time by avoiding linear scans when there is no work.
931 if (!ContainsProfiling && !CoverageNamesVar)
932 return MadeChange;
933
934 // We did not know how many value sites there would be inside
935 // the instrumented function. This is counting the number of instrumented
936 // target value sites to enter it as field in the profile data variable.
937 for (Function &F : M) {
938 InstrProfCntrInstBase *FirstProfInst = nullptr;
939 for (BasicBlock &BB : F) {
940 for (auto I = BB.begin(), E = BB.end(); I != E; I++) {
941 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Val&: I))
942 computeNumValueSiteCounts(Ins: Ind);
943 else {
944 if (FirstProfInst == nullptr &&
945 (isa<InstrProfIncrementInst>(Val: I) || isa<InstrProfCoverInst>(Val: I)))
946 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(Val&: I);
947 // If the MCDCBitmapParameters intrinsic seen, create the bitmaps.
948 if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(Val&: I))
949 static_cast<void>(getOrCreateRegionBitmaps(Inc: Params));
950 }
951 }
952 }
953
954 // Use a profile intrinsic to create the region counters and data variable.
955 // Also create the data variable based on the MCDCParams.
956 if (FirstProfInst != nullptr) {
957 static_cast<void>(getOrCreateRegionCounters(Inc: FirstProfInst));
958 }
959 }
960
961 if (EnableVTableValueProfiling)
962 for (GlobalVariable &GV : M.globals())
963 // Global variables with type metadata are virtual table variables.
964 if (GV.hasMetadata(KindID: LLVMContext::MD_type))
965 getOrCreateVTableProfData(GV: &GV);
966
967 for (Function &F : M)
968 MadeChange |= lowerIntrinsics(F: &F);
969
970 if (CoverageNamesVar) {
971 lowerCoverageData(CoverageNamesVar);
972 MadeChange = true;
973 }
974
975 if (!MadeChange)
976 return false;
977
978 emitVNodes();
979 emitNameData();
980 emitVTableNames();
981
982 // Emit runtime hook for the cases where the target does not unconditionally
983 // require pulling in profile runtime, and coverage is enabled on code that is
984 // not eliminated by the front-end, e.g. unused functions with internal
985 // linkage.
986 if (!NeedsRuntimeHook && ContainsProfiling)
987 emitRuntimeHook();
988
989 emitRegistration();
990 emitUses();
991 emitInitialization();
992 return true;
993}
994
995static FunctionCallee getOrInsertValueProfilingCall(
996 Module &M, const TargetLibraryInfo &TLI,
997 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
998 LLVMContext &Ctx = M.getContext();
999 auto *ReturnTy = Type::getVoidTy(C&: M.getContext());
1000
1001 AttributeList AL;
1002 if (auto AK = TLI.getExtAttrForI32Param(Signed: false))
1003 AL = AL.addParamAttribute(C&: M.getContext(), ArgNo: 2, Kind: AK);
1004
1005 assert((CallType == ValueProfilingCallType::Default ||
1006 CallType == ValueProfilingCallType::MemOp) &&
1007 "Must be Default or MemOp");
1008 Type *ParamTypes[] = {
1009#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
1010#include "llvm/ProfileData/InstrProfData.inc"
1011 };
1012 auto *ValueProfilingCallTy =
1013 FunctionType::get(Result: ReturnTy, Params: ArrayRef(ParamTypes), isVarArg: false);
1014 StringRef FuncName = CallType == ValueProfilingCallType::Default
1015 ? getInstrProfValueProfFuncName()
1016 : getInstrProfValueProfMemOpFuncName();
1017 return M.getOrInsertFunction(Name: FuncName, T: ValueProfilingCallTy, AttributeList: AL);
1018}
1019
1020void InstrLowerer::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
1021 GlobalVariable *Name = Ind->getName();
1022 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1023 uint64_t Index = Ind->getIndex()->getZExtValue();
1024 auto &PD = ProfileDataMap[Name];
1025 PD.NumValueSites[ValueKind] =
1026 std::max(a: PD.NumValueSites[ValueKind], b: (uint32_t)(Index + 1));
1027}
1028
1029void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
1030 // TODO: Value profiling heavily depends on the data section which is omitted
1031 // in lightweight mode. We need to move the value profile pointer to the
1032 // Counter struct to get this working.
1033 assert(
1034 !Opts.profile_correlate &&
1035 "Value profiling is not yet supported with lightweight instrumentation");
1036 GlobalVariable *Name = Ind->getName();
1037 auto It = ProfileDataMap.find(Val: Name);
1038 assert(It != ProfileDataMap.end() && It->second.DataVar &&
1039 "value profiling detected in function with no counter increment");
1040
1041 GlobalVariable *DataVar = It->second.DataVar;
1042 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
1043 uint64_t Index = Ind->getIndex()->getZExtValue();
1044 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
1045 Index += It->second.NumValueSites[Kind];
1046
1047 IRBuilder<> Builder(Ind);
1048 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
1049 llvm::InstrProfValueKind::IPVK_MemOPSize);
1050 CallInst *Call = nullptr;
1051 auto *TLI = &GetTLI(*Ind->getFunction());
1052 auto *NormalizedDataVarPtr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1053 C: DataVar, Ty: PointerType::get(C&: M.getContext(), AddressSpace: 0));
1054
1055 // To support value profiling calls within Windows exception handlers, funclet
1056 // information contained within operand bundles needs to be copied over to
1057 // the library call. This is required for the IR to be processed by the
1058 // WinEHPrepare pass.
1059 SmallVector<OperandBundleDef, 1> OpBundles;
1060 Ind->getOperandBundlesAsDefs(Defs&: OpBundles);
1061 if (!IsMemOpSize) {
1062 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1063 Builder.getInt32(C: Index)};
1064 Call = Builder.CreateCall(Callee: getOrInsertValueProfilingCall(M, TLI: *TLI), Args,
1065 OpBundles);
1066 } else {
1067 Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,
1068 Builder.getInt32(C: Index)};
1069 Call = Builder.CreateCall(
1070 Callee: getOrInsertValueProfilingCall(M, TLI: *TLI, CallType: ValueProfilingCallType::MemOp),
1071 Args, OpBundles);
1072 }
1073 if (auto AK = TLI->getExtAttrForI32Param(Signed: false))
1074 Call->addParamAttr(ArgNo: 2, Kind: AK);
1075 Ind->replaceAllUsesWith(V: Call);
1076 Ind->eraseFromParent();
1077}
1078
1079GlobalVariable *InstrLowerer::getOrCreateBiasVar(StringRef VarName) {
1080 GlobalVariable *Bias = M.getGlobalVariable(Name: VarName);
1081 if (Bias)
1082 return Bias;
1083
1084 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1085
1086 // Compiler must define this variable when runtime counter relocation
1087 // is being used. Runtime has a weak external reference that is used
1088 // to check whether that's the case or not.
1089 Bias = new GlobalVariable(M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
1090 Constant::getNullValue(Ty: Int64Ty), VarName);
1091 Bias->setVisibility(GlobalVariable::HiddenVisibility);
1092 // A definition that's weak (linkonce_odr) without being in a COMDAT
1093 // section wouldn't lead to link errors, but it would lead to a dead
1094 // data word from every TU but one. Putting it in COMDAT ensures there
1095 // will be exactly one data slot in the link.
1096 if (TT.supportsCOMDAT())
1097 Bias->setComdat(M.getOrInsertComdat(Name: VarName));
1098
1099 return Bias;
1100}
1101
1102Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
1103 auto *Counters = getOrCreateRegionCounters(Inc: I);
1104 IRBuilder<> Builder(I);
1105
1106 if (isa<InstrProfTimestampInst>(Val: I))
1107 Counters->setAlignment(Align(8));
1108
1109 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
1110 Ty: Counters->getValueType(), Ptr: Counters, Idx0: 0, Idx1: I->getIndex()->getZExtValue());
1111
1112 if (!isRuntimeCounterRelocationEnabled())
1113 return Addr;
1114
1115 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1116 Function *Fn = I->getParent()->getParent();
1117 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
1118 if (!BiasLI) {
1119 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1120 auto *Bias = getOrCreateBiasVar(VarName: getInstrProfCounterBiasVarName());
1121 BiasLI = EntryBuilder.CreateLoad(Ty: Int64Ty, Ptr: Bias, Name: "profc_bias");
1122 // Bias doesn't change after startup.
1123 BiasLI->setMetadata(KindID: LLVMContext::MD_invariant_load,
1124 Node: MDNode::get(Context&: M.getContext(), MDs: {}));
1125 }
1126 auto *Add = Builder.CreateAdd(LHS: Builder.CreatePtrToInt(V: Addr, DestTy: Int64Ty), RHS: BiasLI);
1127 return Builder.CreateIntToPtr(V: Add, DestTy: Addr->getType());
1128}
1129
1130Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
1131 auto *Bitmaps = getOrCreateRegionBitmaps(Inc: I);
1132 if (!isRuntimeCounterRelocationEnabled())
1133 return Bitmaps;
1134
1135 // Put BiasLI onto the entry block.
1136 Type *Int64Ty = Type::getInt64Ty(C&: M.getContext());
1137 Function *Fn = I->getFunction();
1138 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1139 auto *Bias = getOrCreateBiasVar(VarName: getInstrProfBitmapBiasVarName());
1140 auto *BiasLI = EntryBuilder.CreateLoad(Ty: Int64Ty, Ptr: Bias, Name: "profbm_bias");
1141 // Assume BiasLI invariant (in the function at least)
1142 BiasLI->setMetadata(KindID: LLVMContext::MD_invariant_load,
1143 Node: MDNode::get(Context&: M.getContext(), MDs: {}));
1144
1145 // Add Bias to Bitmaps and put it before the intrinsic.
1146 IRBuilder<> Builder(I);
1147 return Builder.CreatePtrAdd(Ptr: Bitmaps, Offset: BiasLI, Name: "profbm_addr");
1148}
1149
1150void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
1151 auto *Addr = getCounterAddress(I: CoverInstruction);
1152 IRBuilder<> Builder(CoverInstruction);
1153 if (Opts.conditional_counter_update) {
1154 Instruction *SplitBefore = CoverInstruction->getNextNode();
1155 auto &Ctx = CoverInstruction->getParent()->getContext();
1156 auto *Int8Ty = llvm::Type::getInt8Ty(C&: Ctx);
1157 Value *Load = Builder.CreateLoad(Ty: Int8Ty, Ptr: Addr, Name: "pgocount");
1158 Value *Cmp = Builder.CreateIsNotNull(Arg: Load, Name: "pgocount.ifnonzero");
1159 Instruction *ThenBranch =
1160 SplitBlockAndInsertIfThen(Cond: Cmp, SplitBefore, Unreachable: false);
1161 Builder.SetInsertPoint(ThenBranch);
1162 }
1163
1164 // We store zero to represent that this block is covered.
1165 Builder.CreateStore(Val: Builder.getInt8(C: 0), Ptr: Addr);
1166 CoverInstruction->eraseFromParent();
1167}
1168
1169void InstrLowerer::lowerTimestamp(
1170 InstrProfTimestampInst *TimestampInstruction) {
1171 assert(TimestampInstruction->getIndex()->isNullValue() &&
1172 "timestamp probes are always the first probe for a function");
1173 auto &Ctx = M.getContext();
1174 auto *TimestampAddr = getCounterAddress(I: TimestampInstruction);
1175 IRBuilder<> Builder(TimestampInstruction);
1176 auto *CalleeTy =
1177 FunctionType::get(Result: Type::getVoidTy(C&: Ctx), Params: TimestampAddr->getType(), isVarArg: false);
1178 auto Callee = M.getOrInsertFunction(
1179 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), T: CalleeTy);
1180 Builder.CreateCall(Callee, Args: {TimestampAddr});
1181 TimestampInstruction->eraseFromParent();
1182}
1183
1184InstrLowerer::GPUPGOInvariants &
1185InstrLowerer::getOrCreateGPUInvariants(Function *F) {
1186 auto It = GPUInvariantsCache.find(Val: F);
1187 if (It != GPUInvariantsCache.end())
1188 return It->second;
1189
1190 LLVMContext &Context = M.getContext();
1191 auto *Int32Ty = Type::getInt32Ty(C&: Context);
1192
1193 BasicBlock &EntryBB = F->getEntryBlock();
1194 IRBuilder<> Builder(&*EntryBB.getFirstInsertionPt());
1195
1196 Value *Matched = ConstantInt::getTrue(Context);
1197 if (Opts.offload_pgo_sampling > 0) {
1198 FunctionCallee IsSampledFn =
1199 M.getOrInsertFunction(Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(
1200 CallImpl: RTLIB::impl___llvm_profile_sampling_gpu),
1201 RetTy: Int32Ty, Args: Int32Ty);
1202 Value *SampledInt = Builder.CreateCall(
1203 Callee: IsSampledFn, Args: {ConstantInt::get(Ty: Int32Ty, V: Opts.offload_pgo_sampling)},
1204 Name: "pgo.sampled");
1205 Matched = Builder.CreateICmpNE(LHS: SampledInt, RHS: ConstantInt::get(Ty: Int32Ty, V: 0),
1206 Name: "pgo.matched");
1207 }
1208
1209 auto &Inv = GPUInvariantsCache[F];
1210 Inv.Matched = Matched;
1211 return Inv;
1212}
1213
1214void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
1215 IRBuilder<> Builder(Inc);
1216 if (isGPUProfTarget(M)) {
1217 Function *F = Inc->getFunction();
1218 auto &Inv = getOrCreateGPUInvariants(F);
1219
1220 LLVMContext &Context = M.getContext();
1221 auto *Int64Ty = Type::getInt64Ty(C&: Context);
1222 auto *PtrTy = PointerType::getUnqual(C&: Context);
1223
1224 auto *Addr = getCounterAddress(I: Inc);
1225
1226 // Store the device wave/warp size into the profile data struct once per
1227 // function. AMDGPU folds llvm.amdgcn.wavefrontsize to the subtarget's
1228 // constant; other GPUs use their fixed warp size.
1229 if (!Inv.WaveSizeStored) {
1230 Inv.WaveSizeStored = true;
1231 GlobalVariable *NamePtr = Inc->getName();
1232 auto &PD = ProfileDataMap[NamePtr];
1233 if (PD.DataVar) {
1234 IRBuilder<> EntryBuilder(&*F->getEntryBlock().getFirstInsertionPt());
1235 Value *WaveSize16 = nullptr;
1236 // Look the intrinsic up by name so this target-agnostic pass does not
1237 // pull in IntrinsicsAMDGPU.h. AMDGPU folds the intrinsic to the
1238 // subtarget's wavefront size; other GPUs fall back to a 32-lane warp.
1239 if (TT.isAMDGPU()) {
1240 Intrinsic::ID WaveSizeID =
1241 Intrinsic::lookupIntrinsicID(Name: "llvm.amdgcn.wavefrontsize");
1242 if (WaveSizeID != Intrinsic::not_intrinsic) {
1243 Function *WaveSizeFn =
1244 Intrinsic::getOrInsertDeclaration(M: &M, id: WaveSizeID);
1245 Value *WaveSize = EntryBuilder.CreateCall(Callee: WaveSizeFn);
1246 WaveSize16 = EntryBuilder.CreateTrunc(
1247 V: WaveSize, DestTy: Type::getInt16Ty(C&: Context), Name: "wavesize.i16");
1248 }
1249 }
1250 if (!WaveSize16)
1251 WaveSize16 = ConstantInt::get(Ty: Type::getInt16Ty(C&: Context), V: 32);
1252 Value *WaveSizeAddr = EntryBuilder.CreateStructGEP(
1253 Ty: PD.DataVar->getValueType(), Ptr: PD.DataVar, Idx: 9, Name: "profd.wavesize");
1254 EntryBuilder.CreateStore(Val: WaveSize16, Ptr: WaveSizeAddr);
1255 }
1256 }
1257
1258 GlobalVariable *UniformCounters = getOrCreateUniformCounters(Inc);
1259 Value *UniformAddrArg = ConstantPointerNull::get(T: PtrTy);
1260 if (UniformCounters) {
1261 Value *UniformIndices[] = {Builder.getInt32(C: 0), Inc->getIndex()};
1262 Value *UniformAddr = Builder.CreateInBoundsGEP(
1263 Ty: UniformCounters->getValueType(), Ptr: UniformCounters, IdxList: UniformIndices,
1264 Name: "unifctr.addr");
1265 UniformAddrArg =
1266 Builder.CreatePointerBitCastOrAddrSpaceCast(V: UniformAddr, DestTy: PtrTy);
1267 }
1268 Value *CastAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(V: Addr, DestTy: PtrTy);
1269 Value *StepI64 =
1270 Builder.CreateZExtOrTrunc(V: Inc->getStep(), DestTy: Int64Ty, Name: "step.i64");
1271
1272 auto *CalleeTy = FunctionType::get(Result: Type::getVoidTy(C&: Context),
1273 Params: {PtrTy, PtrTy, Int64Ty}, isVarArg: false);
1274 FunctionCallee Callee =
1275 M.getOrInsertFunction(Name: RTLIB::RuntimeLibcallsInfo::getLibcallImplName(
1276 CallImpl: RTLIB::impl___llvm_profile_instrument_gpu),
1277 T: CalleeTy);
1278
1279 if (Opts.offload_pgo_sampling > 0) {
1280 BasicBlock *CurBB = Builder.GetInsertBlock();
1281 BasicBlock *ContBB =
1282 CurBB->splitBasicBlock(I: BasicBlock::iterator(Inc), BBName: "po_cont");
1283 BasicBlock *ThenBB = BasicBlock::Create(Context, Name: "po_then", Parent: F);
1284
1285 CurBB->getTerminator()->eraseFromParent();
1286 IRBuilder<> HeadBuilder(CurBB);
1287 HeadBuilder.CreateCondBr(Cond: Inv.Matched, True: ThenBB, False: ContBB);
1288
1289 IRBuilder<> ThenBuilder(ThenBB);
1290 ThenBuilder.CreateCall(Callee, Args: {CastAddr, UniformAddrArg, StepI64});
1291 ThenBuilder.CreateBr(Dest: ContBB);
1292 } else {
1293 Builder.CreateCall(Callee, Args: {CastAddr, UniformAddrArg, StepI64});
1294 }
1295 Inc->eraseFromParent();
1296 return;
1297 }
1298
1299 auto *Addr = getCounterAddress(I: Inc);
1300 // If promotion is enabled then delay generating atomic updates until
1301 // after promotion is done.
1302 if ((!isCounterPromotionEnabled() && isAtomic()) ||
1303 (Inc->getIndex()->isNullValue() && Opts.atomic_first_counter)) {
1304 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Add, Ptr: Addr, Val: Inc->getStep(),
1305 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
1306 } else {
1307 Value *IncStep = Inc->getStep();
1308 Value *Load = Builder.CreateLoad(Ty: IncStep->getType(), Ptr: Addr, Name: "pgocount");
1309 auto *Count = Builder.CreateAdd(LHS: Load, RHS: Inc->getStep());
1310 auto *Store = Builder.CreateStore(Val: Count, Ptr: Addr);
1311 if (isCounterPromotionEnabled())
1312 PromotionCandidates.emplace_back(args: cast<Instruction>(Val: Load), args&: Store);
1313 }
1314 Inc->eraseFromParent();
1315}
1316
1317void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
1318 ConstantArray *Names =
1319 cast<ConstantArray>(Val: CoverageNamesVar->getInitializer());
1320 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
1321 Constant *NC = Names->getOperand(i_nocapture: I);
1322 Value *V = NC->stripPointerCasts();
1323 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
1324 GlobalVariable *Name = cast<GlobalVariable>(Val: V);
1325
1326 Name->setLinkage(GlobalValue::PrivateLinkage);
1327 ReferencedNames.push_back(x: Name);
1328 if (isa<ConstantExpr>(Val: NC))
1329 NC->dropAllReferences();
1330 }
1331 CoverageNamesVar->eraseFromParent();
1332}
1333
1334void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
1335 InstrProfMCDCTVBitmapUpdate *Update) {
1336 auto &Ctx = M.getContext();
1337 IRBuilder<> Builder(Update);
1338 auto *Int8Ty = Type::getInt8Ty(C&: Ctx);
1339 auto *Int32Ty = Type::getInt32Ty(C&: Ctx);
1340 auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
1341 auto *BitmapAddr = getBitmapAddress(I: Update);
1342
1343 // Load Temp Val + BitmapIdx.
1344 // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
1345 auto *Temp = Builder.CreateAdd(
1346 LHS: Builder.CreateLoad(Ty: Int32Ty, Ptr: MCDCCondBitmapAddr, Name: "mcdc.temp"),
1347 RHS: Update->getBitmapIndex());
1348
1349 // Calculate byte offset using div8.
1350 // %1 = lshr i32 %mcdc.temp, 3
1351 auto *BitmapByteOffset = Builder.CreateLShr(LHS: Temp, RHS: 0x3);
1352
1353 // Add byte offset to section base byte address.
1354 // %4 = getelementptr inbounds i8, ptr @__profbm_test, i32 %1
1355 auto *BitmapByteAddr =
1356 Builder.CreateInBoundsPtrAdd(Ptr: BitmapAddr, Offset: BitmapByteOffset);
1357
1358 // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8)
1359 // %5 = and i32 %mcdc.temp, 7
1360 // %6 = trunc i32 %5 to i8
1361 auto *BitToSet = Builder.CreateTrunc(V: Builder.CreateAnd(LHS: Temp, RHS: 0x7), DestTy: Int8Ty);
1362
1363 // Shift bit offset left to form a bitmap.
1364 // %7 = shl i8 1, %6
1365 auto *ShiftedVal = Builder.CreateShl(LHS: Builder.getInt8(C: 0x1), RHS: BitToSet);
1366
1367 // Load profile bitmap byte.
1368 // %mcdc.bits = load i8, ptr %4, align 1
1369 auto *Bitmap = Builder.CreateLoad(Ty: Int8Ty, Ptr: BitmapByteAddr, Name: "mcdc.bits");
1370
1371 if (isAtomic()) {
1372 // If ((Bitmap & Val) != Val), then execute atomic (Bitmap |= Val).
1373 // Note, just-loaded Bitmap might not be up-to-date. Use it just for
1374 // early testing.
1375 auto *Masked = Builder.CreateAnd(LHS: Bitmap, RHS: ShiftedVal);
1376 auto *ShouldStore = Builder.CreateICmpNE(LHS: Masked, RHS: ShiftedVal);
1377
1378 // Assume updating will be rare.
1379 auto *Unlikely = MDBuilder(Ctx).createUnlikelyBranchWeights();
1380 Instruction *ThenBranch =
1381 SplitBlockAndInsertIfThen(Cond: ShouldStore, SplitBefore: Update, Unreachable: false, BranchWeights: Unlikely);
1382
1383 // Execute if (unlikely(ShouldStore)).
1384 Builder.SetInsertPoint(ThenBranch);
1385 Builder.CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: BitmapByteAddr, Val: ShiftedVal,
1386 Align: MaybeAlign(), Ordering: AtomicOrdering::Monotonic);
1387 } else {
1388 // Perform logical OR of profile bitmap byte and shifted bit offset.
1389 // %8 = or i8 %mcdc.bits, %7
1390 auto *Result = Builder.CreateOr(LHS: Bitmap, RHS: ShiftedVal);
1391
1392 // Store the updated profile bitmap byte.
1393 // store i8 %8, ptr %3, align 1
1394 Builder.CreateStore(Val: Result, Ptr: BitmapByteAddr);
1395 }
1396
1397 Update->eraseFromParent();
1398}
1399
1400/// Get the name of a profiling variable for a particular function.
1401static std::string getVarName(const InstrumentationOptions &Opts,
1402 InstrProfInstBase *Inc, StringRef Prefix,
1403 bool &Renamed) {
1404 StringRef NamePrefix = getInstrProfNameVarPrefix();
1405 StringRef Name = Inc->getName()->getName().substr(Start: NamePrefix.size());
1406 Function *F = Inc->getParent()->getParent();
1407 Module *M = F->getParent();
1408 if (!Opts.hash_based_counter_split || !isIRPGOFlagSet(M) ||
1409 !canRenameComdatFunc(F: *F)) {
1410 Renamed = false;
1411 return (Prefix + Name).str();
1412 }
1413 Renamed = true;
1414 uint64_t FuncHash = Inc->getHash()->getZExtValue();
1415 SmallVector<char, 24> HashPostfix;
1416 if (Name.ends_with(Suffix: (Twine(".") + Twine(FuncHash)).toStringRef(Out&: HashPostfix)))
1417 return (Prefix + Name).str();
1418 return (Prefix + Name + "." + Twine(FuncHash)).str();
1419}
1420
1421static inline bool shouldRecordFunctionAddr(Function *F) {
1422 // Only record function addresses if IR PGO is enabled or if clang value
1423 // profiling is enabled. Recording function addresses greatly increases object
1424 // file size, because it prevents the inliner from deleting functions that
1425 // have been inlined everywhere.
1426 if (!profDataReferencedByCode(M: *F->getParent()))
1427 return false;
1428
1429 // Check the linkage
1430 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
1431 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
1432 !HasAvailableExternallyLinkage)
1433 return true;
1434
1435 // A function marked 'alwaysinline' with available_externally linkage can't
1436 // have its address taken. Doing so would create an undefined external ref to
1437 // the function, which would fail to link.
1438 if (HasAvailableExternallyLinkage &&
1439 F->hasFnAttribute(Kind: Attribute::AlwaysInline))
1440 return false;
1441
1442 // Prohibit function address recording if the function is both internal and
1443 // COMDAT. This avoids the profile data variable referencing internal symbols
1444 // in COMDAT.
1445 if (F->hasLocalLinkage() && F->hasComdat())
1446 return false;
1447
1448 // Check uses of this function for other than direct calls or invokes to it.
1449 // Inline virtual functions have linkeOnceODR linkage. When a key method
1450 // exists, the vtable will only be emitted in the TU where the key method
1451 // is defined. In a TU where vtable is not available, the function won't
1452 // be 'addresstaken'. If its address is not recorded here, the profile data
1453 // with missing address may be picked by the linker leading to missing
1454 // indirect call target info.
1455 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
1456}
1457
1458static inline bool shouldUsePublicSymbol(Function *Fn) {
1459 // It isn't legal to make an alias of this function at all
1460 if (Fn->isDeclarationForLinker())
1461 return true;
1462
1463 // Symbols with local linkage can just use the symbol directly without
1464 // introducing relocations
1465 if (Fn->hasLocalLinkage())
1466 return true;
1467
1468 // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some
1469 // unfavorable interaction between the new alias and the alias renaming done
1470 // in LowerTypeTests under ThinLTO. For comdat functions that would normally
1471 // be deduplicated, but the renaming scheme ends up preventing renaming, since
1472 // it creates unique names for each alias, resulting in duplicated symbols. In
1473 // the future, we should update the CFI related passes to migrate these
1474 // aliases to the same module as the jump-table they refer to will be defined.
1475 if (Fn->hasMetadata(KindID: LLVMContext::MD_type))
1476 return true;
1477
1478 // For comdat functions, an alias would need the same linkage as the original
1479 // function and hidden visibility. There is no point in adding an alias with
1480 // identical linkage an visibility to avoid introducing symbolic relocations.
1481 if (Fn->hasComdat() &&
1482 (Fn->getVisibility() == GlobalValue::VisibilityTypes::HiddenVisibility))
1483 return true;
1484
1485 // its OK to use an alias
1486 return false;
1487}
1488
1489static inline Constant *getFuncAddrForProfData(Function *Fn) {
1490 auto *Int8PtrTy = PointerType::getUnqual(C&: Fn->getContext());
1491 // Store a nullptr in __llvm_profd, if we shouldn't use a real address
1492 if (!shouldRecordFunctionAddr(F: Fn))
1493 return ConstantPointerNull::get(T: Int8PtrTy);
1494
1495 // If we can't use an alias, we must use the public symbol, even though this
1496 // may require a symbolic relocation.
1497 if (shouldUsePublicSymbol(Fn))
1498 return Fn;
1499
1500 // For GPU targets, weak functions cannot use private aliases because
1501 // LTO may pick a different TU's copy, leaving the alias undefined
1502 if (isGPUProfTarget(M: *Fn->getParent()) &&
1503 GlobalValue::isWeakForLinker(Linkage: Fn->getLinkage()))
1504 return Fn;
1505
1506 // When possible use a private alias to avoid symbolic relocations.
1507 auto *GA = GlobalAlias::create(Linkage: GlobalValue::LinkageTypes::PrivateLinkage,
1508 Name: Fn->getName() + ".local", Aliasee: Fn);
1509
1510 // When the instrumented function is a COMDAT function, we cannot use a
1511 // private alias. If we did, we would create reference to a local label in
1512 // this function's section. If this version of the function isn't selected by
1513 // the linker, then the metadata would introduce a reference to a discarded
1514 // section. So, for COMDAT functions, we need to adjust the linkage of the
1515 // alias. Using hidden visibility avoids a dynamic relocation and an entry in
1516 // the dynamic symbol table.
1517 //
1518 // Note that this handles COMDAT functions with visibility other than Hidden,
1519 // since that case is covered in shouldUsePublicSymbol()
1520 if (Fn->hasComdat()) {
1521 GA->setLinkage(Fn->getLinkage());
1522 GA->setVisibility(GlobalValue::VisibilityTypes::HiddenVisibility);
1523 }
1524
1525 // appendToCompilerUsed(*Fn->getParent(), {GA});
1526
1527 return GA;
1528}
1529
1530static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
1531 // NVPTX is an ELF target but PTX does not expose sections or linker symbols.
1532 if (TT.isNVPTX())
1533 return true;
1534
1535 // compiler-rt uses linker support to get data/counters/name start/end for
1536 // ELF, COFF, Mach-O, XCOFF, and Wasm.
1537 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
1538 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF() ||
1539 TT.isOSBinFormatWasm())
1540 return false;
1541
1542 return true;
1543}
1544
1545void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO,
1546 StringRef CounterGroupName) {
1547 // Place lowered global variables in a comdat group if the associated function
1548 // or global variable is a COMDAT. This will make sure that only one copy of
1549 // global variable (e.g. function counters) of the COMDAT function will be
1550 // emitted after linking.
1551 bool NeedComdat = needsComdatForCounter(GV: *GO, M);
1552 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
1553
1554 if (!UseComdat)
1555 return;
1556
1557 // Keep in mind that this pass may run before the inliner, so we need to
1558 // create a new comdat group (for counters, profiling data, etc). If we use
1559 // the comdat of the parent function, that will result in relocations against
1560 // discarded sections.
1561 //
1562 // If the data variable is referenced by code, non-counter variables (notably
1563 // profiling data) and counters have to be in different comdats for COFF
1564 // because the Visual C++ linker will report duplicate symbol errors if there
1565 // are multiple external symbols with the same name marked
1566 // IMAGE_COMDAT_SELECT_ASSOCIATIVE.
1567 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
1568 ? GV->getName()
1569 : CounterGroupName;
1570 Comdat *C = M.getOrInsertComdat(Name: GroupName);
1571
1572 if (!NeedComdat) {
1573 // Object file format must be ELF since `UseComdat && !NeedComdat` is true.
1574 //
1575 // For ELF, when not using COMDAT, put counters, data and values into a
1576 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
1577 // allows -z start-stop-gc to discard the entire group when the function is
1578 // discarded.
1579 C->setSelectionKind(Comdat::NoDeduplicate);
1580 }
1581 GV->setComdat(C);
1582 // COFF doesn't allow the comdat group leader to have private linkage, so
1583 // upgrade private linkage to internal linkage to produce a symbol table
1584 // entry.
1585 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
1586 GV->setLinkage(GlobalValue::InternalLinkage);
1587}
1588
1589static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {
1590 if (!profDataReferencedByCode(M: *GV->getParent()))
1591 return false;
1592
1593 if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&
1594 !GV->hasAvailableExternallyLinkage())
1595 return true;
1596
1597 // This avoids the profile data from referencing internal symbols in
1598 // COMDAT.
1599 if (GV->hasLocalLinkage() && GV->hasComdat())
1600 return false;
1601
1602 return true;
1603}
1604
1605// FIXME: Introduce an internal alias like what's done for functions to reduce
1606// the number of relocation entries.
1607static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {
1608 // Store a nullptr in __profvt_ if a real address shouldn't be used.
1609 if (!shouldRecordVTableAddr(GV))
1610 return ConstantPointerNull::get(T: PointerType::getUnqual(C&: GV->getContext()));
1611
1612 return GV;
1613}
1614
1615void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) {
1616 assert(Opts.profile_correlate != ProfCorrelatorKind::DebugInfo &&
1617 "Value profiling is not supported with lightweight instrumentation");
1618 if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
1619 return;
1620
1621 // Skip llvm internal global variable or __prof variables.
1622 if (GV->getName().starts_with(Prefix: "llvm.") ||
1623 GV->getName().starts_with(Prefix: "__llvm") ||
1624 GV->getName().starts_with(Prefix: "__prof"))
1625 return;
1626
1627 // VTableProfData already created
1628 auto It = VTableDataMap.find(Val: GV);
1629 if (It != VTableDataMap.end() && It->second)
1630 return;
1631
1632 GlobalValue::LinkageTypes Linkage = GV->getLinkage();
1633 GlobalValue::VisibilityTypes Visibility = GV->getVisibility();
1634
1635 // This is to keep consistent with per-function profile data
1636 // for correctness.
1637 if (TT.isOSBinFormatXCOFF()) {
1638 Linkage = GlobalValue::InternalLinkage;
1639 Visibility = GlobalValue::DefaultVisibility;
1640 }
1641
1642 LLVMContext &Ctx = M.getContext();
1643 Type *DataTypes[] = {
1644#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
1645#include "llvm/ProfileData/InstrProfData.inc"
1646#undef INSTR_PROF_VTABLE_DATA
1647 };
1648
1649 auto *DataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes));
1650
1651 // Used by INSTR_PROF_VTABLE_DATA MACRO
1652 Constant *VTableAddr = getVTableAddrForProfData(GV);
1653 const std::string PGOVTableName = getIRPGOObjectName(GO: *GV);
1654 // Record the length of the vtable. This is needed since vtable pointers
1655 // loaded from C++ objects might be from the middle of a vtable definition.
1656 uint32_t VTableSizeVal = GV->getGlobalSize(DL: M.getDataLayout());
1657
1658 Constant *DataVals[] = {
1659#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
1660#include "llvm/ProfileData/InstrProfData.inc"
1661#undef INSTR_PROF_VTABLE_DATA
1662 };
1663
1664 auto *Data =
1665 new GlobalVariable(M, DataTy, /*constant=*/false, Linkage,
1666 ConstantStruct::get(T: DataTy, V: DataVals),
1667 getInstrProfVTableVarPrefix() + PGOVTableName);
1668
1669 Data->setVisibility(Visibility);
1670 Data->setSection(getInstrProfSectionName(IPSK: IPSK_vtab, OF: TT.getObjectFormat()));
1671 Data->setAlignment(Align(8));
1672
1673 maybeSetComdat(GV: Data, GO: GV, CounterGroupName: Data->getName());
1674
1675 VTableDataMap[GV] = Data;
1676
1677 ReferencedVTables.push_back(x: GV);
1678
1679 // VTable <Hash, Addr> is used by runtime but not referenced by other
1680 // sections. Conservatively mark it linker retained.
1681 UsedVars.push_back(x: Data);
1682}
1683
1684GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc,
1685 InstrProfSectKind IPSK) {
1686 GlobalVariable *NamePtr = Inc->getName();
1687
1688 // Match the linkage and visibility of the name global.
1689 Function *Fn = Inc->getParent()->getParent();
1690 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
1691 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1692
1693 // Use internal rather than private linkage so the counter variable shows up
1694 // in the symbol table when using debug info for correlation.
1695 if (Opts.profile_correlate == ProfCorrelatorKind::DebugInfo &&
1696 TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage)
1697 Linkage = GlobalValue::InternalLinkage;
1698
1699 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1700 // symbols in the same csect won't be discarded. When there are duplicate weak
1701 // symbols, we can NOT guarantee that the relocations get resolved to the
1702 // intended weak symbol, so we can not ensure the correctness of the relative
1703 // CounterPtr, so we have to use private linkage for counter and data symbols.
1704 if (TT.isOSBinFormatXCOFF()) {
1705 Linkage = GlobalValue::PrivateLinkage;
1706 Visibility = GlobalValue::DefaultVisibility;
1707 }
1708 // Move the name variable to the right section.
1709 bool Renamed;
1710 GlobalVariable *Ptr;
1711 StringRef VarPrefix;
1712 std::string VarName;
1713 if (IPSK == IPSK_cnts) {
1714 VarPrefix = getInstrProfCountersVarPrefix();
1715 VarName = getVarName(Opts, Inc, Prefix: VarPrefix, Renamed);
1716 InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Val: Inc);
1717 Ptr = createRegionCounters(Inc: CntrIncrement, Name: VarName, Linkage);
1718 } else if (IPSK == IPSK_bitmap) {
1719 VarPrefix = getInstrProfBitmapVarPrefix();
1720 VarName = getVarName(Opts, Inc, Prefix: VarPrefix, Renamed);
1721 InstrProfMCDCBitmapInstBase *BitmapUpdate =
1722 dyn_cast<InstrProfMCDCBitmapInstBase>(Val: Inc);
1723 Ptr = createRegionBitmaps(Inc: BitmapUpdate, Name: VarName, Linkage);
1724 } else {
1725 llvm_unreachable("Profile Section must be for Counters or Bitmaps");
1726 }
1727
1728 Ptr->setVisibility(Visibility);
1729 Ptr->setSection(getInstrProfSectionName(IPSK, OF: TT.getObjectFormat()));
1730 Ptr->setLinkage(Linkage);
1731 if (isGPUProfTarget(M) && !Ptr->hasComdat()) {
1732 Ptr->setComdat(M.getOrInsertComdat(Name: VarName));
1733 Ptr->setLinkage(GlobalValue::LinkOnceODRLinkage);
1734 Ptr->setVisibility(GlobalValue::ProtectedVisibility);
1735 } else {
1736 maybeSetComdat(GV: Ptr, GO: Fn, CounterGroupName: VarName);
1737 }
1738 return Ptr;
1739}
1740
1741GlobalVariable *
1742InstrLowerer::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
1743 StringRef Name,
1744 GlobalValue::LinkageTypes Linkage) {
1745 uint64_t NumBytes = Inc->getNumBitmapBytes();
1746 auto *BitmapTy = ArrayType::get(ElementType: Type::getInt8Ty(C&: M.getContext()), NumElements: NumBytes);
1747 auto GV = new GlobalVariable(M, BitmapTy, false, Linkage,
1748 Constant::getNullValue(Ty: BitmapTy), Name);
1749 GV->setAlignment(Align(1));
1750 return GV;
1751}
1752
1753GlobalVariable *
1754InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
1755 GlobalVariable *NamePtr = Inc->getName();
1756 auto &PD = ProfileDataMap[NamePtr];
1757 if (PD.RegionBitmaps)
1758 return PD.RegionBitmaps;
1759
1760 // If RegionBitmaps doesn't already exist, create it by first setting up
1761 // the corresponding profile section.
1762 auto *BitmapPtr = setupProfileSection(Inc, IPSK: IPSK_bitmap);
1763 PD.RegionBitmaps = BitmapPtr;
1764 PD.NumBitmapBytes = Inc->getNumBitmapBytes();
1765
1766 if (PD.NumBitmapBytes &&
1767 Opts.profile_correlate == ProfCorrelatorKind::DebugInfo) {
1768 LLVMContext &Ctx = M.getContext();
1769 Function *Fn = Inc->getParent()->getParent();
1770 if (auto *SP = Fn->getSubprogram()) {
1771 DIBuilder DB(M, true, SP->getUnit());
1772 Metadata *FunctionNameAnnotation[] = {
1773 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::FunctionNameAttributeName),
1774 MDString::get(Context&: Ctx, Str: getPGOFuncNameVarInitializer(NameVar: NamePtr)),
1775 };
1776 Metadata *NumBitmapBitsAnnotation[] = {
1777 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::NumBitmapBitsAttributeName),
1778 ConstantAsMetadata::get(C: Inc->getNumBitmapBits()),
1779 };
1780 auto Annotations = DB.getOrCreateArray(Elements: {
1781 MDNode::get(Context&: Ctx, MDs: FunctionNameAnnotation),
1782 MDNode::get(Context&: Ctx, MDs: NumBitmapBitsAnnotation),
1783 });
1784 auto *DICounter = DB.createGlobalVariableExpression(
1785 Context: SP, Name: BitmapPtr->getName(), /*LinkageName=*/StringRef(), File: SP->getFile(),
1786 /*LineNo=*/0, Ty: DB.createUnspecifiedType(Name: "Profile Bitmap Type"),
1787 IsLocalToUnit: BitmapPtr->hasLocalLinkage(), /*IsDefined=*/isDefined: true, /*Expr=*/nullptr,
1788 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1789 Annotations);
1790 BitmapPtr->addDebugInfo(GV: DICounter);
1791 DB.finalizeSubprogram(SP);
1792 DB.finalize();
1793 }
1794
1795 // Mark the bitmap variable as used so that it isn't optimized out.
1796 CompilerUsedVars.push_back(x: PD.RegionBitmaps);
1797 }
1798
1799 return PD.RegionBitmaps;
1800}
1801
1802GlobalVariable *
1803InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
1804 GlobalValue::LinkageTypes Linkage) {
1805 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1806 auto &Ctx = M.getContext();
1807 GlobalVariable *GV;
1808 if (isa<InstrProfCoverInst>(Val: Inc)) {
1809 auto *CounterTy = Type::getInt8Ty(C&: Ctx);
1810 auto *CounterArrTy = ArrayType::get(ElementType: CounterTy, NumElements: NumCounters);
1811 // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
1812 std::vector<Constant *> InitialValues(NumCounters,
1813 Constant::getAllOnesValue(Ty: CounterTy));
1814 GV = new GlobalVariable(M, CounterArrTy, false, Linkage,
1815 ConstantArray::get(T: CounterArrTy, V: InitialValues),
1816 Name);
1817 GV->setAlignment(Align(1));
1818 } else {
1819 auto *CounterTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NumCounters);
1820 GV = new GlobalVariable(M, CounterTy, false, Linkage,
1821 Constant::getNullValue(Ty: CounterTy), Name);
1822 GV->setAlignment(Align(8));
1823 }
1824 return GV;
1825}
1826
1827GlobalVariable *
1828InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
1829 GlobalVariable *NamePtr = Inc->getName();
1830 auto &PD = ProfileDataMap[NamePtr];
1831 if (PD.RegionCounters)
1832 return PD.RegionCounters;
1833
1834 // If RegionCounters doesn't already exist, create it by first setting up
1835 // the corresponding profile section.
1836 auto *CounterPtr = setupProfileSection(Inc, IPSK: IPSK_cnts);
1837 PD.RegionCounters = CounterPtr;
1838
1839 if (Opts.profile_correlate == ProfCorrelatorKind::DebugInfo) {
1840 LLVMContext &Ctx = M.getContext();
1841 Function *Fn = Inc->getParent()->getParent();
1842 if (auto *SP = Fn->getSubprogram()) {
1843 DIBuilder DB(M, true, SP->getUnit());
1844 Metadata *FunctionNameAnnotation[] = {
1845 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::FunctionNameAttributeName),
1846 MDString::get(Context&: Ctx, Str: getPGOFuncNameVarInitializer(NameVar: NamePtr)),
1847 };
1848 Metadata *CFGHashAnnotation[] = {
1849 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::CFGHashAttributeName),
1850 ConstantAsMetadata::get(C: Inc->getHash()),
1851 };
1852 Metadata *NumCountersAnnotation[] = {
1853 MDString::get(Context&: Ctx, Str: InstrProfCorrelator::NumCountersAttributeName),
1854 ConstantAsMetadata::get(C: Inc->getNumCounters()),
1855 };
1856 auto Annotations = DB.getOrCreateArray(Elements: {
1857 MDNode::get(Context&: Ctx, MDs: FunctionNameAnnotation),
1858 MDNode::get(Context&: Ctx, MDs: CFGHashAnnotation),
1859 MDNode::get(Context&: Ctx, MDs: NumCountersAnnotation),
1860 });
1861 auto *DICounter = DB.createGlobalVariableExpression(
1862 Context: SP, Name: CounterPtr->getName(), /*LinkageName=*/StringRef(), File: SP->getFile(),
1863 /*LineNo=*/0, Ty: DB.createUnspecifiedType(Name: "Profile Data Type"),
1864 IsLocalToUnit: CounterPtr->hasLocalLinkage(), /*IsDefined=*/isDefined: true, /*Expr=*/nullptr,
1865 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1866 Annotations);
1867 CounterPtr->addDebugInfo(GV: DICounter);
1868 DB.finalizeSubprogram(SP);
1869 DB.finalize();
1870 }
1871
1872 // Mark the counter variable as used so that it isn't optimized out.
1873 CompilerUsedVars.push_back(x: PD.RegionCounters);
1874 }
1875
1876 // Create uniform counters before the data variable so that
1877 // UniformCounterPtr can reference them in createDataVariable().
1878 getOrCreateUniformCounters(Inc);
1879
1880 // Create the data variable (if it doesn't already exist).
1881 createDataVariable(Inc);
1882
1883 return PD.RegionCounters;
1884}
1885
1886GlobalVariable *
1887InstrLowerer::getOrCreateUniformCounters(InstrProfCntrInstBase *Inc) {
1888 // Uniform counters are only meaningful for GPU profile targets.
1889 if (!isGPUProfTarget(M))
1890 return nullptr;
1891
1892 GlobalVariable *NamePtr = Inc->getName();
1893 auto &PD = ProfileDataMap[NamePtr];
1894 if (PD.UniformCounters)
1895 return PD.UniformCounters;
1896
1897 assert(PD.RegionCounters && "region counters must be created first");
1898
1899 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1900
1901 LLVMContext &Ctx = M.getContext();
1902 ArrayType *CounterTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NumCounters);
1903
1904 bool Renamed;
1905 std::string VarName = getVarName(Opts, Inc, Prefix: "__llvm_prf_unifcnt_", Renamed);
1906
1907 auto *GV = new GlobalVariable(M, CounterTy, false, NamePtr->getLinkage(),
1908 Constant::getNullValue(Ty: CounterTy), VarName);
1909 GV->setAlignment(Align(8));
1910
1911 GV->setSection(getInstrProfSectionName(IPSK: IPSK_ucnts, OF: TT.getObjectFormat()));
1912
1913 GV->setComdat(M.getOrInsertComdat(Name: VarName));
1914 GV->setLinkage(GlobalValue::LinkOnceODRLinkage);
1915 GV->setVisibility(GlobalValue::ProtectedVisibility);
1916
1917 PD.UniformCounters = GV;
1918 CompilerUsedVars.push_back(x: GV);
1919
1920 return PD.UniformCounters;
1921}
1922
1923void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
1924 // When debug information is correlated to profile data, a data variable
1925 // is not needed.
1926 if (Opts.profile_correlate == ProfCorrelatorKind::DebugInfo)
1927 return;
1928
1929 GlobalVariable *NamePtr = Inc->getName();
1930 auto &PD = ProfileDataMap[NamePtr];
1931
1932 // Return if data variable was already created.
1933 if (PD.DataVar)
1934 return;
1935
1936 LLVMContext &Ctx = M.getContext();
1937
1938 Function *Fn = Inc->getParent()->getParent();
1939 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
1940 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1941
1942 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1943 // symbols in the same csect won't be discarded. When there are duplicate weak
1944 // symbols, we can NOT guarantee that the relocations get resolved to the
1945 // intended weak symbol, so we can not ensure the correctness of the relative
1946 // CounterPtr, so we have to use private linkage for counter and data symbols.
1947 if (TT.isOSBinFormatXCOFF()) {
1948 Linkage = GlobalValue::PrivateLinkage;
1949 Visibility = GlobalValue::DefaultVisibility;
1950 }
1951
1952 bool NeedComdat = needsComdatForCounter(GV: *Fn, M);
1953 bool Renamed;
1954
1955 // The Data Variable section is anchored to profile counters.
1956 std::string CntsVarName =
1957 getVarName(Opts, Inc, Prefix: getInstrProfCountersVarPrefix(), Renamed);
1958 std::string DataVarName =
1959 getVarName(Opts, Inc, Prefix: getInstrProfDataVarPrefix(), Renamed);
1960
1961 auto *Int8PtrTy = PointerType::getUnqual(C&: Ctx);
1962 // Allocate statically the array of pointers to value profile nodes for
1963 // the current function.
1964 Constant *ValuesPtrExpr = ConstantPointerNull::get(T: Int8PtrTy);
1965 uint64_t NS = 0;
1966 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1967 NS += PD.NumValueSites[Kind];
1968 if (NS > 0 && Opts.vp_static_alloc &&
1969 !needsRuntimeRegistrationOfSectionRange(TT)) {
1970 ArrayType *ValuesTy = ArrayType::get(ElementType: Type::getInt64Ty(C&: Ctx), NumElements: NS);
1971 auto *ValuesVar = new GlobalVariable(
1972 M, ValuesTy, false, Linkage, Constant::getNullValue(Ty: ValuesTy),
1973 getVarName(Opts, Inc, Prefix: getInstrProfValuesVarPrefix(), Renamed));
1974 ValuesVar->setVisibility(Visibility);
1975 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *ValuesVar);
1976 ValuesVar->setSection(
1977 getInstrProfSectionName(IPSK: IPSK_vals, OF: TT.getObjectFormat()));
1978 ValuesVar->setAlignment(Align(8));
1979 maybeSetComdat(GV: ValuesVar, GO: Fn, CounterGroupName: CntsVarName);
1980 ValuesPtrExpr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(
1981 C: ValuesVar, Ty: PointerType::get(C&: Fn->getContext(), AddressSpace: 0));
1982 }
1983
1984 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1985
1986 Constant *CounterPtr = PD.RegionCounters;
1987 Constant *UniformCounterPtr = PD.UniformCounters;
1988
1989 uint64_t NumBitmapBytes = PD.NumBitmapBytes;
1990
1991 // Create data variable.
1992 auto *IntPtrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
1993 auto *Int16Ty = Type::getInt16Ty(C&: Ctx);
1994 auto *Int16ArrayTy = ArrayType::get(ElementType: Int16Ty, NumElements: IPVK_Last + 1);
1995 auto *DataTy = getProfileDataTy();
1996
1997 Constant *FunctionAddr = getFuncAddrForProfData(Fn);
1998
1999 Constant *Int16ArrayVals[IPVK_Last + 1];
2000 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
2001 Int16ArrayVals[Kind] = ConstantInt::get(Ty: Int16Ty, V: PD.NumValueSites[Kind]);
2002
2003 uint16_t OffloadDeviceWaveSizeVal = 0;
2004
2005 if (isGPUProfTarget(M)) {
2006 // For GPU targets, weak functions need weak linkage for their profile data
2007 // aliases to allow linker deduplication across TUs
2008 if (GlobalValue::isWeakForLinker(Linkage: Fn->getLinkage()))
2009 Linkage = Fn->getLinkage();
2010 else
2011 Linkage = GlobalValue::ExternalLinkage;
2012 Visibility = GlobalValue::ProtectedVisibility;
2013 }
2014 // If the data variable is not referenced by code (if we don't emit
2015 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
2016 // data variable live under linker GC, the data variable can be private. This
2017 // optimization applies to ELF.
2018 //
2019 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
2020 // to be false.
2021 //
2022 // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
2023 // that other copies must have the same CFG and cannot have value profiling.
2024 // If no hash suffix, other profd copies may be referenced by code.
2025 if (!isGPUProfTarget(M) && NS == 0 &&
2026 !(DataReferencedByCode && NeedComdat && !Renamed) &&
2027 (TT.isOSBinFormatELF() ||
2028 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
2029 Linkage = GlobalValue::PrivateLinkage;
2030 Visibility = GlobalValue::DefaultVisibility;
2031 }
2032 // GPU-target ELF objects are always ET_DYN, so non-local symbols with
2033 // default visibility are preemptible. The CounterPtr label difference
2034 // emits a REL32 relocation that lld rejects against preemptible targets.
2035 if (TT.isGPU() && TT.isOSBinFormatELF() &&
2036 !GlobalValue::isLocalLinkage(Linkage))
2037 Visibility = GlobalValue::ProtectedVisibility;
2038 auto *Data =
2039 new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName);
2040
2041 Constant *RelativeCounterPtr;
2042 Constant *RelativeUniformCounterPtr = ConstantInt::get(Ty: IntPtrTy, V: 0);
2043 GlobalVariable *BitmapPtr = PD.RegionBitmaps;
2044 Constant *RelativeBitmapPtr = ConstantInt::get(Ty: IntPtrTy, V: 0);
2045 InstrProfSectKind DataSectionKind;
2046 // With binary profile correlation, profile data is not loaded into memory.
2047 // profile data must reference profile counter with an absolute relocation.
2048 if (Opts.profile_correlate == ProfCorrelatorKind::Binary) {
2049 DataSectionKind = IPSK_covdata;
2050 RelativeCounterPtr = ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy);
2051 if (BitmapPtr != nullptr)
2052 RelativeBitmapPtr = ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy);
2053 if (UniformCounterPtr != nullptr)
2054 RelativeUniformCounterPtr =
2055 ConstantExpr::getPtrToInt(C: UniformCounterPtr, Ty: IntPtrTy);
2056 } else if (TT.isNVPTX()) {
2057 // The NVPTX target cannot handle self-referencing constant expressions in
2058 // global initializers at all. Use absolute pointers and have the runtime
2059 // registration convert them to relative offsets.
2060 DataSectionKind = IPSK_data;
2061 RelativeCounterPtr = ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy);
2062 } else {
2063 // Reference the counter variable with a label difference (link-time
2064 // constant).
2065 DataSectionKind = IPSK_data;
2066 RelativeCounterPtr =
2067 ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: CounterPtr, Ty: IntPtrTy),
2068 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2069 if (BitmapPtr != nullptr)
2070 RelativeBitmapPtr =
2071 ConstantExpr::getSub(C1: ConstantExpr::getPtrToInt(C: BitmapPtr, Ty: IntPtrTy),
2072 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2073 if (UniformCounterPtr != nullptr)
2074 RelativeUniformCounterPtr = ConstantExpr::getSub(
2075 C1: ConstantExpr::getPtrToInt(C: UniformCounterPtr, Ty: IntPtrTy),
2076 C2: ConstantExpr::getPtrToInt(C: Data, Ty: IntPtrTy));
2077 }
2078
2079 Constant *DataVals[] = {
2080#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
2081#include "llvm/ProfileData/InstrProfData.inc"
2082 };
2083 Data->setInitializer(ConstantStruct::get(T: DataTy, V: DataVals));
2084
2085 Data->setVisibility(Visibility);
2086 Data->setSection(
2087 getInstrProfSectionName(IPSK: DataSectionKind, OF: TT.getObjectFormat()));
2088 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
2089 if (isGPUProfTarget(M) && !Data->hasComdat()) {
2090 Data->setComdat(M.getOrInsertComdat(Name: CntsVarName));
2091 Data->setLinkage(GlobalValue::LinkOnceODRLinkage);
2092 } else {
2093 maybeSetComdat(GV: Data, GO: Fn, CounterGroupName: CntsVarName);
2094 }
2095
2096 PD.DataVar = Data;
2097
2098 // Mark the data variable as used so that it isn't stripped out.
2099 CompilerUsedVars.push_back(x: Data);
2100 // Now that the linkage set by the FE has been passed to the data and counter
2101 // variables, reset Name variable's linkage and visibility to private so that
2102 // it can be removed later by the compiler.
2103 NamePtr->setLinkage(GlobalValue::PrivateLinkage);
2104 // Collect the referenced names to be used by emitNameData.
2105 ReferencedNames.push_back(x: NamePtr);
2106}
2107
2108void InstrLowerer::emitVNodes() {
2109 if (!Opts.vp_static_alloc)
2110 return;
2111
2112 // For now only support this on platforms that do
2113 // not require runtime registration to discover
2114 // named section start/end.
2115 if (needsRuntimeRegistrationOfSectionRange(TT))
2116 return;
2117
2118 size_t TotalNS = 0;
2119 for (auto &PD : ProfileDataMap) {
2120 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
2121 TotalNS += PD.second.NumValueSites[Kind];
2122 }
2123
2124 if (!TotalNS)
2125 return;
2126
2127 uint64_t NumCounters = TotalNS * Opts.vp_counters_per_site;
2128// Heuristic for small programs with very few total value sites.
2129// The default value of vp-counters-per-site is chosen based on
2130// the observation that large apps usually have a low percentage
2131// of value sites that actually have any profile data, and thus
2132// the average number of counters per site is low. For small
2133// apps with very few sites, this may not be true. Bump up the
2134// number of counters in this case.
2135#define INSTR_PROF_MIN_VAL_COUNTS 10
2136 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
2137 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, b: (int)NumCounters * 2);
2138
2139 auto &Ctx = M.getContext();
2140 Type *VNodeTypes[] = {
2141#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
2142#include "llvm/ProfileData/InstrProfData.inc"
2143 };
2144 auto *VNodeTy = StructType::get(Context&: Ctx, Elements: ArrayRef(VNodeTypes));
2145
2146 ArrayType *VNodesTy = ArrayType::get(ElementType: VNodeTy, NumElements: NumCounters);
2147 auto *VNodesVar = new GlobalVariable(
2148 M, VNodesTy, false, GlobalValue::PrivateLinkage,
2149 Constant::getNullValue(Ty: VNodesTy), getInstrProfVNodesVarName());
2150 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *VNodesVar);
2151 VNodesVar->setSection(
2152 getInstrProfSectionName(IPSK: IPSK_vnodes, OF: TT.getObjectFormat()));
2153 VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(Ty: VNodesTy));
2154 // VNodesVar is used by runtime but not referenced via relocation by other
2155 // sections. Conservatively make it linker retained.
2156 UsedVars.push_back(x: VNodesVar);
2157}
2158
2159// Build the per-TU device-PGO sections struct: section start/stop bounds for
2160// names/counters/data/uniform-counters plus the raw version. Returns null if it
2161// already exists.
2162static GlobalVariable *emitGPUOffloadSectionsStruct(Module &M,
2163 StringRef CUIDPostfix) {
2164 std::string Name = ("__llvm_profile_sections" + CUIDPostfix).str();
2165 if (M.getNamedValue(Name))
2166 return nullptr;
2167
2168 LLVMContext &Ctx = M.getContext();
2169 unsigned AS = M.getDataLayout().getDefaultGlobalsAddressSpace();
2170 auto Extern = [&](StringRef Sym, Type *Ty, bool IsConst,
2171 GlobalValue::VisibilityTypes Vis) {
2172 GlobalVariable *GV = M.getNamedGlobal(Name: Sym);
2173 if (!GV) {
2174 GV = new GlobalVariable(M, Ty, IsConst, GlobalValue::ExternalLinkage,
2175 nullptr, Sym, nullptr,
2176 GlobalValue::NotThreadLocal, AS);
2177 GV->setVisibility(Vis);
2178 }
2179 return GV;
2180 };
2181 // Section bounds are hidden i8 markers; raw_version is an i64 constant.
2182 auto *I8 = Type::getInt8Ty(C&: Ctx);
2183 auto Hidden = GlobalValue::HiddenVisibility;
2184 Constant *Fields[] = {Extern("__start___llvm_prf_names", I8, false, Hidden),
2185 Extern("__stop___llvm_prf_names", I8, false, Hidden),
2186 Extern("__start___llvm_prf_cnts", I8, false, Hidden),
2187 Extern("__stop___llvm_prf_cnts", I8, false, Hidden),
2188 Extern("__start___llvm_prf_data", I8, false, Hidden),
2189 Extern("__stop___llvm_prf_data", I8, false, Hidden),
2190 Extern("__start___llvm_prf_ucnts", I8, false, Hidden),
2191 Extern("__stop___llvm_prf_ucnts", I8, false, Hidden),
2192 Extern("__llvm_profile_raw_version",
2193 Type::getInt64Ty(C&: Ctx), true,
2194 GlobalValue::DefaultVisibility)};
2195 auto *PtrTy = PointerType::get(C&: Ctx, AddressSpace: AS);
2196 auto *STy = StructType::get(
2197 Context&: Ctx, Elements: {PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy, PtrTy});
2198 auto *GV = new GlobalVariable(M, STy, /*isConstant=*/true,
2199 GlobalValue::ExternalLinkage,
2200 ConstantStruct::get(T: STy, V: Fields), Name, nullptr,
2201 GlobalValue::NotThreadLocal, AS);
2202 GV->setVisibility(GlobalValue::ProtectedVisibility);
2203 return GV;
2204}
2205
2206void InstrLowerer::emitNameData() {
2207 if (ReferencedNames.empty())
2208 return;
2209
2210 std::string CompressedNameStr;
2211 if (Error E = collectPGOFuncNameStrings(NameVars: ReferencedNames, Result&: CompressedNameStr,
2212 doCompression: DoInstrProfNameCompression)) {
2213 report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false);
2214 }
2215
2216 auto &Ctx = M.getContext();
2217 auto *NamesVal =
2218 ConstantDataArray::getString(Context&: Ctx, Initializer: StringRef(CompressedNameStr), AddNull: false);
2219 std::string NamesVarName = std::string(getInstrProfNamesVarName());
2220 GlobalValue::LinkageTypes NamesLinkage = GlobalValue::PrivateLinkage;
2221 GlobalValue::VisibilityTypes NamesVisibility = GlobalValue::DefaultVisibility;
2222 std::string GPUCUIDPostfix;
2223 if (isGPUProfTarget(M)) {
2224 if (auto *GV = M.getNamedGlobal(Name: getInstrProfNamesVarPostfixVarName())) {
2225 if (auto *Init =
2226 dyn_cast_or_null<ConstantDataArray>(Val: GV->getInitializer())) {
2227 if (Init->isCString()) {
2228 GPUCUIDPostfix = Init->getAsCString().str();
2229 NamesVarName += GPUCUIDPostfix;
2230 NamesLinkage = GlobalValue::ExternalLinkage;
2231 NamesVisibility = GlobalValue::ProtectedVisibility;
2232 removeFromUsedLists(
2233 M, ShouldRemove: [GV](Constant *C) { return C->stripPointerCasts() == GV; });
2234 GV->eraseFromParent();
2235 }
2236 }
2237 }
2238 }
2239 NamesVar = new GlobalVariable(M, NamesVal->getType(), true, NamesLinkage,
2240 NamesVal, NamesVarName);
2241 NamesVar->setVisibility(NamesVisibility);
2242
2243 NamesSize = CompressedNameStr.size();
2244 setGlobalVariableLargeSection(TargetTriple: TT, GV&: *NamesVar);
2245 std::string NamesSectionName =
2246 Opts.profile_correlate == ProfCorrelatorKind::Binary
2247 ? getInstrProfSectionName(IPSK: IPSK_covname, OF: TT.getObjectFormat())
2248 : getInstrProfSectionName(IPSK: IPSK_name, OF: TT.getObjectFormat());
2249 NamesVar->setSection(NamesSectionName);
2250 // On COFF, it's important to reduce the alignment down to 1 to prevent the
2251 // linker from inserting padding before the start of the names section or
2252 // between names entries.
2253 NamesVar->setAlignment(Align(1));
2254 // NamesVar is used by runtime but not referenced via relocation by other
2255 // sections. Conservatively make it linker retained.
2256 UsedVars.push_back(x: NamesVar);
2257
2258 for (auto *NamePtr : ReferencedNames)
2259 NamePtr->eraseFromParent();
2260
2261 // Emit the device sections struct only when this TU produced profile data, so
2262 // its section start/stop references are backed by a real section.
2263 bool HasData = llvm::any_of(Range&: ProfileDataMap,
2264 P: [](const auto &KV) { return KV.second.DataVar; });
2265 if (!GPUCUIDPostfix.empty() && HasData)
2266 if (GlobalVariable *GV = emitGPUOffloadSectionsStruct(M, CUIDPostfix: GPUCUIDPostfix))
2267 CompilerUsedVars.push_back(x: GV);
2268}
2269
2270void InstrLowerer::emitVTableNames() {
2271 if (!EnableVTableValueProfiling || ReferencedVTables.empty())
2272 return;
2273
2274 // Collect the PGO names of referenced vtables and compress them.
2275 std::string CompressedVTableNames;
2276 if (Error E = collectVTableStrings(VTables: ReferencedVTables, Result&: CompressedVTableNames,
2277 doCompression: DoInstrProfNameCompression)) {
2278 report_fatal_error(reason: Twine(toString(E: std::move(E))), gen_crash_diag: false);
2279 }
2280
2281 auto &Ctx = M.getContext();
2282 auto *VTableNamesVal = ConstantDataArray::getString(
2283 Context&: Ctx, Initializer: StringRef(CompressedVTableNames), AddNull: false /* AddNull */);
2284 GlobalVariable *VTableNamesVar =
2285 new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */,
2286 GlobalValue::PrivateLinkage, VTableNamesVal,
2287 getInstrProfVTableNamesVarName());
2288 VTableNamesVar->setSection(
2289 getInstrProfSectionName(IPSK: IPSK_vname, OF: TT.getObjectFormat()));
2290 VTableNamesVar->setAlignment(Align(1));
2291 // Make VTableNames linker retained.
2292 UsedVars.push_back(x: VTableNamesVar);
2293}
2294
2295void InstrLowerer::emitRegistration() {
2296 if (!needsRuntimeRegistrationOfSectionRange(TT))
2297 return;
2298
2299 // Construct the function.
2300 auto *VoidTy = Type::getVoidTy(C&: M.getContext());
2301 auto *VoidPtrTy = PointerType::getUnqual(C&: M.getContext());
2302 auto *Int64Ty = Type::getInt64Ty(C&: M.getContext());
2303 auto *RegisterFTy = FunctionType::get(Result: VoidTy, isVarArg: false);
2304 auto *RegisterF = Function::Create(Ty: RegisterFTy, Linkage: GlobalValue::InternalLinkage,
2305 N: getInstrProfRegFuncsName(), M);
2306 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2307 if (Options.NoRedZone)
2308 RegisterF->addFnAttr(Kind: Attribute::NoRedZone);
2309
2310 auto *RuntimeRegisterTy = FunctionType::get(Result: VoidTy, Params: VoidPtrTy, isVarArg: false);
2311 auto *RuntimeRegisterF =
2312 Function::Create(Ty: RuntimeRegisterTy, Linkage: GlobalVariable::ExternalLinkage,
2313 N: getInstrProfRegFuncName(), M);
2314
2315 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: RegisterF));
2316 for (Value *Data : CompilerUsedVars)
2317 if (!isa<Function>(Val: Data))
2318 // Check for addrspace cast when profiling GPU
2319 IRB.CreateCall(Callee: RuntimeRegisterF,
2320 Args: IRB.CreatePointerBitCastOrAddrSpaceCast(V: Data, DestTy: VoidPtrTy));
2321 for (Value *Data : UsedVars)
2322 if (Data != NamesVar && !isa<Function>(Val: Data))
2323 IRB.CreateCall(Callee: RuntimeRegisterF,
2324 Args: IRB.CreatePointerBitCastOrAddrSpaceCast(V: Data, DestTy: VoidPtrTy));
2325
2326 if (NamesVar) {
2327 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
2328 auto *NamesRegisterTy =
2329 FunctionType::get(Result: VoidTy, Params: ArrayRef(ParamTypes), isVarArg: false);
2330 auto *NamesRegisterF =
2331 Function::Create(Ty: NamesRegisterTy, Linkage: GlobalVariable::ExternalLinkage,
2332 N: getInstrProfNamesRegFuncName(), M);
2333 IRB.CreateCall(Callee: NamesRegisterF, Args: {IRB.CreatePointerBitCastOrAddrSpaceCast(
2334 V: NamesVar, DestTy: VoidPtrTy),
2335 IRB.getInt64(C: NamesSize)});
2336 }
2337
2338 IRB.CreateRetVoid();
2339}
2340
2341bool InstrLowerer::emitRuntimeHook() {
2342 // GPU profiling data is read directly by the host offload runtime. We do not
2343 // need the standard runtime hook.
2344 if (TT.isGPU())
2345 return false;
2346
2347 // We expect the linker to be invoked with -u<hook_var> flag for Linux
2348 // in which case there is no need to emit the external variable.
2349 if (TT.isOSLinux() || TT.isOSAIX())
2350 return false;
2351
2352 // If the module's provided its own runtime, we don't need to do anything.
2353 if (M.getGlobalVariable(Name: getInstrProfRuntimeHookVarName()))
2354 return false;
2355
2356 // Declare an external variable that will pull in the runtime initialization.
2357 auto *Int32Ty = Type::getInt32Ty(C&: M.getContext());
2358 auto *Var =
2359 new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
2360 nullptr, getInstrProfRuntimeHookVarName());
2361 Var->setVisibility(GlobalValue::HiddenVisibility);
2362
2363 if (TT.isOSBinFormatELF() && !TT.isPS()) {
2364 // Mark the user variable as used so that it isn't stripped out.
2365 CompilerUsedVars.push_back(x: Var);
2366 } else {
2367 // Make a function that uses it.
2368 auto *User = Function::Create(Ty: FunctionType::get(Result: Int32Ty, isVarArg: false),
2369 Linkage: GlobalValue::LinkOnceODRLinkage,
2370 N: getInstrProfRuntimeHookVarUseFuncName(), M);
2371 User->addFnAttr(Kind: Attribute::NoInline);
2372 if (Options.NoRedZone)
2373 User->addFnAttr(Kind: Attribute::NoRedZone);
2374 User->setVisibility(GlobalValue::HiddenVisibility);
2375 if (TT.supportsCOMDAT())
2376 User->setComdat(M.getOrInsertComdat(Name: User->getName()));
2377 // Explicitly mark this function as cold since it is never called.
2378 User->setEntryCount(Count: 0);
2379
2380 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: User));
2381 auto *Load = IRB.CreateLoad(Ty: Int32Ty, Ptr: Var);
2382 IRB.CreateRet(V: Load);
2383
2384 // Mark the function as used so that it isn't stripped out.
2385 CompilerUsedVars.push_back(x: User);
2386 }
2387 return true;
2388}
2389
2390void InstrLowerer::emitUses() {
2391 // The metadata sections are parallel arrays. Optimizers (e.g.
2392 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
2393 // we conservatively retain all unconditionally in the compiler.
2394 //
2395 // On ELF and Mach-O, the linker can guarantee the associated sections will be
2396 // retained or discarded as a unit, so llvm.compiler.used is sufficient.
2397 // Similarly on COFF, if prof data is not referenced by code we use one comdat
2398 // and ensure this GC property as well. Otherwise, we have to conservatively
2399 // make all of the sections retained by the linker.
2400 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
2401 (TT.isOSBinFormatCOFF() && !DataReferencedByCode))
2402 appendToCompilerUsed(M, Values: CompilerUsedVars);
2403 else
2404 appendToUsed(M, Values: CompilerUsedVars);
2405
2406 // We do not add proper references from used metadata sections to NamesVar and
2407 // VNodesVar, so we have to be conservative and place them in llvm.used
2408 // regardless of the target,
2409 appendToUsed(M, Values: UsedVars);
2410}
2411
2412void InstrLowerer::emitInitialization() {
2413 // Create ProfileFileName variable. Don't don't this for the
2414 // context-sensitive instrumentation lowering: This lowering is after
2415 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
2416 // have already create the variable before LTO/ThinLTO linking.
2417 if (!IsCS)
2418 createProfileFileNameVar(M, InstrProfileOutput: Options.InstrProfileOutput);
2419 Function *RegisterF = M.getFunction(Name: getInstrProfRegFuncsName());
2420 if (!RegisterF)
2421 return;
2422
2423 // Create the initialization function.
2424 auto *VoidTy = Type::getVoidTy(C&: M.getContext());
2425 auto *F = Function::Create(Ty: FunctionType::get(Result: VoidTy, isVarArg: false),
2426 Linkage: GlobalValue::InternalLinkage,
2427 N: getInstrProfInitFuncName(), M);
2428 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2429 F->addFnAttr(Kind: Attribute::NoInline);
2430 if (Options.NoRedZone)
2431 F->addFnAttr(Kind: Attribute::NoRedZone);
2432
2433 // Add the basic block and the necessary calls.
2434 IRBuilder<> IRB(BasicBlock::Create(Context&: M.getContext(), Name: "", Parent: F));
2435 IRB.CreateCall(Callee: RegisterF, Args: {});
2436 IRB.CreateRetVoid();
2437
2438 appendToGlobalCtors(M, F, Priority: 0);
2439}
2440
2441namespace llvm {
2442// Create the variable for profile sampling.
2443void createProfileSamplingVar(Module &M) {
2444 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
2445 IntegerType *SamplingVarTy;
2446 Constant *ValueZero;
2447 const InstrumentationOptions &Opts =
2448 M.getContext().getLibraryOptions().get<InstrumentationOptions>();
2449 if (getSampledInstrumentationConfig(Opts).UseShort) {
2450 SamplingVarTy = Type::getInt16Ty(C&: M.getContext());
2451 ValueZero = Constant::getIntegerValue(Ty: SamplingVarTy, V: APInt(16, 0));
2452 } else {
2453 SamplingVarTy = Type::getInt32Ty(C&: M.getContext());
2454 ValueZero = Constant::getIntegerValue(Ty: SamplingVarTy, V: APInt(32, 0));
2455 }
2456 auto SamplingVar = new GlobalVariable(
2457 M, SamplingVarTy, false, GlobalValue::WeakAnyLinkage, ValueZero, VarName);
2458 SamplingVar->setVisibility(GlobalValue::DefaultVisibility);
2459 SamplingVar->setThreadLocal(true);
2460 Triple TT(M.getTargetTriple());
2461 if (TT.supportsCOMDAT()) {
2462 SamplingVar->setLinkage(GlobalValue::ExternalLinkage);
2463 SamplingVar->setComdat(M.getOrInsertComdat(Name: VarName));
2464 }
2465 appendToCompilerUsed(M, Values: SamplingVar);
2466}
2467} // namespace llvm
2468
2469// For GPU targets: Allocate contiguous arrays for all profile data.
2470// This solves the linker reordering problem by using ONE symbol per section
2471// type, so there's nothing for the linker to reorder.
2472StructType *InstrLowerer::getProfileDataTy() {
2473 if (ProfileDataTy)
2474 return ProfileDataTy;
2475
2476 auto &Ctx = M.getContext();
2477 auto *IntPtrTy = M.getDataLayout().getIntPtrType(C&: M.getContext());
2478 auto *Int16Ty = Type::getInt16Ty(C&: Ctx);
2479 auto *Int16ArrayTy = ArrayType::get(ElementType: Int16Ty, NumElements: IPVK_Last + 1);
2480 Type *DataTypes[] = {
2481#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
2482#include "llvm/ProfileData/InstrProfData.inc"
2483 };
2484 ProfileDataTy = StructType::get(Context&: Ctx, Elements: ArrayRef(DataTypes));
2485 return ProfileDataTy;
2486}
2487