1//===- Metadata.cpp - Implement Metadata classes --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Metadata classes.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Metadata.h"
14#include "LLVMContextImpl.h"
15#include "MetadataImpl.h"
16#include "llvm/ADT/APFloat.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/DenseSet.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/SmallString.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/StringMap.h"
27#include "llvm/ADT/StringRef.h"
28#include "llvm/ADT/Twine.h"
29#include "llvm/IR/Argument.h"
30#include "llvm/IR/BasicBlock.h"
31#include "llvm/IR/Constant.h"
32#include "llvm/IR/ConstantRange.h"
33#include "llvm/IR/ConstantRangeList.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DebugInfoMetadata.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/DebugProgramInstruction.h"
38#include "llvm/IR/Function.h"
39#include "llvm/IR/GlobalObject.h"
40#include "llvm/IR/GlobalVariable.h"
41#include "llvm/IR/Instruction.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Module.h"
45#include "llvm/IR/ProfDataUtils.h"
46#include "llvm/IR/TrackingMDRef.h"
47#include "llvm/IR/Type.h"
48#include "llvm/IR/Value.h"
49#include "llvm/Support/Casting.h"
50#include "llvm/Support/CommandLine.h"
51
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/MathExtras.h"
54#include "llvm/Support/ModRef.h"
55#include <cassert>
56#include <cstddef>
57#include <cstdint>
58#include <type_traits>
59#include <utility>
60#include <vector>
61
62using namespace llvm;
63
64namespace llvm {
65extern cl::opt<bool> ProfcheckDisableMetadataFixes;
66}
67
68MetadataAsValue::MetadataAsValue(Type *Ty, Metadata *MD)
69 : Value(Ty, MetadataAsValueVal), MD(MD) {
70 track();
71}
72
73MetadataAsValue::~MetadataAsValue() {
74 getType()->getContext().pImpl->MetadataAsValues.erase(Val: MD);
75 untrack();
76}
77
78/// Canonicalize metadata arguments to intrinsics.
79///
80/// To support bitcode upgrades (and assembly semantic sugar) for \a
81/// MetadataAsValue, we need to canonicalize certain metadata.
82///
83/// - nullptr is replaced by an empty MDNode.
84/// - An MDNode with a single null operand is replaced by an empty MDNode.
85/// - An MDNode whose only operand is a \a ConstantAsMetadata gets skipped.
86///
87/// This maintains readability of bitcode from when metadata was a type of
88/// value, and these bridges were unnecessary.
89static Metadata *canonicalizeMetadataForValue(LLVMContext &Context,
90 Metadata *MD) {
91 if (!MD)
92 // !{}
93 return MDNode::get(Context, MDs: {});
94
95 // Return early if this isn't a single-operand MDNode.
96 auto *N = dyn_cast<MDNode>(Val: MD);
97 if (!N || N->getNumOperands() != 1)
98 return MD;
99
100 if (!N->getOperand(I: 0))
101 // !{}
102 return MDNode::get(Context, MDs: {});
103
104 if (auto *C = dyn_cast<ConstantAsMetadata>(Val: N->getOperand(I: 0)))
105 // Look through the MDNode.
106 return C;
107
108 return MD;
109}
110
111MetadataAsValue *MetadataAsValue::get(LLVMContext &Context, Metadata *MD) {
112 MD = canonicalizeMetadataForValue(Context, MD);
113 auto *&Entry = Context.pImpl->MetadataAsValues[MD];
114 if (!Entry)
115 Entry = new MetadataAsValue(Type::getMetadataTy(C&: Context), MD);
116 return Entry;
117}
118
119MetadataAsValue *MetadataAsValue::getIfExists(LLVMContext &Context,
120 Metadata *MD) {
121 MD = canonicalizeMetadataForValue(Context, MD);
122 auto &Store = Context.pImpl->MetadataAsValues;
123 return Store.lookup(Val: MD);
124}
125
126void MetadataAsValue::handleChangedMetadata(Metadata *MD) {
127 LLVMContext &Context = getContext();
128 MD = canonicalizeMetadataForValue(Context, MD);
129 auto &Store = Context.pImpl->MetadataAsValues;
130
131 // Stop tracking the old metadata.
132 Store.erase(Val: this->MD);
133 untrack();
134 this->MD = nullptr;
135
136 // Start tracking MD, or RAUW if necessary.
137 auto *&Entry = Store[MD];
138 if (Entry) {
139 replaceAllUsesWith(V: Entry);
140 delete this;
141 return;
142 }
143
144 this->MD = MD;
145 track();
146 Entry = this;
147}
148
149void MetadataAsValue::track() {
150 if (MD)
151 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
152}
153
154void MetadataAsValue::untrack() {
155 if (MD)
156 MetadataTracking::untrack(MD);
157}
158
159DbgVariableRecord *DebugValueUser::getUser() {
160 return static_cast<DbgVariableRecord *>(this);
161}
162const DbgVariableRecord *DebugValueUser::getUser() const {
163 return static_cast<const DbgVariableRecord *>(this);
164}
165
166void DebugValueUser::handleChangedValue(void *Old, Metadata *New) {
167 // NOTE: We could inform the "owner" that a value has changed through
168 // getOwner, if needed.
169 auto OldMD = static_cast<Metadata **>(Old);
170 ptrdiff_t Idx = std::distance(first: &*DebugValues.begin(), last: OldMD);
171 // If replacing a ValueAsMetadata with a nullptr, replace it with a
172 // PoisonValue instead.
173 if (OldMD && isa<ValueAsMetadata>(Val: *OldMD) && !New) {
174 auto *OldVAM = cast<ValueAsMetadata>(Val: *OldMD);
175 New = ValueAsMetadata::get(V: PoisonValue::get(T: OldVAM->getValue()->getType()));
176 }
177 resetDebugValue(Idx, DebugValue: New);
178}
179
180void DebugValueUser::trackDebugValue(size_t Idx) {
181 assert(Idx < 3 && "Invalid debug value index.");
182 Metadata *&MD = DebugValues[Idx];
183 if (MD)
184 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
185}
186
187void DebugValueUser::trackDebugValues() {
188 for (Metadata *&MD : DebugValues)
189 if (MD)
190 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
191}
192
193void DebugValueUser::untrackDebugValue(size_t Idx) {
194 assert(Idx < 3 && "Invalid debug value index.");
195 Metadata *&MD = DebugValues[Idx];
196 if (MD)
197 MetadataTracking::untrack(MD);
198}
199
200void DebugValueUser::untrackDebugValues() {
201 for (Metadata *&MD : DebugValues)
202 if (MD)
203 MetadataTracking::untrack(MD);
204}
205
206void DebugValueUser::retrackDebugValues(DebugValueUser &X) {
207 assert(DebugValueUser::operator==(X) && "Expected values to match");
208 for (const auto &[MD, XMD] : zip(t&: DebugValues, u&: X.DebugValues))
209 if (XMD)
210 MetadataTracking::retrack(MD&: XMD, New&: MD);
211 X.DebugValues.fill(u: nullptr);
212}
213
214bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) {
215 assert(Ref && "Expected live reference");
216 assert((Owner || *static_cast<Metadata **>(Ref) == &MD) &&
217 "Reference without owner must be direct");
218 if (auto *R = ReplaceableMetadataImpl::getOrCreate(MD)) {
219 R->addRef(Ref, Owner);
220 return true;
221 }
222 if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(Val: &MD)) {
223 assert(!PH->Use && "Placeholders can only be used once");
224 assert(!Owner && "Unexpected callback to owner");
225 PH->Use = static_cast<Metadata **>(Ref);
226 return true;
227 }
228 return false;
229}
230
231void MetadataTracking::untrack(void *Ref, Metadata &MD) {
232 assert(Ref && "Expected live reference");
233 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD))
234 R->dropRef(Ref);
235 else if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(Val: &MD))
236 PH->Use = nullptr;
237}
238
239bool MetadataTracking::retrack(void *Ref, Metadata &MD, void *New) {
240 assert(Ref && "Expected live reference");
241 assert(New && "Expected live reference");
242 assert(Ref != New && "Expected change");
243 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) {
244 R->moveRef(Ref, New, MD);
245 return true;
246 }
247 assert(!isa<DistinctMDOperandPlaceholder>(MD) &&
248 "Unexpected move of an MDOperand");
249 assert(!isReplaceable(MD) &&
250 "Expected un-replaceable metadata, since we didn't move a reference");
251 return false;
252}
253
254bool MetadataTracking::isReplaceable(const Metadata &MD) {
255 return ReplaceableMetadataImpl::isReplaceable(MD);
256}
257
258SmallVector<Metadata *> ReplaceableMetadataImpl::getAllArgListUsers() {
259 SmallVector<std::pair<OwnerTy, uint64_t> *> MDUsersWithID;
260 for (auto Pair : UseMap) {
261 OwnerTy Owner = Pair.second.first;
262 if (Owner.isNull())
263 continue;
264 if (!isa<Metadata *>(Val: Owner))
265 continue;
266 Metadata *OwnerMD = cast<Metadata *>(Val&: Owner);
267 if (OwnerMD->getMetadataID() == Metadata::DIArgListKind)
268 MDUsersWithID.push_back(Elt: &UseMap[Pair.first]);
269 }
270 llvm::sort(C&: MDUsersWithID, Comp: [](auto UserA, auto UserB) {
271 return UserA->second < UserB->second;
272 });
273 SmallVector<Metadata *> MDUsers;
274 for (auto *UserWithID : MDUsersWithID)
275 MDUsers.push_back(Elt: cast<Metadata *>(Val&: UserWithID->first));
276 return MDUsers;
277}
278
279SmallVector<DbgVariableRecord *>
280ReplaceableMetadataImpl::getAllDbgVariableRecordUsers() {
281 SmallVector<std::pair<OwnerTy, uint64_t> *> DVRUsersWithID;
282 for (auto Pair : UseMap) {
283 OwnerTy Owner = Pair.second.first;
284 if (Owner.isNull())
285 continue;
286 if (!isa<DebugValueUser *>(Val: Owner))
287 continue;
288 DVRUsersWithID.push_back(Elt: &UseMap[Pair.first]);
289 }
290 // Order DbgVariableRecord users in reverse-creation order. Normal dbg.value
291 // users of MetadataAsValues are ordered by their UseList, i.e. reverse order
292 // of when they were added: we need to replicate that here. The structure of
293 // debug-info output depends on the ordering of intrinsics, thus we need
294 // to keep them consistent for comparisons sake.
295 llvm::sort(C&: DVRUsersWithID, Comp: [](auto UserA, auto UserB) {
296 return UserA->second > UserB->second;
297 });
298 SmallVector<DbgVariableRecord *> DVRUsers;
299 for (auto UserWithID : DVRUsersWithID)
300 DVRUsers.push_back(Elt: cast<DebugValueUser *>(Val&: UserWithID->first)->getUser());
301 return DVRUsers;
302}
303
304void ReplaceableMetadataImpl::addRef(void *Ref, OwnerTy Owner) {
305 bool WasInserted =
306 UseMap.insert(KV: std::make_pair(x&: Ref, y: std::make_pair(x&: Owner, y&: NextIndex)))
307 .second;
308 (void)WasInserted;
309 assert(WasInserted && "Expected to add a reference");
310
311 ++NextIndex;
312 assert(NextIndex != 0 && "Unexpected overflow");
313}
314
315void ReplaceableMetadataImpl::dropRef(void *Ref) {
316 bool WasErased = UseMap.erase(Val: Ref);
317 (void)WasErased;
318 assert(WasErased && "Expected to drop a reference");
319}
320
321void ReplaceableMetadataImpl::moveRef(void *Ref, void *New,
322 const Metadata &MD) {
323 auto I = UseMap.find(Val: Ref);
324 assert(I != UseMap.end() && "Expected to move a reference");
325 auto OwnerAndIndex = I->second;
326 UseMap.erase(I);
327 bool WasInserted = UseMap.insert(KV: std::make_pair(x&: New, y&: OwnerAndIndex)).second;
328 (void)WasInserted;
329 assert(WasInserted && "Expected to add a reference");
330
331 // Check that the references are direct if there's no owner.
332 (void)MD;
333 assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) &&
334 "Reference without owner must be direct");
335 assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) &&
336 "Reference without owner must be direct");
337}
338
339void ReplaceableMetadataImpl::SalvageDebugInfo(const Constant &C) {
340 if (!C.isUsedByMetadata()) {
341 return;
342 }
343
344 LLVMContext &Context = C.getType()->getContext();
345 auto &Store = Context.pImpl->ValuesAsMetadata;
346 auto I = Store.find(Val: &C);
347 ValueAsMetadata *MD = I->second;
348 using UseTy =
349 std::pair<void *, std::pair<MetadataTracking::OwnerTy, uint64_t>>;
350 // Copy out uses and update value of Constant used by debug info metadata with
351 // poison below
352 SmallVector<UseTy, 8> Uses(MD->UseMap.begin(), MD->UseMap.end());
353
354 for (const auto &Pair : Uses) {
355 MetadataTracking::OwnerTy Owner = Pair.second.first;
356 if (!Owner)
357 continue;
358 // Check for MetadataAsValue.
359 if (isa<MetadataAsValue *>(Val: Owner)) {
360 cast<MetadataAsValue *>(Val&: Owner)->handleChangedMetadata(
361 MD: ValueAsMetadata::get(V: PoisonValue::get(T: C.getType())));
362 continue;
363 }
364 if (!isa<Metadata *>(Val: Owner))
365 continue;
366 auto *OwnerMD = dyn_cast_if_present<MDNode>(Val: cast<Metadata *>(Val&: Owner));
367 if (!OwnerMD)
368 continue;
369 if (isa<DINode>(Val: OwnerMD)) {
370 OwnerMD->handleChangedOperand(
371 Ref: Pair.first, New: ValueAsMetadata::get(V: PoisonValue::get(T: C.getType())));
372 }
373 }
374}
375
376void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) {
377 if (UseMap.empty())
378 return;
379
380 // Copy out uses since UseMap will get touched below.
381 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
382 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
383 llvm::sort(C&: Uses, Comp: [](const UseTy &L, const UseTy &R) {
384 return L.second.second < R.second.second;
385 });
386 for (const auto &Pair : Uses) {
387 // Check that this Ref hasn't disappeared after RAUW (when updating a
388 // previous Ref).
389 if (!UseMap.count(Val: Pair.first))
390 continue;
391
392 OwnerTy Owner = Pair.second.first;
393 if (!Owner) {
394 // Update unowned tracking references directly.
395 Metadata *&Ref = *static_cast<Metadata **>(Pair.first);
396 Ref = MD;
397 if (MD)
398 MetadataTracking::track(MD&: Ref);
399 UseMap.erase(Val: Pair.first);
400 continue;
401 }
402
403 // Check for MetadataAsValue.
404 if (isa<MetadataAsValue *>(Val: Owner)) {
405 cast<MetadataAsValue *>(Val&: Owner)->handleChangedMetadata(MD);
406 continue;
407 }
408
409 if (auto *DVU = dyn_cast<DebugValueUser *>(Val&: Owner)) {
410 DVU->handleChangedValue(Old: Pair.first, New: MD);
411 continue;
412 }
413
414 // There's a Metadata owner -- dispatch.
415 Metadata *OwnerMD = cast<Metadata *>(Val&: Owner);
416 switch (OwnerMD->getMetadataID()) {
417#define HANDLE_METADATA_LEAF(CLASS) \
418 case Metadata::CLASS##Kind: \
419 cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \
420 continue;
421#include "llvm/IR/Metadata.def"
422 default:
423 llvm_unreachable("Invalid metadata subclass");
424 }
425 }
426 assert(UseMap.empty() && "Expected all uses to be replaced");
427}
428
429void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) {
430 if (UseMap.empty())
431 return;
432
433 if (!ResolveUsers) {
434 UseMap.clear();
435 return;
436 }
437
438 // Copy out uses since UseMap could get touched below.
439 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
440 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
441 llvm::sort(C&: Uses, Comp: [](const UseTy &L, const UseTy &R) {
442 return L.second.second < R.second.second;
443 });
444 UseMap.clear();
445 for (const auto &Pair : Uses) {
446 auto Owner = Pair.second.first;
447 if (!Owner)
448 continue;
449 if (!isa<Metadata *>(Val: Owner))
450 continue;
451
452 // Resolve MDNodes that point at this.
453 auto *OwnerMD = dyn_cast_if_present<MDNode>(Val: cast<Metadata *>(Val&: Owner));
454 if (!OwnerMD)
455 continue;
456 if (OwnerMD->isResolved())
457 continue;
458 OwnerMD->decrementUnresolvedOperandCount();
459 }
460}
461
462// Special handing of DIArgList is required in the RemoveDIs project, see
463// commentry in DIArgList::handleChangedOperand for details. Hidden behind
464// conditional compilation to avoid a compile time regression.
465ReplaceableMetadataImpl *ReplaceableMetadataImpl::getOrCreate(Metadata &MD) {
466 if (auto *N = dyn_cast<MDNode>(Val: &MD)) {
467 return !N->isResolved() || N->isAlwaysReplaceable()
468 ? N->Context.getOrCreateReplaceableUses()
469 : nullptr;
470 }
471 if (auto ArgList = dyn_cast<DIArgList>(Val: &MD))
472 return ArgList;
473 return dyn_cast<ValueAsMetadata>(Val: &MD);
474}
475
476ReplaceableMetadataImpl *ReplaceableMetadataImpl::getIfExists(Metadata &MD) {
477 if (auto *N = dyn_cast<MDNode>(Val: &MD)) {
478 return !N->isResolved() || N->isAlwaysReplaceable()
479 ? N->Context.getReplaceableUses()
480 : nullptr;
481 }
482 if (auto ArgList = dyn_cast<DIArgList>(Val: &MD))
483 return ArgList;
484 return dyn_cast<ValueAsMetadata>(Val: &MD);
485}
486
487bool ReplaceableMetadataImpl::isReplaceable(const Metadata &MD) {
488 if (auto *N = dyn_cast<MDNode>(Val: &MD))
489 return !N->isResolved() || N->isAlwaysReplaceable();
490 return isa<ValueAsMetadata>(Val: &MD) || isa<DIArgList>(Val: &MD);
491}
492
493static DISubprogram *getLocalFunctionMetadata(Value *V) {
494 assert(V && "Expected value");
495 if (auto *A = dyn_cast<Argument>(Val: V)) {
496 if (auto *Fn = A->getParent())
497 return Fn->getSubprogram();
498 return nullptr;
499 }
500
501 if (BasicBlock *BB = cast<Instruction>(Val: V)->getParent()) {
502 if (auto *Fn = BB->getParent())
503 return Fn->getSubprogram();
504 return nullptr;
505 }
506
507 return nullptr;
508}
509
510ValueAsMetadata *ValueAsMetadata::get(Value *V) {
511 assert(V && "Unexpected null Value");
512
513 auto &Context = V->getContext();
514 auto *&Entry = Context.pImpl->ValuesAsMetadata[V];
515 if (!Entry) {
516 assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) &&
517 "Expected constant or function-local value");
518 assert(!V->IsUsedByMD && "Expected this to be the only metadata use");
519 V->IsUsedByMD = true;
520 if (auto *C = dyn_cast<Constant>(Val: V))
521 Entry = new ConstantAsMetadata(C);
522 else
523 Entry = new LocalAsMetadata(V);
524 }
525
526 return Entry;
527}
528
529ValueAsMetadata *ValueAsMetadata::getIfExists(Value *V) {
530 assert(V && "Unexpected null Value");
531 return V->getContext().pImpl->ValuesAsMetadata.lookup(Val: V);
532}
533
534void ValueAsMetadata::handleDeletion(Value *V) {
535 assert(V && "Expected valid value");
536
537 auto &Store = V->getType()->getContext().pImpl->ValuesAsMetadata;
538 auto I = Store.find(Val: V);
539 if (I == Store.end())
540 return;
541
542 // Remove old entry from the map.
543 ValueAsMetadata *MD = I->second;
544 assert(MD && "Expected valid metadata");
545 assert(MD->getValue() == V && "Expected valid mapping");
546 Store.erase(I);
547
548 // Delete the metadata.
549 MD->replaceAllUsesWith(MD: nullptr);
550 delete MD;
551}
552
553void ValueAsMetadata::handleRAUW(Value *From, Value *To) {
554 assert(From && "Expected valid value");
555 assert(To && "Expected valid value");
556 assert(From != To && "Expected changed value");
557 assert(&From->getContext() == &To->getContext() && "Expected same context");
558
559 LLVMContext &Context = From->getType()->getContext();
560 auto &Store = Context.pImpl->ValuesAsMetadata;
561 auto I = Store.find(Val: From);
562 if (I == Store.end()) {
563 assert(!From->IsUsedByMD && "Expected From not to be used by metadata");
564 return;
565 }
566
567 // Remove old entry from the map.
568 assert(From->IsUsedByMD && "Expected From to be used by metadata");
569 From->IsUsedByMD = false;
570 ValueAsMetadata *MD = I->second;
571 assert(MD && "Expected valid metadata");
572 assert(MD->getValue() == From && "Expected valid mapping");
573 Store.erase(I);
574
575 if (isa<LocalAsMetadata>(Val: MD)) {
576 if (auto *C = dyn_cast<Constant>(Val: To)) {
577 // Local became a constant.
578 MD->replaceAllUsesWith(MD: ConstantAsMetadata::get(C));
579 delete MD;
580 return;
581 }
582 if (getLocalFunctionMetadata(V: From) && getLocalFunctionMetadata(V: To) &&
583 getLocalFunctionMetadata(V: From) != getLocalFunctionMetadata(V: To)) {
584 // DISubprogram changed.
585 MD->replaceAllUsesWith(MD: nullptr);
586 delete MD;
587 return;
588 }
589 } else if (!isa<Constant>(Val: To)) {
590 // Changed to function-local value.
591 MD->replaceAllUsesWith(MD: nullptr);
592 delete MD;
593 return;
594 }
595
596 auto *&Entry = Store[To];
597 if (Entry) {
598 // The target already exists.
599 MD->replaceAllUsesWith(MD: Entry);
600 delete MD;
601 return;
602 }
603
604 // Update MD in place (and update the map entry).
605 assert(!To->IsUsedByMD && "Expected this to be the only metadata use");
606 To->IsUsedByMD = true;
607 MD->V = To;
608 Entry = MD;
609}
610
611//===----------------------------------------------------------------------===//
612// MDString implementation.
613//
614
615MDString *MDString::get(LLVMContext &Context, StringRef Str) {
616 auto &Store = Context.pImpl->MDStringCache;
617 auto I = Store.try_emplace(Key: Str);
618 auto &MapEntry = I.first->getValue();
619 if (!I.second)
620 return &MapEntry;
621 MapEntry.Entry = &*I.first;
622 return &MapEntry;
623}
624
625MDString *MDString::getIfExists(LLVMContext &Context, StringRef Str) {
626 auto &Store = Context.pImpl->MDStringCache;
627 auto I = Store.find(Key: Str);
628 if (I == Store.end())
629 return nullptr;
630 return &I->getValue();
631}
632
633StringRef MDString::getString() const {
634 assert(Entry && "Expected to find string map entry");
635 return Entry->first();
636}
637
638//===----------------------------------------------------------------------===//
639// MDNode implementation.
640//
641
642// Assert that the MDNode types will not be unaligned by the objects
643// prepended to them.
644#define HANDLE_MDNODE_LEAF(CLASS) \
645 static_assert( \
646 alignof(uint64_t) >= alignof(CLASS), \
647 "Alignment is insufficient after objects prepended to " #CLASS);
648#include "llvm/IR/Metadata.def"
649
650void *MDNode::operator new(size_t Size, size_t NumOps, StorageType Storage) {
651 // uint64_t is the most aligned type we need support (ensured by static_assert
652 // above)
653 size_t AllocSize =
654 alignTo(Value: Header::getAllocSize(Storage, NumOps), Align: alignof(uint64_t));
655 char *Mem = reinterpret_cast<char *>(::operator new(AllocSize + Size));
656 Header *H = new (Mem + AllocSize - sizeof(Header)) Header(NumOps, Storage);
657 return reinterpret_cast<void *>(H + 1);
658}
659
660void MDNode::operator delete(void *N) {
661 Header *H = reinterpret_cast<Header *>(N) - 1;
662 void *Mem = H->getAllocation();
663 H->~Header();
664 ::operator delete(Mem);
665}
666
667MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage,
668 ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2)
669 : Metadata(ID, Storage), Context(Context) {
670 unsigned Op = 0;
671 for (Metadata *MD : Ops1)
672 setOperand(I: Op++, New: MD);
673 for (Metadata *MD : Ops2)
674 setOperand(I: Op++, New: MD);
675
676 if (!isUniqued())
677 return;
678
679 // Count the unresolved operands. If there are any, RAUW support will be
680 // added lazily on first reference.
681 countUnresolvedOperands();
682}
683
684TempMDNode MDNode::clone() const {
685 switch (getMetadataID()) {
686 default:
687 llvm_unreachable("Invalid MDNode subclass");
688#define HANDLE_MDNODE_LEAF(CLASS) \
689 case CLASS##Kind: \
690 return cast<CLASS>(this)->cloneImpl();
691#include "llvm/IR/Metadata.def"
692 }
693}
694
695MDNode::Header::Header(size_t NumOps, StorageType Storage) {
696 IsLarge = isLarge(NumOps);
697 IsResizable = isResizable(Storage);
698 SmallSize = getSmallSize(NumOps, IsResizable, IsLarge);
699 if (IsLarge) {
700 SmallNumOps = 0;
701 new (getLargePtr()) LargeStorageVector();
702 getLarge().resize(N: NumOps);
703 return;
704 }
705 SmallNumOps = NumOps;
706 MDOperand *O = reinterpret_cast<MDOperand *>(this) - SmallSize;
707 for (MDOperand *E = O + SmallSize; O != E;)
708 (void)new (O++) MDOperand();
709}
710
711MDNode::Header::~Header() {
712 if (IsLarge) {
713 getLarge().~LargeStorageVector();
714 return;
715 }
716 MDOperand *O = reinterpret_cast<MDOperand *>(this);
717 for (MDOperand *E = O - SmallSize; O != E; --O)
718 (O - 1)->~MDOperand();
719}
720
721void *MDNode::Header::getSmallPtr() {
722 static_assert(alignof(MDOperand) <= alignof(Header),
723 "MDOperand too strongly aligned");
724 return reinterpret_cast<char *>(const_cast<Header *>(this)) -
725 sizeof(MDOperand) * SmallSize;
726}
727
728void MDNode::Header::resize(size_t NumOps) {
729 assert(IsResizable && "Node is not resizable");
730 if (operands().size() == NumOps)
731 return;
732
733 if (IsLarge)
734 getLarge().resize(N: NumOps);
735 else if (NumOps <= SmallSize)
736 resizeSmall(NumOps);
737 else
738 resizeSmallToLarge(NumOps);
739}
740
741void MDNode::Header::resizeSmall(size_t NumOps) {
742 assert(!IsLarge && "Expected a small MDNode");
743 assert(NumOps <= SmallSize && "NumOps too large for small resize");
744
745 MutableArrayRef<MDOperand> ExistingOps = operands();
746 assert(NumOps != ExistingOps.size() && "Expected a different size");
747
748 int NumNew = (int)NumOps - (int)ExistingOps.size();
749 MDOperand *O = ExistingOps.end();
750 for (int I = 0, E = NumNew; I < E; ++I)
751 (O++)->reset();
752 for (int I = 0, E = NumNew; I > E; --I)
753 (--O)->reset();
754 SmallNumOps = NumOps;
755 assert(O == operands().end() && "Operands not (un)initialized until the end");
756}
757
758void MDNode::Header::resizeSmallToLarge(size_t NumOps) {
759 assert(!IsLarge && "Expected a small MDNode");
760 assert(NumOps > SmallSize && "Expected NumOps to be larger than allocation");
761 LargeStorageVector NewOps;
762 NewOps.resize(N: NumOps);
763 llvm::move(Range: operands(), Out: NewOps.begin());
764 resizeSmall(NumOps: 0);
765 new (getLargePtr()) LargeStorageVector(std::move(NewOps));
766 IsLarge = true;
767}
768
769static bool isOperandUnresolved(Metadata *Op) {
770 if (auto *N = dyn_cast_or_null<MDNode>(Val: Op))
771 return !N->isResolved();
772 return false;
773}
774
775void MDNode::countUnresolvedOperands() {
776 assert(getNumUnresolved() == 0 && "Expected unresolved ops to be uncounted");
777 assert(isUniqued() && "Expected this to be uniqued");
778 setNumUnresolved(count_if(Range: operands(), P: isOperandUnresolved));
779}
780
781void MDNode::makeUniqued() {
782 assert(isTemporary() && "Expected this to be temporary");
783 assert(!isResolved() && "Expected this to be unresolved");
784
785 // Enable uniquing callbacks.
786 for (auto &Op : mutable_operands())
787 Op.reset(MD: Op.get(), Owner: this);
788
789 // Make this 'uniqued'.
790 Storage = Uniqued;
791 countUnresolvedOperands();
792 if (!getNumUnresolved()) {
793 dropReplaceableUses();
794 assert(isResolved() && "Expected this to be resolved");
795 }
796
797 assert(isUniqued() && "Expected this to be uniqued");
798}
799
800void MDNode::makeDistinct() {
801 assert(isTemporary() && "Expected this to be temporary");
802 assert(!isResolved() && "Expected this to be unresolved");
803
804 // Drop RAUW support and store as a distinct node.
805 dropReplaceableUses();
806 storeDistinctInContext();
807
808 assert(isDistinct() && "Expected this to be distinct");
809 assert(isResolved() && "Expected this to be resolved");
810}
811
812void MDNode::resolve() {
813 assert(isUniqued() && "Expected this to be uniqued");
814 assert(!isResolved() && "Expected this to be unresolved");
815
816 setNumUnresolved(0);
817 dropReplaceableUses();
818
819 assert(isResolved() && "Expected this to be resolved");
820}
821
822void MDNode::dropReplaceableUses() {
823 assert(!getNumUnresolved() && "Unexpected unresolved operand");
824
825 // Drop any RAUW support.
826 if (Context.hasReplaceableUses())
827 Context.takeReplaceableUses()->resolveAllUses();
828}
829
830void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) {
831 assert(isUniqued() && "Expected this to be uniqued");
832 assert(getNumUnresolved() != 0 && "Expected unresolved operands");
833
834 // Check if an operand was resolved.
835 if (!isOperandUnresolved(Op: Old)) {
836 if (isOperandUnresolved(Op: New))
837 // An operand was un-resolved!
838 setNumUnresolved(getNumUnresolved() + 1);
839 } else if (!isOperandUnresolved(Op: New))
840 decrementUnresolvedOperandCount();
841}
842
843void MDNode::decrementUnresolvedOperandCount() {
844 assert(!isResolved() && "Expected this to be unresolved");
845 if (isTemporary())
846 return;
847
848 assert(isUniqued() && "Expected this to be uniqued");
849 setNumUnresolved(getNumUnresolved() - 1);
850 if (getNumUnresolved())
851 return;
852
853 // Last unresolved operand has just been resolved.
854 dropReplaceableUses();
855 assert(isResolved() && "Expected this to become resolved");
856}
857
858void MDNode::resolveCycles() {
859 if (isResolved())
860 return;
861
862 // Resolve this node immediately.
863 resolve();
864
865 // Resolve all operands.
866 for (const auto &Op : operands()) {
867 auto *N = dyn_cast_or_null<MDNode>(Val: Op);
868 if (!N)
869 continue;
870
871 assert(!N->isTemporary() &&
872 "Expected all forward declarations to be resolved");
873 if (!N->isResolved())
874 N->resolveCycles();
875 }
876}
877
878static bool hasSelfReference(MDNode *N) {
879 return llvm::is_contained(Range: N->operands(), Element: N);
880}
881
882MDNode *MDNode::replaceWithPermanentImpl() {
883 switch (getMetadataID()) {
884 default:
885 // If this type isn't uniquable, replace with a distinct node.
886 return replaceWithDistinctImpl();
887
888#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
889 case CLASS##Kind: \
890 break;
891#include "llvm/IR/Metadata.def"
892 }
893
894 // Even if this type is uniquable, self-references have to be distinct.
895 if (hasSelfReference(N: this))
896 return replaceWithDistinctImpl();
897 return replaceWithUniquedImpl();
898}
899
900MDNode *MDNode::replaceWithUniquedImpl() {
901 // Try to uniquify in place.
902 MDNode *UniquedNode = uniquify();
903
904 if (UniquedNode == this) {
905 makeUniqued();
906 return this;
907 }
908
909 // Collision, so RAUW instead.
910 replaceAllUsesWith(MD: UniquedNode);
911 deleteAsSubclass();
912 return UniquedNode;
913}
914
915MDNode *MDNode::replaceWithDistinctImpl() {
916 makeDistinct();
917 return this;
918}
919
920void MDTuple::recalculateHash() {
921 setHash(MDTupleInfo::KeyTy::calculateHash(N: this));
922}
923
924void MDNode::dropAllReferences() {
925 for (unsigned I = 0, E = getNumOperands(); I != E; ++I)
926 setOperand(I, New: nullptr);
927 if (Context.hasReplaceableUses()) {
928 Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false);
929 (void)Context.takeReplaceableUses();
930 }
931}
932
933void MDNode::handleChangedOperand(void *Ref, Metadata *New) {
934 unsigned Op = static_cast<MDOperand *>(Ref) - op_begin();
935 assert(Op < getNumOperands() && "Expected valid operand");
936
937 if (!isUniqued()) {
938 // This node is not uniqued. Just set the operand and be done with it.
939 setOperand(I: Op, New);
940 return;
941 }
942
943 // This node is uniqued.
944 eraseFromStore();
945
946 Metadata *Old = getOperand(I: Op);
947 setOperand(I: Op, New);
948
949 // Drop uniquing for self-reference cycles and deleted constants.
950 if (New == this || (!New && Old && isa<ConstantAsMetadata>(Val: Old))) {
951 if (!isResolved())
952 resolve();
953 storeDistinctInContext();
954 return;
955 }
956
957 // Re-unique the node.
958 auto *Uniqued = uniquify();
959 if (Uniqued == this) {
960 if (!isResolved())
961 resolveAfterOperandChange(Old, New);
962 return;
963 }
964
965 // Collision.
966 if (!isResolved()) {
967 // Still unresolved, so RAUW.
968 //
969 // First, clear out all operands to prevent any recursion (similar to
970 // dropAllReferences(), but we still need the use-list).
971 for (unsigned O = 0, E = getNumOperands(); O != E; ++O)
972 setOperand(I: O, New: nullptr);
973 if (Context.hasReplaceableUses())
974 Context.getReplaceableUses()->replaceAllUsesWith(MD: Uniqued);
975 deleteAsSubclass();
976 return;
977 }
978
979 // Store in non-uniqued form if RAUW isn't possible.
980 storeDistinctInContext();
981}
982
983void MDNode::deleteAsSubclass() {
984 switch (getMetadataID()) {
985 default:
986 llvm_unreachable("Invalid subclass of MDNode");
987#define HANDLE_MDNODE_LEAF(CLASS) \
988 case CLASS##Kind: \
989 delete cast<CLASS>(this); \
990 break;
991#include "llvm/IR/Metadata.def"
992 }
993}
994
995template <class T, class InfoT>
996static T *uniquifyImpl(T *N, DenseSet<T *, InfoT> &Store) {
997 if (T *U = getUniqued(Store, N))
998 return U;
999
1000 Store.insert(N);
1001 return N;
1002}
1003
1004template <class NodeTy> struct MDNode::HasCachedHash {
1005 template <class U>
1006 static std::true_type check(SameType<void (U::*)(unsigned), &U::setHash> *);
1007 template <class U> static std::false_type check(...);
1008
1009 static constexpr bool value = decltype(check<NodeTy>(nullptr))::value;
1010};
1011
1012MDNode *MDNode::uniquify() {
1013 assert(!hasSelfReference(this) && "Cannot uniquify a self-referencing node");
1014
1015 // Try to insert into uniquing store.
1016 switch (getMetadataID()) {
1017 default:
1018 llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
1019#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
1020 case CLASS##Kind: { \
1021 CLASS *SubclassThis = cast<CLASS>(this); \
1022 dispatchRecalculateHash(SubclassThis); \
1023 return uniquifyImpl(SubclassThis, getContext().pImpl->CLASS##s); \
1024 }
1025#include "llvm/IR/Metadata.def"
1026 }
1027}
1028
1029void MDNode::eraseFromStore() {
1030 switch (getMetadataID()) {
1031 default:
1032 llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
1033#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
1034 case CLASS##Kind: \
1035 getContext().pImpl->CLASS##s.erase(cast<CLASS>(this)); \
1036 break;
1037#include "llvm/IR/Metadata.def"
1038 }
1039}
1040
1041MDTuple *MDTuple::getImpl(LLVMContext &Context, ArrayRef<Metadata *> MDs,
1042 StorageType Storage, bool ShouldCreate) {
1043 unsigned Hash = 0;
1044 if (Storage == Uniqued) {
1045 MDTupleInfo::KeyTy Key(MDs);
1046 if (auto *N = getUniqued(Store&: Context.pImpl->MDTuples, Key))
1047 return N;
1048 if (!ShouldCreate)
1049 return nullptr;
1050 Hash = Key.getHash();
1051 } else {
1052 assert(ShouldCreate && "Expected non-uniqued nodes to always be created");
1053 }
1054
1055 return storeImpl(N: new (MDs.size(), Storage)
1056 MDTuple(Context, Storage, Hash, MDs),
1057 Storage, Store&: Context.pImpl->MDTuples);
1058}
1059
1060void MDNode::deleteTemporary(MDNode *N) {
1061 assert(N->isTemporary() && "Expected temporary node");
1062 N->replaceAllUsesWith(MD: nullptr);
1063 N->deleteAsSubclass();
1064}
1065
1066void MDNode::storeDistinctInContext() {
1067 assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses");
1068 assert(!getNumUnresolved() && "Unexpected unresolved nodes");
1069 Storage = Distinct;
1070 assert(isResolved() && "Expected this to be resolved");
1071
1072 // Reset the hash.
1073 switch (getMetadataID()) {
1074 default:
1075 llvm_unreachable("Invalid subclass of MDNode");
1076#define HANDLE_MDNODE_LEAF(CLASS) \
1077 case CLASS##Kind: { \
1078 dispatchResetHash(cast<CLASS>(this)); \
1079 break; \
1080 }
1081#include "llvm/IR/Metadata.def"
1082 }
1083
1084 getContext().pImpl->DistinctMDNodes.push_back(x: this);
1085}
1086
1087void MDNode::replaceOperandWith(unsigned I, Metadata *New) {
1088 if (getOperand(I) == New)
1089 return;
1090
1091 if (!isUniqued()) {
1092 setOperand(I, New);
1093 return;
1094 }
1095
1096 handleChangedOperand(Ref: mutable_begin() + I, New);
1097}
1098
1099void MDNode::setOperand(unsigned I, Metadata *New) {
1100 assert(I < getNumOperands());
1101 mutable_begin()[I].reset(MD: New, Owner: isUniqued() ? this : nullptr);
1102}
1103
1104/// Get a node or a self-reference that looks like it.
1105///
1106/// Special handling for finding self-references, for use by \a
1107/// MDNode::concatenate() and \a MDNode::intersect() to maintain behaviour from
1108/// when self-referencing nodes were still uniqued. If the first operand has
1109/// the same operands as \c Ops, return the first operand instead.
1110static MDNode *getOrSelfReference(LLVMContext &Context,
1111 ArrayRef<Metadata *> Ops) {
1112 if (!Ops.empty())
1113 if (MDNode *N = dyn_cast_or_null<MDNode>(Val: Ops[0]))
1114 if (N->getNumOperands() == Ops.size() && N == N->getOperand(I: 0)) {
1115 for (unsigned I = 1, E = Ops.size(); I != E; ++I)
1116 if (Ops[I] != N->getOperand(I))
1117 return MDNode::get(Context, MDs: Ops);
1118 return N;
1119 }
1120
1121 return MDNode::get(Context, MDs: Ops);
1122}
1123
1124MDNode *MDNode::concatenate(MDNode *A, MDNode *B) {
1125 if (!A)
1126 return B;
1127 if (!B)
1128 return A;
1129
1130 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end());
1131 MDs.insert(Start: B->op_begin(), End: B->op_end());
1132
1133 // FIXME: This preserves long-standing behaviour, but is it really the right
1134 // behaviour? Or was that an unintended side-effect of node uniquing?
1135 return getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1136}
1137
1138MDNode *MDNode::intersect(MDNode *A, MDNode *B) {
1139 if (!A || !B)
1140 return nullptr;
1141
1142 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end());
1143 SmallPtrSet<Metadata *, 4> BSet(B->op_begin(), B->op_end());
1144 MDs.remove_if(P: [&](Metadata *MD) { return !BSet.count(Ptr: MD); });
1145
1146 // FIXME: This preserves long-standing behaviour, but is it really the right
1147 // behaviour? Or was that an unintended side-effect of node uniquing?
1148 return getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1149}
1150
1151MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) {
1152 if (!A || !B)
1153 return nullptr;
1154
1155 // Take the intersection of domains then union the scopes
1156 // within those domains
1157 SmallPtrSet<const MDNode *, 16> ADomains;
1158 SmallPtrSet<const MDNode *, 16> IntersectDomains;
1159 SmallSetVector<Metadata *, 4> MDs;
1160 for (const MDOperand &MDOp : A->operands())
1161 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1162 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1163 ADomains.insert(Ptr: Domain);
1164
1165 for (const MDOperand &MDOp : B->operands())
1166 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1167 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1168 if (ADomains.contains(Ptr: Domain)) {
1169 IntersectDomains.insert(Ptr: Domain);
1170 MDs.insert(X: MDOp);
1171 }
1172
1173 for (const MDOperand &MDOp : A->operands())
1174 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1175 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1176 if (IntersectDomains.contains(Ptr: Domain))
1177 MDs.insert(X: MDOp);
1178
1179 return MDs.empty() ? nullptr
1180 : getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1181}
1182
1183MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) {
1184 if (!A || !B)
1185 return nullptr;
1186
1187 APFloat AVal = mdconst::extract<ConstantFP>(MD: A->getOperand(I: 0))->getValueAPF();
1188 APFloat BVal = mdconst::extract<ConstantFP>(MD: B->getOperand(I: 0))->getValueAPF();
1189 if (AVal < BVal)
1190 return A;
1191 return B;
1192}
1193
1194// Call instructions with branch weights are only used in SamplePGO as
1195// documented in
1196/// https://llvm.org/docs/BranchWeightMetadata.html#callinst).
1197MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B,
1198 const Instruction *AInstr,
1199 const Instruction *BInstr) {
1200 assert(A && B && AInstr && BInstr && "Caller should guarantee");
1201 auto &Ctx = AInstr->getContext();
1202 MDBuilder MDHelper(Ctx);
1203
1204 // LLVM IR verifier verifies !prof metadata has at least 2 operands.
1205 assert(A->getNumOperands() >= 2 && B->getNumOperands() >= 2 &&
1206 "!prof annotations should have no less than 2 operands");
1207 MDString *AMDS = dyn_cast<MDString>(Val: A->getOperand(I: 0));
1208 MDString *BMDS = dyn_cast<MDString>(Val: B->getOperand(I: 0));
1209 // LLVM IR verfier verifies first operand is MDString.
1210 assert(AMDS != nullptr && BMDS != nullptr &&
1211 "first operand should be a non-null MDString");
1212 StringRef AProfName = AMDS->getString();
1213 StringRef BProfName = BMDS->getString();
1214 if (AProfName == MDProfLabels::BranchWeights &&
1215 BProfName == MDProfLabels::BranchWeights) {
1216 ConstantInt *AInstrWeight = mdconst::dyn_extract<ConstantInt>(
1217 MD: A->getOperand(I: getBranchWeightOffset(ProfileData: A)));
1218 ConstantInt *BInstrWeight = mdconst::dyn_extract<ConstantInt>(
1219 MD: B->getOperand(I: getBranchWeightOffset(ProfileData: B)));
1220 assert(AInstrWeight && BInstrWeight && "verified by LLVM verifier");
1221 return MDNode::get(Context&: Ctx,
1222 MDs: {MDHelper.createString(Str: MDProfLabels::BranchWeights),
1223 MDHelper.createConstant(C: ConstantInt::get(
1224 Ty: Type::getInt64Ty(C&: Ctx),
1225 V: SaturatingAdd(X: AInstrWeight->getZExtValue(),
1226 Y: BInstrWeight->getZExtValue())))});
1227 }
1228 return nullptr;
1229}
1230
1231// Pass in both instructions and nodes. Instruction information (e.g.,
1232// instruction type) helps interpret profiles and make implementation clearer.
1233MDNode *MDNode::getMergedProfMetadata(MDNode *A, MDNode *B,
1234 const Instruction *AInstr,
1235 const Instruction *BInstr) {
1236 // Check that it is legal to merge prof metadata based on the opcode.
1237 auto IsLegal = [](const Instruction &I) -> bool {
1238 switch (I.getOpcode()) {
1239 case Instruction::Invoke:
1240 case Instruction::CondBr:
1241 case Instruction::Switch:
1242 case Instruction::Call:
1243 case Instruction::IndirectBr:
1244 case Instruction::Select:
1245 case Instruction::CallBr:
1246 return true;
1247 default:
1248 return false;
1249 }
1250 };
1251 if (AInstr && !IsLegal(*AInstr))
1252 return nullptr;
1253 if (BInstr && !IsLegal(*BInstr))
1254 return nullptr;
1255
1256 if (!(A && B)) {
1257 return A ? A : B;
1258 }
1259
1260 assert(AInstr->getMetadata(LLVMContext::MD_prof) == A &&
1261 "Caller should guarantee");
1262 assert(BInstr->getMetadata(LLVMContext::MD_prof) == B &&
1263 "Caller should guarantee");
1264
1265 const CallInst *ACall = dyn_cast<CallInst>(Val: AInstr);
1266 const CallInst *BCall = dyn_cast<CallInst>(Val: BInstr);
1267
1268 // Both ACall and BCall are direct callsites.
1269 if (ACall && BCall && ACall->getCalledFunction() &&
1270 BCall->getCalledFunction())
1271 return mergeDirectCallProfMetadata(A, B, AInstr, BInstr);
1272
1273 if (A == B && !ProfcheckDisableMetadataFixes)
1274 return A;
1275
1276 // The rest of the cases are not implemented but could be added
1277 // when there are use cases.
1278 return nullptr;
1279}
1280
1281static bool isContiguous(const ConstantRange &A, const ConstantRange &B) {
1282 return A.getUpper() == B.getLower() || A.getLower() == B.getUpper();
1283}
1284
1285static bool canBeMerged(const ConstantRange &A, const ConstantRange &B) {
1286 return !A.intersectWith(CR: B).isEmptySet() || isContiguous(A, B);
1287}
1288
1289static bool tryMergeRange(SmallVectorImpl<ConstantInt *> &EndPoints,
1290 ConstantInt *Low, ConstantInt *High) {
1291 ConstantRange NewRange(Low->getValue(), High->getValue());
1292 unsigned Size = EndPoints.size();
1293 const APInt &LB = EndPoints[Size - 2]->getValue();
1294 const APInt &LE = EndPoints[Size - 1]->getValue();
1295 ConstantRange LastRange(LB, LE);
1296 if (canBeMerged(A: NewRange, B: LastRange)) {
1297 ConstantRange Union = LastRange.unionWith(CR: NewRange);
1298 Type *Ty = High->getType();
1299 EndPoints[Size - 2] =
1300 cast<ConstantInt>(Val: ConstantInt::get(Ty, V: Union.getLower()));
1301 EndPoints[Size - 1] =
1302 cast<ConstantInt>(Val: ConstantInt::get(Ty, V: Union.getUpper()));
1303 return true;
1304 }
1305 return false;
1306}
1307
1308static void addRange(SmallVectorImpl<ConstantInt *> &EndPoints,
1309 ConstantInt *Low, ConstantInt *High) {
1310 if (!EndPoints.empty())
1311 if (tryMergeRange(EndPoints, Low, High))
1312 return;
1313
1314 EndPoints.push_back(Elt: Low);
1315 EndPoints.push_back(Elt: High);
1316}
1317
1318MDNode *MDNode::getMergedCalleeTypeMetadata(const MDNode *A, const MDNode *B) {
1319 // Drop the callee_type metadata if either of the call instructions do not
1320 // have it.
1321 if (!A || !B)
1322 return nullptr;
1323 SmallVector<Metadata *, 8> AB;
1324 SmallPtrSet<Metadata *, 8> MergedCallees;
1325 auto AddUniqueCallees = [&AB, &MergedCallees](const MDNode *N) {
1326 for (Metadata *MD : N->operands()) {
1327 if (MergedCallees.insert(Ptr: MD).second)
1328 AB.push_back(Elt: MD);
1329 }
1330 };
1331 AddUniqueCallees(A);
1332 AddUniqueCallees(B);
1333 return MDNode::get(Context&: A->getContext(), MDs: AB);
1334}
1335
1336MDNode *MDNode::getMergedAllocTokenMetadata(const MDNode *A, const MDNode *B) {
1337 // Drop !alloc_token metadata if either instruction lacks it to avoid mis-
1338 // classifying unclassified allocations, where the fallback token must be
1339 // used instead.
1340 if (!A || !B)
1341 return nullptr;
1342 if (A == B)
1343 return const_cast<MDNode *>(A);
1344 if (A->getNumOperands() != 2 || B->getNumOperands() != 2)
1345 return nullptr;
1346 auto *CIA = mdconst::dyn_extract_or_null<ConstantInt>(MD: A->getOperand(I: 1));
1347 auto *CIB = mdconst::dyn_extract_or_null<ConstantInt>(MD: B->getOperand(I: 1));
1348 if (!CIA || !CIB)
1349 return nullptr;
1350
1351 MDString *NameA = dyn_cast<MDString>(Val: A->getOperand(I: 0));
1352 MDString *NameB = dyn_cast<MDString>(Val: B->getOperand(I: 0));
1353 if (!NameA || !NameB)
1354 return nullptr;
1355
1356 if (NameA == NameB)
1357 return CIA->isOne() ? const_cast<MDNode *>(A) : const_cast<MDNode *>(B);
1358
1359 LLVMContext &Ctx = A->getContext();
1360 StringRef StrA = NameA->getString();
1361 StringRef StrB = NameB->getString();
1362
1363 SmallString<64> Buffer;
1364 Buffer.reserve(N: StrA.size() + 1 + StrB.size());
1365 Buffer.append(RHS: StrA);
1366 Buffer.push_back(Elt: '|');
1367 Buffer.append(RHS: StrB);
1368
1369 bool MergedContainsPointer = CIA->isOne() || CIB->isOne();
1370 Metadata *Ops[] = {MDString::get(Context&: Ctx, Str: Buffer),
1371 ConstantAsMetadata::get(C: ConstantInt::get(
1372 Ty: Type::getInt1Ty(C&: Ctx), V: MergedContainsPointer))};
1373 return MDNode::get(Context&: Ctx, MDs: Ops);
1374}
1375
1376MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) {
1377 // Given two ranges, we want to compute the union of the ranges. This
1378 // is slightly complicated by having to combine the intervals and merge
1379 // the ones that overlap.
1380
1381 if (!A || !B)
1382 return nullptr;
1383
1384 if (A == B)
1385 return A;
1386
1387 // First, walk both lists in order of the lower boundary of each interval.
1388 // At each step, try to merge the new interval to the last one we added.
1389 SmallVector<ConstantInt *, 4> EndPoints;
1390 unsigned AI = 0;
1391 unsigned BI = 0;
1392 unsigned AN = A->getNumOperands() / 2;
1393 unsigned BN = B->getNumOperands() / 2;
1394 while (AI < AN && BI < BN) {
1395 ConstantInt *ALow = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI));
1396 ConstantInt *BLow = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI));
1397
1398 if (ALow->getValue().slt(RHS: BLow->getValue())) {
1399 addRange(EndPoints, Low: ALow,
1400 High: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI + 1)));
1401 ++AI;
1402 } else {
1403 addRange(EndPoints, Low: BLow,
1404 High: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI + 1)));
1405 ++BI;
1406 }
1407 }
1408 while (AI < AN) {
1409 addRange(EndPoints, Low: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI)),
1410 High: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI + 1)));
1411 ++AI;
1412 }
1413 while (BI < BN) {
1414 addRange(EndPoints, Low: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI)),
1415 High: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI + 1)));
1416 ++BI;
1417 }
1418
1419 // We haven't handled wrap in the previous merge,
1420 // if we have at least 2 ranges (4 endpoints) we have to try to merge
1421 // the last and first ones.
1422 unsigned Size = EndPoints.size();
1423 if (Size > 2) {
1424 ConstantInt *FB = EndPoints[0];
1425 ConstantInt *FE = EndPoints[1];
1426 if (tryMergeRange(EndPoints, Low: FB, High: FE)) {
1427 for (unsigned i = 0; i < Size - 2; ++i) {
1428 EndPoints[i] = EndPoints[i + 2];
1429 }
1430 EndPoints.resize(N: Size - 2);
1431 }
1432 }
1433
1434 // If in the end we have a single range, it is possible that it is now the
1435 // full range. Just drop the metadata in that case.
1436 if (EndPoints.size() == 2) {
1437 ConstantRange Range(EndPoints[0]->getValue(), EndPoints[1]->getValue());
1438 if (Range.isFullSet())
1439 return nullptr;
1440 }
1441
1442 SmallVector<Metadata *, 4> MDs;
1443 MDs.reserve(N: EndPoints.size());
1444 for (auto *I : EndPoints)
1445 MDs.push_back(Elt: ConstantAsMetadata::get(C: I));
1446 return MDNode::get(Context&: A->getContext(), MDs);
1447}
1448
1449MDNode *MDNode::getMostGenericNoFPClass(MDNode *A, MDNode *B) {
1450 if (!A || !B)
1451 return nullptr;
1452
1453 if (A == B)
1454 return A;
1455
1456 ConstantInt *AVal = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 0));
1457 ConstantInt *BVal = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 0));
1458 unsigned Intersect = AVal->getZExtValue() & BVal->getZExtValue();
1459 if (Intersect == 0)
1460 return nullptr;
1461
1462 return MDNode::get(Context&: A->getContext(), MDs: ConstantAsMetadata::get(C: ConstantInt::get(
1463 Ty: AVal->getType(), V: Intersect)));
1464}
1465
1466MDNode *MDNode::getMostGenericNoaliasAddrspace(MDNode *A, MDNode *B) {
1467 if (!A || !B)
1468 return nullptr;
1469
1470 if (A == B)
1471 return A;
1472
1473 SmallVector<ConstantRange> RangeListA, RangeListB;
1474 for (unsigned I = 0, E = A->getNumOperands() / 2; I != E; ++I) {
1475 auto *LowA = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * I + 0));
1476 auto *HighA = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * I + 1));
1477 RangeListA.push_back(Elt: ConstantRange(LowA->getValue(), HighA->getValue()));
1478 }
1479
1480 for (unsigned I = 0, E = B->getNumOperands() / 2; I != E; ++I) {
1481 auto *LowB = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * I + 0));
1482 auto *HighB = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * I + 1));
1483 RangeListB.push_back(Elt: ConstantRange(LowB->getValue(), HighB->getValue()));
1484 }
1485
1486 ConstantRangeList CRLA(RangeListA);
1487 ConstantRangeList CRLB(RangeListB);
1488 ConstantRangeList Result = CRLA.intersectWith(CRL: CRLB);
1489 if (Result.empty())
1490 return nullptr;
1491
1492 SmallVector<Metadata *> MDs;
1493 for (const ConstantRange &CR : Result) {
1494 MDs.push_back(Elt: ConstantAsMetadata::get(
1495 C: ConstantInt::get(Context&: A->getContext(), V: CR.getLower())));
1496 MDs.push_back(Elt: ConstantAsMetadata::get(
1497 C: ConstantInt::get(Context&: A->getContext(), V: CR.getUpper())));
1498 }
1499
1500 return MDNode::get(Context&: A->getContext(), MDs);
1501}
1502
1503MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) {
1504 if (!A || !B)
1505 return nullptr;
1506
1507 ConstantInt *AVal = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 0));
1508 ConstantInt *BVal = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 0));
1509 if (AVal->getZExtValue() < BVal->getZExtValue())
1510 return A;
1511 return B;
1512}
1513
1514CaptureComponents MDNode::toCaptureComponents(const MDNode *MD) {
1515 if (!MD)
1516 return CaptureComponents::All;
1517
1518 CaptureComponents CC = CaptureComponents::None;
1519 for (Metadata *Op : MD->operands()) {
1520 CaptureComponents Component =
1521 StringSwitch<CaptureComponents>(cast<MDString>(Val: Op)->getString())
1522 .Case(S: "address", Value: CaptureComponents::Address)
1523 .Case(S: "address_is_null", Value: CaptureComponents::AddressIsNull)
1524 .Case(S: "provenance", Value: CaptureComponents::Provenance)
1525 .Case(S: "read_provenance", Value: CaptureComponents::ReadProvenance);
1526 CC |= Component;
1527 }
1528 return CC;
1529}
1530
1531MDNode *MDNode::fromCaptureComponents(LLVMContext &Ctx, CaptureComponents CC) {
1532 assert(!capturesNothing(CC) && "Can't encode captures(none)");
1533 if (capturesAll(CC))
1534 return nullptr;
1535
1536 SmallVector<Metadata *> Components;
1537 if (capturesAddressIsNullOnly(CC))
1538 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "address_is_null"));
1539 else if (capturesAddress(CC))
1540 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "address"));
1541 if (capturesReadProvenanceOnly(CC))
1542 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "read_provenance"));
1543 else if (capturesFullProvenance(CC))
1544 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "provenance"));
1545 return MDNode::get(Context&: Ctx, MDs: Components);
1546}
1547
1548//===----------------------------------------------------------------------===//
1549// NamedMDNode implementation.
1550//
1551
1552static SmallVector<TrackingMDRef, 4> &getNMDOps(void *Operands) {
1553 return *(SmallVector<TrackingMDRef, 4> *)Operands;
1554}
1555
1556NamedMDNode::NamedMDNode(const Twine &N)
1557 : Name(N.str()), Operands(new SmallVector<TrackingMDRef, 4>()) {}
1558
1559NamedMDNode::~NamedMDNode() {
1560 dropAllReferences();
1561 delete &getNMDOps(Operands);
1562}
1563
1564unsigned NamedMDNode::getNumOperands() const {
1565 return (unsigned)getNMDOps(Operands).size();
1566}
1567
1568MDNode *NamedMDNode::getOperand(unsigned i) const {
1569 assert(i < getNumOperands() && "Invalid Operand number!");
1570 auto *N = getNMDOps(Operands)[i].get();
1571 return cast_or_null<MDNode>(Val: N);
1572}
1573
1574void NamedMDNode::addOperand(MDNode *M) { getNMDOps(Operands).emplace_back(Args&: M); }
1575
1576void NamedMDNode::setOperand(unsigned I, MDNode *New) {
1577 assert(I < getNumOperands() && "Invalid operand number");
1578 getNMDOps(Operands)[I].reset(MD: New);
1579}
1580
1581void NamedMDNode::eraseFromParent() { getParent()->eraseNamedMetadata(NMD: this); }
1582
1583void NamedMDNode::clearOperands() { getNMDOps(Operands).clear(); }
1584
1585StringRef NamedMDNode::getName() const { return StringRef(Name); }
1586
1587//===----------------------------------------------------------------------===//
1588// Instruction Metadata method implementations.
1589//
1590
1591unsigned &Value::getMetadataIndex() {
1592 if (auto *I = dyn_cast<Instruction>(Val: this))
1593 return I->MetadataIndex;
1594 return cast<GlobalObject>(Val: this)->MetadataIndex;
1595}
1596
1597unsigned Value::getMetadataIndex() const {
1598 return const_cast<Value *>(this)->getMetadataIndex();
1599}
1600
1601MDNode *Value::getMetadata(StringRef Kind) const {
1602 unsigned KindID = getContext().getMDKindID(Name: Kind);
1603 return getMetadataImpl(KindID);
1604}
1605
1606MDNode *Value::getMetadataImpl(unsigned KindID) const {
1607 const LLVMContext &Ctx = getContext();
1608 unsigned Idx = getMetadataIndex();
1609 while (Idx) {
1610 const MDAttachment &A = Ctx.pImpl->Metadatas[Idx];
1611 if (A.MDKind == KindID)
1612 return A.Node;
1613 Idx = A.Next;
1614 }
1615 return nullptr;
1616}
1617
1618void GlobalObject::getMetadata(unsigned KindID,
1619 SmallVectorImpl<MDNode *> &MDs) const {
1620 const LLVMContext &Ctx = getContext();
1621 unsigned Idx = MetadataIndex;
1622 while (Idx) {
1623 const MDAttachment &A = Ctx.pImpl->Metadatas[Idx];
1624 if (A.MDKind == KindID)
1625 MDs.push_back(Elt: A.Node);
1626 Idx = A.Next;
1627 }
1628 // We store metadata in reverse order, so reverse for output.
1629 std::reverse(first: MDs.begin(), last: MDs.end());
1630}
1631
1632void GlobalObject::getMetadata(StringRef Kind,
1633 SmallVectorImpl<MDNode *> &MDs) const {
1634 getMetadata(KindID: getContext().getMDKindID(Name: Kind), MDs);
1635}
1636
1637void Value::getAllMetadata(
1638 SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const {
1639 const LLVMContext &Ctx = getContext();
1640 unsigned Idx = getMetadataIndex();
1641 while (Idx) {
1642 const MDAttachment &A = Ctx.pImpl->Metadatas[Idx];
1643 MDs.emplace_back(Args: A.MDKind, Args: A.Node);
1644 Idx = A.Next;
1645 }
1646 // We store metadata in reverse order, so reverse for output in insertion
1647 // order. Sort by metadata ID for stable output.
1648 if (MDs.size() > 1) {
1649 std::reverse(first: MDs.begin(), last: MDs.end());
1650 llvm::stable_sort(Range&: MDs, C: less_first());
1651 }
1652}
1653
1654void Value::setMetadata(unsigned KindID, MDNode *Node) {
1655 assert(isa<Instruction>(this) || isa<GlobalObject>(this));
1656
1657 if (getMetadataIndex() != 0)
1658 eraseMetadata(KindID);
1659 if (Node)
1660 addMetadata(KindID, MD&: *Node);
1661}
1662
1663void Value::setMetadata(StringRef Kind, MDNode *Node) {
1664 if (!Node && getMetadataIndex() == 0)
1665 return;
1666 setMetadata(KindID: getContext().getMDKindID(Name: Kind), Node);
1667}
1668
1669void Value::addMetadata(unsigned KindID, MDNode &MD) {
1670 const LLVMContext &Ctx = getContext();
1671 unsigned &Idx = getMetadataIndex();
1672 unsigned NewIdx = Ctx.pImpl->MetadataRecycleHead;
1673 if (NewIdx == 0) {
1674 NewIdx = Ctx.pImpl->Metadatas.size();
1675 if (NewIdx == 0)
1676 NewIdx = 1;
1677 Ctx.pImpl->Metadatas.resize(N: NewIdx + 1);
1678 } else {
1679 Ctx.pImpl->MetadataRecycleHead = Ctx.pImpl->Metadatas[NewIdx].Next;
1680#ifndef NDEBUG
1681 Ctx.pImpl->MetadataRecycleSize -= 1;
1682#endif
1683 }
1684 Ctx.pImpl->Metadatas[NewIdx] =
1685 MDAttachment{.Next: Idx, .MDKind: KindID, .Node: TrackingMDNodeRef(&MD)};
1686 Idx = NewIdx;
1687}
1688
1689void Value::addMetadata(StringRef Kind, MDNode &MD) {
1690 addMetadata(KindID: getContext().getMDKindID(Name: Kind), MD);
1691}
1692
1693bool Value::eraseMetadata(unsigned KindID) {
1694 bool Changed = false;
1695 eraseMetadataIf(Pred: [&Changed, KindID](unsigned MDKind, MDNode *) {
1696 Changed |= MDKind == KindID;
1697 return MDKind == KindID;
1698 });
1699 return Changed;
1700}
1701
1702void Value::eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred) {
1703 unsigned *Idx = &getMetadataIndex();
1704 const LLVMContext &Ctx = getContext();
1705 while (*Idx) {
1706 MDAttachment &A = Ctx.pImpl->Metadatas[*Idx];
1707 if (Pred(A.MDKind, A.Node)) {
1708 A.Node.reset();
1709 unsigned FreeIdx = *Idx;
1710 *Idx = A.Next;
1711 A.Next = Ctx.pImpl->MetadataRecycleHead;
1712 Ctx.pImpl->MetadataRecycleHead = FreeIdx;
1713#ifndef NDEBUG
1714 Ctx.pImpl->MetadataRecycleSize += 1;
1715#endif
1716 } else {
1717 Idx = &A.Next;
1718 }
1719 }
1720}
1721
1722void Value::clearMetadata() {
1723 eraseMetadataIf(Pred: [](unsigned, MDNode *) { return true; });
1724}
1725
1726void Instruction::setMetadata(StringRef Kind, MDNode *Node) {
1727 if (!Node && MetadataIndex == 0)
1728 return;
1729 setMetadata(KindID: getContext().getMDKindID(Name: Kind), Node);
1730}
1731
1732MDNode *Instruction::getMetadataImpl(StringRef Kind) const {
1733 const LLVMContext &Ctx = getContext();
1734 unsigned KindID = Ctx.getMDKindID(Name: Kind);
1735 if (KindID == LLVMContext::MD_dbg)
1736 return DbgLoc.getAsMDNode();
1737 return Value::getMetadataImpl(KindID);
1738}
1739
1740void Instruction::eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred) {
1741 if (DbgLoc && Pred(LLVMContext::MD_dbg, DbgLoc.getAsMDNode()))
1742 DbgLoc = {};
1743
1744 Value::eraseMetadataIf(Pred);
1745}
1746
1747void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) {
1748 if (!hasMetadataOtherThanDebugLoc())
1749 return; // Nothing to remove!
1750
1751 SmallSet<unsigned, 32> KnownSet(llvm::from_range, KnownIDs);
1752
1753 // A DIAssignID attachment is debug metadata, don't drop it.
1754 KnownSet.insert(V: LLVMContext::MD_DIAssignID);
1755
1756 Value::eraseMetadataIf(Pred: [&KnownSet](unsigned MDKind, MDNode *Node) {
1757 return !KnownSet.count(V: MDKind);
1758 });
1759}
1760
1761void Instruction::updateDIAssignIDMapping(DIAssignID *ID) {
1762 auto &IDToInstrs = getContext().pImpl->AssignmentIDToInstrs;
1763 if (const DIAssignID *CurrentID =
1764 cast_or_null<DIAssignID>(Val: getMetadata(KindID: LLVMContext::MD_DIAssignID))) {
1765 // Nothing to do if the ID isn't changing.
1766 if (ID == CurrentID)
1767 return;
1768
1769 // Unmap this instruction from its current ID.
1770 auto InstrsIt = IDToInstrs.find(Val: CurrentID);
1771 assert(InstrsIt != IDToInstrs.end() &&
1772 "Expect existing attachment to be mapped");
1773
1774 auto &InstVec = InstrsIt->second;
1775 auto *InstIt = llvm::find(Range&: InstVec, Val: this);
1776 assert(InstIt != InstVec.end() &&
1777 "Expect instruction to be mapped to attachment");
1778 // The vector contains a ptr to this. If this is the only element in the
1779 // vector, remove the ID:vector entry, otherwise just remove the
1780 // instruction from the vector.
1781 if (InstVec.size() == 1)
1782 IDToInstrs.erase(I: InstrsIt);
1783 else
1784 InstVec.erase(CI: InstIt);
1785 }
1786
1787 // Map this instruction to the new ID.
1788 if (ID)
1789 IDToInstrs[ID].push_back(Elt: this);
1790}
1791
1792void Instruction::setMetadata(unsigned KindID, MDNode *Node) {
1793 if (!Node && !hasMetadata())
1794 return;
1795
1796 // Handle 'dbg' as a special case since it is not stored in the hash table.
1797 if (KindID == LLVMContext::MD_dbg) {
1798 DbgLoc = DebugLoc(cast_or_null<DILocation>(Val: Node));
1799 return;
1800 }
1801
1802 // Update DIAssignID to Instruction(s) mapping.
1803 if (KindID == LLVMContext::MD_DIAssignID) {
1804 // The DIAssignID tracking infrastructure doesn't support RAUWing temporary
1805 // nodes with DIAssignIDs. The cast_or_null below would also catch this, but
1806 // having a dedicated assert helps make this obvious.
1807 assert((!Node || !Node->isTemporary()) &&
1808 "Temporary DIAssignIDs are invalid");
1809 updateDIAssignIDMapping(ID: cast_or_null<DIAssignID>(Val: Node));
1810 }
1811
1812 Value::setMetadata(KindID, Node);
1813}
1814
1815void Instruction::addAnnotationMetadata(SmallVector<StringRef> Annotations) {
1816 SmallVector<Metadata *, 4> Names;
1817 if (auto *Existing = getMetadata(KindID: LLVMContext::MD_annotation)) {
1818 SmallSetVector<StringRef, 2> AnnotationsSet(Annotations.begin(),
1819 Annotations.end());
1820 auto *Tuple = cast<MDTuple>(Val: Existing);
1821 for (auto &N : Tuple->operands()) {
1822 if (isa<MDString>(Val: N.get())) {
1823 Names.push_back(Elt: N);
1824 continue;
1825 }
1826 auto *MDAnnotationTuple = cast<MDTuple>(Val: N);
1827 if (any_of(Range: MDAnnotationTuple->operands(), P: [&AnnotationsSet](auto &Op) {
1828 return AnnotationsSet.contains(key: cast<MDString>(Op)->getString());
1829 }))
1830 return;
1831 Names.push_back(Elt: N);
1832 }
1833 }
1834
1835 MDBuilder MDB(getContext());
1836 SmallVector<Metadata *> MDAnnotationStrings;
1837 for (StringRef Annotation : Annotations)
1838 MDAnnotationStrings.push_back(Elt: MDB.createString(Str: Annotation));
1839 MDNode *InfoTuple = MDTuple::get(Context&: getContext(), MDs: MDAnnotationStrings);
1840 Names.push_back(Elt: InfoTuple);
1841 MDNode *MD = MDTuple::get(Context&: getContext(), MDs: Names);
1842 setMetadata(KindID: LLVMContext::MD_annotation, Node: MD);
1843}
1844
1845void Instruction::addAnnotationMetadata(StringRef Name) {
1846 SmallVector<Metadata *, 4> Names;
1847 if (auto *Existing = getMetadata(KindID: LLVMContext::MD_annotation)) {
1848 auto *Tuple = cast<MDTuple>(Val: Existing);
1849 for (auto &N : Tuple->operands()) {
1850 if (isa<MDString>(Val: N.get()) &&
1851 cast<MDString>(Val: N.get())->getString() == Name)
1852 return;
1853 Names.push_back(Elt: N.get());
1854 }
1855 }
1856
1857 MDBuilder MDB(getContext());
1858 Names.push_back(Elt: MDB.createString(Str: Name));
1859 MDNode *MD = MDTuple::get(Context&: getContext(), MDs: Names);
1860 setMetadata(KindID: LLVMContext::MD_annotation, Node: MD);
1861}
1862
1863AAMDNodes Instruction::getAAMetadata() const {
1864 AAMDNodes Result;
1865 if (hasMetadataOtherThanDebugLoc()) {
1866 unsigned Idx = MetadataIndex;
1867 const auto &Metadatas = getContext().pImpl->Metadatas;
1868 while (Idx) {
1869 const MDAttachment &A = Metadatas[Idx];
1870 switch (A.MDKind) {
1871 case LLVMContext::MD_tbaa:
1872 Result.TBAA = A.Node;
1873 break;
1874 case LLVMContext::MD_tbaa_struct:
1875 Result.TBAAStruct = A.Node;
1876 break;
1877 case LLVMContext::MD_alias_scope:
1878 Result.Scope = A.Node;
1879 break;
1880 case LLVMContext::MD_noalias:
1881 Result.NoAlias = A.Node;
1882 break;
1883 case LLVMContext::MD_noalias_addrspace:
1884 Result.NoAliasAddrSpace = A.Node;
1885 break;
1886 }
1887 Idx = A.Next;
1888 }
1889 }
1890 return Result;
1891}
1892
1893void Instruction::setAAMetadata(const AAMDNodes &N) {
1894 setMetadata(KindID: LLVMContext::MD_tbaa, Node: N.TBAA);
1895 setMetadata(KindID: LLVMContext::MD_tbaa_struct, Node: N.TBAAStruct);
1896 setMetadata(KindID: LLVMContext::MD_alias_scope, Node: N.Scope);
1897 setMetadata(KindID: LLVMContext::MD_noalias, Node: N.NoAlias);
1898 setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: N.NoAliasAddrSpace);
1899}
1900
1901void Instruction::setNoSanitizeMetadata() {
1902 setMetadata(KindID: llvm::LLVMContext::MD_nosanitize,
1903 Node: llvm::MDNode::get(Context&: getContext(), MDs: {}));
1904}
1905
1906void Instruction::getAllMetadataImpl(
1907 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
1908 Result.clear();
1909
1910 // Handle 'dbg' as a special case since it is not stored in the hash table.
1911 if (DbgLoc) {
1912 Result.push_back(
1913 Elt: std::make_pair(x: (unsigned)LLVMContext::MD_dbg, y: DbgLoc.getAsMDNode()));
1914 }
1915 Value::getAllMetadata(MDs&: Result);
1916}
1917
1918bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) const {
1919 assert((getOpcode() == Instruction::CondBr ||
1920 getOpcode() == Instruction::Select ||
1921 getOpcode() == Instruction::Call ||
1922 getOpcode() == Instruction::Invoke ||
1923 getOpcode() == Instruction::IndirectBr ||
1924 getOpcode() == Instruction::Switch) &&
1925 "Looking for branch weights on something besides branch");
1926
1927 return ::extractProfTotalWeight(I: *this, TotalWeights&: TotalVal);
1928}
1929
1930void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) {
1931 SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
1932 Other->getAllMetadata(MDs);
1933 for (auto &MD : MDs) {
1934 // We need to adjust the type metadata offset.
1935 if (Offset != 0 && MD.first == LLVMContext::MD_type) {
1936 auto *OffsetConst = cast<ConstantInt>(
1937 Val: cast<ConstantAsMetadata>(Val: MD.second->getOperand(I: 0))->getValue());
1938 Metadata *TypeId = MD.second->getOperand(I: 1);
1939 auto *NewOffsetMD = ConstantAsMetadata::get(C: ConstantInt::get(
1940 Ty: OffsetConst->getType(), V: OffsetConst->getValue() + Offset));
1941 addMetadata(KindID: LLVMContext::MD_type,
1942 MD&: *MDNode::get(Context&: getContext(), MDs: {NewOffsetMD, TypeId}));
1943 continue;
1944 }
1945 // If an offset adjustment was specified we need to modify the DIExpression
1946 // to prepend the adjustment:
1947 // !DIExpression(DW_OP_plus, Offset, [original expr])
1948 auto *Attachment = MD.second;
1949 if (Offset != 0 && MD.first == LLVMContext::MD_dbg) {
1950 DIGlobalVariable *GV = dyn_cast<DIGlobalVariable>(Val: Attachment);
1951 DIExpression *E = nullptr;
1952 if (!GV) {
1953 auto *GVE = cast<DIGlobalVariableExpression>(Val: Attachment);
1954 GV = GVE->getVariable();
1955 E = GVE->getExpression();
1956 }
1957 ArrayRef<uint64_t> OrigElements;
1958 if (E)
1959 OrigElements = E->getElements();
1960 std::vector<uint64_t> Elements(OrigElements.size() + 2);
1961 Elements[0] = dwarf::DW_OP_plus_uconst;
1962 Elements[1] = Offset;
1963 llvm::copy(Range&: OrigElements, Out: Elements.begin() + 2);
1964 E = DIExpression::get(Context&: getContext(), Elements);
1965 Attachment = DIGlobalVariableExpression::get(Context&: getContext(), Variable: GV, Expression: E);
1966 }
1967 addMetadata(KindID: MD.first, MD&: *Attachment);
1968 }
1969}
1970
1971void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) {
1972 addMetadata(
1973 KindID: LLVMContext::MD_type,
1974 MD&: *MDTuple::get(Context&: getContext(),
1975 MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1976 Ty: Type::getInt64Ty(C&: getContext()), V: Offset)),
1977 TypeID}));
1978}
1979
1980void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) {
1981 // Remove any existing vcall visibility metadata first in case we are
1982 // updating.
1983 eraseMetadata(KindID: LLVMContext::MD_vcall_visibility);
1984 addMetadata(KindID: LLVMContext::MD_vcall_visibility,
1985 MD&: *MDNode::get(Context&: getContext(),
1986 MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1987 Ty: Type::getInt64Ty(C&: getContext()), V: Visibility))}));
1988}
1989
1990GlobalObject::VCallVisibility GlobalObject::getVCallVisibility() const {
1991 if (MDNode *MD = getMetadata(KindID: LLVMContext::MD_vcall_visibility)) {
1992 uint64_t Val = cast<ConstantInt>(
1993 Val: cast<ConstantAsMetadata>(Val: MD->getOperand(I: 0))->getValue())
1994 ->getZExtValue();
1995 assert(Val <= 2 && "unknown vcall visibility!");
1996 return (VCallVisibility)Val;
1997 }
1998 return VCallVisibility::VCallVisibilityPublic;
1999}
2000
2001void Function::setSubprogram(DISubprogram *SP) {
2002 setMetadata(KindID: LLVMContext::MD_dbg, Node: SP);
2003}
2004
2005DISubprogram *Function::getSubprogram() const {
2006 return cast_or_null<DISubprogram>(Val: getMetadata(KindID: LLVMContext::MD_dbg));
2007}
2008
2009bool Function::shouldEmitDebugInfoForProfiling() const {
2010 if (DISubprogram *SP = getSubprogram()) {
2011 if (DICompileUnit *CU = SP->getUnit()) {
2012 return CU->getDebugInfoForProfiling();
2013 }
2014 }
2015 return false;
2016}
2017
2018void GlobalVariable::addDebugInfo(DIGlobalVariableExpression *GV) {
2019 addMetadata(KindID: LLVMContext::MD_dbg, MD&: *GV);
2020}
2021
2022void GlobalVariable::getDebugInfo(
2023 SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const {
2024 SmallVector<MDNode *, 1> MDs;
2025 getMetadata(KindID: LLVMContext::MD_dbg, MDs);
2026 for (MDNode *MD : MDs)
2027 GVs.push_back(Elt: cast<DIGlobalVariableExpression>(Val: MD));
2028}
2029