1//===- Metadata.cpp - Implement Metadata classes --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Metadata classes.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Metadata.h"
14#include "LLVMContextImpl.h"
15#include "MetadataImpl.h"
16#include "llvm/ADT/APFloat.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/DenseSet.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/ADT/Twine.h"
28#include "llvm/IR/Argument.h"
29#include "llvm/IR/BasicBlock.h"
30#include "llvm/IR/Constant.h"
31#include "llvm/IR/ConstantRange.h"
32#include "llvm/IR/ConstantRangeList.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DebugInfoMetadata.h"
35#include "llvm/IR/DebugLoc.h"
36#include "llvm/IR/DebugProgramInstruction.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/GlobalObject.h"
39#include "llvm/IR/GlobalVariable.h"
40#include "llvm/IR/Instruction.h"
41#include "llvm/IR/LLVMContext.h"
42#include "llvm/IR/MDBuilder.h"
43#include "llvm/IR/Module.h"
44#include "llvm/IR/ProfDataUtils.h"
45#include "llvm/IR/TrackingMDRef.h"
46#include "llvm/IR/Type.h"
47#include "llvm/IR/Value.h"
48#include "llvm/Support/Casting.h"
49#include "llvm/Support/ErrorHandling.h"
50#include "llvm/Support/MathExtras.h"
51#include <cassert>
52#include <cstddef>
53#include <cstdint>
54#include <type_traits>
55#include <utility>
56#include <vector>
57
58using namespace llvm;
59
60MetadataAsValue::MetadataAsValue(Type *Ty, Metadata *MD)
61 : Value(Ty, MetadataAsValueVal), MD(MD) {
62 track();
63}
64
65MetadataAsValue::~MetadataAsValue() {
66 getType()->getContext().pImpl->MetadataAsValues.erase(Val: MD);
67 untrack();
68}
69
70/// Canonicalize metadata arguments to intrinsics.
71///
72/// To support bitcode upgrades (and assembly semantic sugar) for \a
73/// MetadataAsValue, we need to canonicalize certain metadata.
74///
75/// - nullptr is replaced by an empty MDNode.
76/// - An MDNode with a single null operand is replaced by an empty MDNode.
77/// - An MDNode whose only operand is a \a ConstantAsMetadata gets skipped.
78///
79/// This maintains readability of bitcode from when metadata was a type of
80/// value, and these bridges were unnecessary.
81static Metadata *canonicalizeMetadataForValue(LLVMContext &Context,
82 Metadata *MD) {
83 if (!MD)
84 // !{}
85 return MDNode::get(Context, MDs: {});
86
87 // Return early if this isn't a single-operand MDNode.
88 auto *N = dyn_cast<MDNode>(Val: MD);
89 if (!N || N->getNumOperands() != 1)
90 return MD;
91
92 if (!N->getOperand(I: 0))
93 // !{}
94 return MDNode::get(Context, MDs: {});
95
96 if (auto *C = dyn_cast<ConstantAsMetadata>(Val: N->getOperand(I: 0)))
97 // Look through the MDNode.
98 return C;
99
100 return MD;
101}
102
103MetadataAsValue *MetadataAsValue::get(LLVMContext &Context, Metadata *MD) {
104 MD = canonicalizeMetadataForValue(Context, MD);
105 auto *&Entry = Context.pImpl->MetadataAsValues[MD];
106 if (!Entry)
107 Entry = new MetadataAsValue(Type::getMetadataTy(C&: Context), MD);
108 return Entry;
109}
110
111MetadataAsValue *MetadataAsValue::getIfExists(LLVMContext &Context,
112 Metadata *MD) {
113 MD = canonicalizeMetadataForValue(Context, MD);
114 auto &Store = Context.pImpl->MetadataAsValues;
115 return Store.lookup(Val: MD);
116}
117
118void MetadataAsValue::handleChangedMetadata(Metadata *MD) {
119 LLVMContext &Context = getContext();
120 MD = canonicalizeMetadataForValue(Context, MD);
121 auto &Store = Context.pImpl->MetadataAsValues;
122
123 // Stop tracking the old metadata.
124 Store.erase(Val: this->MD);
125 untrack();
126 this->MD = nullptr;
127
128 // Start tracking MD, or RAUW if necessary.
129 auto *&Entry = Store[MD];
130 if (Entry) {
131 replaceAllUsesWith(V: Entry);
132 delete this;
133 return;
134 }
135
136 this->MD = MD;
137 track();
138 Entry = this;
139}
140
141void MetadataAsValue::track() {
142 if (MD)
143 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
144}
145
146void MetadataAsValue::untrack() {
147 if (MD)
148 MetadataTracking::untrack(MD);
149}
150
151DbgVariableRecord *DebugValueUser::getUser() {
152 return static_cast<DbgVariableRecord *>(this);
153}
154const DbgVariableRecord *DebugValueUser::getUser() const {
155 return static_cast<const DbgVariableRecord *>(this);
156}
157
158void DebugValueUser::handleChangedValue(void *Old, Metadata *New) {
159 // NOTE: We could inform the "owner" that a value has changed through
160 // getOwner, if needed.
161 auto OldMD = static_cast<Metadata **>(Old);
162 ptrdiff_t Idx = std::distance(first: &*DebugValues.begin(), last: OldMD);
163 // If replacing a ValueAsMetadata with a nullptr, replace it with a
164 // PoisonValue instead.
165 if (OldMD && isa<ValueAsMetadata>(Val: *OldMD) && !New) {
166 auto *OldVAM = cast<ValueAsMetadata>(Val: *OldMD);
167 New = ValueAsMetadata::get(V: PoisonValue::get(T: OldVAM->getValue()->getType()));
168 }
169 resetDebugValue(Idx, DebugValue: New);
170}
171
172void DebugValueUser::trackDebugValue(size_t Idx) {
173 assert(Idx < 3 && "Invalid debug value index.");
174 Metadata *&MD = DebugValues[Idx];
175 if (MD)
176 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
177}
178
179void DebugValueUser::trackDebugValues() {
180 for (Metadata *&MD : DebugValues)
181 if (MD)
182 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
183}
184
185void DebugValueUser::untrackDebugValue(size_t Idx) {
186 assert(Idx < 3 && "Invalid debug value index.");
187 Metadata *&MD = DebugValues[Idx];
188 if (MD)
189 MetadataTracking::untrack(MD);
190}
191
192void DebugValueUser::untrackDebugValues() {
193 for (Metadata *&MD : DebugValues)
194 if (MD)
195 MetadataTracking::untrack(MD);
196}
197
198void DebugValueUser::retrackDebugValues(DebugValueUser &X) {
199 assert(DebugValueUser::operator==(X) && "Expected values to match");
200 for (const auto &[MD, XMD] : zip(t&: DebugValues, u&: X.DebugValues))
201 if (XMD)
202 MetadataTracking::retrack(MD&: XMD, New&: MD);
203 X.DebugValues.fill(u: nullptr);
204}
205
206bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) {
207 assert(Ref && "Expected live reference");
208 assert((Owner || *static_cast<Metadata **>(Ref) == &MD) &&
209 "Reference without owner must be direct");
210 if (auto *R = ReplaceableMetadataImpl::getOrCreate(MD)) {
211 R->addRef(Ref, Owner);
212 return true;
213 }
214 if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(Val: &MD)) {
215 assert(!PH->Use && "Placeholders can only be used once");
216 assert(!Owner && "Unexpected callback to owner");
217 PH->Use = static_cast<Metadata **>(Ref);
218 return true;
219 }
220 return false;
221}
222
223void MetadataTracking::untrack(void *Ref, Metadata &MD) {
224 assert(Ref && "Expected live reference");
225 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD))
226 R->dropRef(Ref);
227 else if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(Val: &MD))
228 PH->Use = nullptr;
229}
230
231bool MetadataTracking::retrack(void *Ref, Metadata &MD, void *New) {
232 assert(Ref && "Expected live reference");
233 assert(New && "Expected live reference");
234 assert(Ref != New && "Expected change");
235 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) {
236 R->moveRef(Ref, New, MD);
237 return true;
238 }
239 assert(!isa<DistinctMDOperandPlaceholder>(MD) &&
240 "Unexpected move of an MDOperand");
241 assert(!isReplaceable(MD) &&
242 "Expected un-replaceable metadata, since we didn't move a reference");
243 return false;
244}
245
246bool MetadataTracking::isReplaceable(const Metadata &MD) {
247 return ReplaceableMetadataImpl::isReplaceable(MD);
248}
249
250SmallVector<Metadata *> ReplaceableMetadataImpl::getAllArgListUsers() {
251 SmallVector<std::pair<OwnerTy, uint64_t> *> MDUsersWithID;
252 for (auto Pair : UseMap) {
253 OwnerTy Owner = Pair.second.first;
254 if (Owner.isNull())
255 continue;
256 if (!isa<Metadata *>(Val: Owner))
257 continue;
258 Metadata *OwnerMD = cast<Metadata *>(Val&: Owner);
259 if (OwnerMD->getMetadataID() == Metadata::DIArgListKind)
260 MDUsersWithID.push_back(Elt: &UseMap[Pair.first]);
261 }
262 llvm::sort(C&: MDUsersWithID, Comp: [](auto UserA, auto UserB) {
263 return UserA->second < UserB->second;
264 });
265 SmallVector<Metadata *> MDUsers;
266 for (auto *UserWithID : MDUsersWithID)
267 MDUsers.push_back(Elt: cast<Metadata *>(Val&: UserWithID->first));
268 return MDUsers;
269}
270
271SmallVector<DbgVariableRecord *>
272ReplaceableMetadataImpl::getAllDbgVariableRecordUsers() {
273 SmallVector<std::pair<OwnerTy, uint64_t> *> DVRUsersWithID;
274 for (auto Pair : UseMap) {
275 OwnerTy Owner = Pair.second.first;
276 if (Owner.isNull())
277 continue;
278 if (!isa<DebugValueUser *>(Val: Owner))
279 continue;
280 DVRUsersWithID.push_back(Elt: &UseMap[Pair.first]);
281 }
282 // Order DbgVariableRecord users in reverse-creation order. Normal dbg.value
283 // users of MetadataAsValues are ordered by their UseList, i.e. reverse order
284 // of when they were added: we need to replicate that here. The structure of
285 // debug-info output depends on the ordering of intrinsics, thus we need
286 // to keep them consistent for comparisons sake.
287 llvm::sort(C&: DVRUsersWithID, Comp: [](auto UserA, auto UserB) {
288 return UserA->second > UserB->second;
289 });
290 SmallVector<DbgVariableRecord *> DVRUsers;
291 for (auto UserWithID : DVRUsersWithID)
292 DVRUsers.push_back(Elt: cast<DebugValueUser *>(Val&: UserWithID->first)->getUser());
293 return DVRUsers;
294}
295
296void ReplaceableMetadataImpl::addRef(void *Ref, OwnerTy Owner) {
297 bool WasInserted =
298 UseMap.insert(KV: std::make_pair(x&: Ref, y: std::make_pair(x&: Owner, y&: NextIndex)))
299 .second;
300 (void)WasInserted;
301 assert(WasInserted && "Expected to add a reference");
302
303 ++NextIndex;
304 assert(NextIndex != 0 && "Unexpected overflow");
305}
306
307void ReplaceableMetadataImpl::dropRef(void *Ref) {
308 bool WasErased = UseMap.erase(Val: Ref);
309 (void)WasErased;
310 assert(WasErased && "Expected to drop a reference");
311}
312
313void ReplaceableMetadataImpl::moveRef(void *Ref, void *New,
314 const Metadata &MD) {
315 auto I = UseMap.find(Val: Ref);
316 assert(I != UseMap.end() && "Expected to move a reference");
317 auto OwnerAndIndex = I->second;
318 UseMap.erase(I);
319 bool WasInserted = UseMap.insert(KV: std::make_pair(x&: New, y&: OwnerAndIndex)).second;
320 (void)WasInserted;
321 assert(WasInserted && "Expected to add a reference");
322
323 // Check that the references are direct if there's no owner.
324 (void)MD;
325 assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) &&
326 "Reference without owner must be direct");
327 assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) &&
328 "Reference without owner must be direct");
329}
330
331void ReplaceableMetadataImpl::SalvageDebugInfo(const Constant &C) {
332 if (!C.isUsedByMetadata()) {
333 return;
334 }
335
336 LLVMContext &Context = C.getType()->getContext();
337 auto &Store = Context.pImpl->ValuesAsMetadata;
338 auto I = Store.find(Val: &C);
339 ValueAsMetadata *MD = I->second;
340 using UseTy =
341 std::pair<void *, std::pair<MetadataTracking::OwnerTy, uint64_t>>;
342 // Copy out uses and update value of Constant used by debug info metadata with
343 // poison below
344 SmallVector<UseTy, 8> Uses(MD->UseMap.begin(), MD->UseMap.end());
345
346 for (const auto &Pair : Uses) {
347 MetadataTracking::OwnerTy Owner = Pair.second.first;
348 if (!Owner)
349 continue;
350 // Check for MetadataAsValue.
351 if (isa<MetadataAsValue *>(Val: Owner)) {
352 cast<MetadataAsValue *>(Val&: Owner)->handleChangedMetadata(
353 MD: ValueAsMetadata::get(V: PoisonValue::get(T: C.getType())));
354 continue;
355 }
356 if (!isa<Metadata *>(Val: Owner))
357 continue;
358 auto *OwnerMD = dyn_cast_if_present<MDNode>(Val: cast<Metadata *>(Val&: Owner));
359 if (!OwnerMD)
360 continue;
361 if (isa<DINode>(Val: OwnerMD)) {
362 OwnerMD->handleChangedOperand(
363 Ref: Pair.first, New: ValueAsMetadata::get(V: PoisonValue::get(T: C.getType())));
364 }
365 }
366}
367
368void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) {
369 if (UseMap.empty())
370 return;
371
372 // Copy out uses since UseMap will get touched below.
373 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
374 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
375 llvm::sort(C&: Uses, Comp: [](const UseTy &L, const UseTy &R) {
376 return L.second.second < R.second.second;
377 });
378 for (const auto &Pair : Uses) {
379 // Check that this Ref hasn't disappeared after RAUW (when updating a
380 // previous Ref).
381 if (!UseMap.count(Val: Pair.first))
382 continue;
383
384 OwnerTy Owner = Pair.second.first;
385 if (!Owner) {
386 // Update unowned tracking references directly.
387 Metadata *&Ref = *static_cast<Metadata **>(Pair.first);
388 Ref = MD;
389 if (MD)
390 MetadataTracking::track(MD&: Ref);
391 UseMap.erase(Val: Pair.first);
392 continue;
393 }
394
395 // Check for MetadataAsValue.
396 if (isa<MetadataAsValue *>(Val: Owner)) {
397 cast<MetadataAsValue *>(Val&: Owner)->handleChangedMetadata(MD);
398 continue;
399 }
400
401 if (auto *DVU = dyn_cast<DebugValueUser *>(Val&: Owner)) {
402 DVU->handleChangedValue(Old: Pair.first, New: MD);
403 continue;
404 }
405
406 // There's a Metadata owner -- dispatch.
407 Metadata *OwnerMD = cast<Metadata *>(Val&: Owner);
408 switch (OwnerMD->getMetadataID()) {
409#define HANDLE_METADATA_LEAF(CLASS) \
410 case Metadata::CLASS##Kind: \
411 cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \
412 continue;
413#include "llvm/IR/Metadata.def"
414 default:
415 llvm_unreachable("Invalid metadata subclass");
416 }
417 }
418 assert(UseMap.empty() && "Expected all uses to be replaced");
419}
420
421void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) {
422 if (UseMap.empty())
423 return;
424
425 if (!ResolveUsers) {
426 UseMap.clear();
427 return;
428 }
429
430 // Copy out uses since UseMap could get touched below.
431 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
432 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
433 llvm::sort(C&: Uses, Comp: [](const UseTy &L, const UseTy &R) {
434 return L.second.second < R.second.second;
435 });
436 UseMap.clear();
437 for (const auto &Pair : Uses) {
438 auto Owner = Pair.second.first;
439 if (!Owner)
440 continue;
441 if (!isa<Metadata *>(Val: Owner))
442 continue;
443
444 // Resolve MDNodes that point at this.
445 auto *OwnerMD = dyn_cast_if_present<MDNode>(Val: cast<Metadata *>(Val&: Owner));
446 if (!OwnerMD)
447 continue;
448 if (OwnerMD->isResolved())
449 continue;
450 OwnerMD->decrementUnresolvedOperandCount();
451 }
452}
453
454// Special handing of DIArgList is required in the RemoveDIs project, see
455// commentry in DIArgList::handleChangedOperand for details. Hidden behind
456// conditional compilation to avoid a compile time regression.
457ReplaceableMetadataImpl *ReplaceableMetadataImpl::getOrCreate(Metadata &MD) {
458 if (auto *N = dyn_cast<MDNode>(Val: &MD)) {
459 return !N->isResolved() || N->isAlwaysReplaceable()
460 ? N->Context.getOrCreateReplaceableUses()
461 : nullptr;
462 }
463 if (auto ArgList = dyn_cast<DIArgList>(Val: &MD))
464 return ArgList;
465 return dyn_cast<ValueAsMetadata>(Val: &MD);
466}
467
468ReplaceableMetadataImpl *ReplaceableMetadataImpl::getIfExists(Metadata &MD) {
469 if (auto *N = dyn_cast<MDNode>(Val: &MD)) {
470 return !N->isResolved() || N->isAlwaysReplaceable()
471 ? N->Context.getReplaceableUses()
472 : nullptr;
473 }
474 if (auto ArgList = dyn_cast<DIArgList>(Val: &MD))
475 return ArgList;
476 return dyn_cast<ValueAsMetadata>(Val: &MD);
477}
478
479bool ReplaceableMetadataImpl::isReplaceable(const Metadata &MD) {
480 if (auto *N = dyn_cast<MDNode>(Val: &MD))
481 return !N->isResolved() || N->isAlwaysReplaceable();
482 return isa<ValueAsMetadata>(Val: &MD) || isa<DIArgList>(Val: &MD);
483}
484
485static DISubprogram *getLocalFunctionMetadata(Value *V) {
486 assert(V && "Expected value");
487 if (auto *A = dyn_cast<Argument>(Val: V)) {
488 if (auto *Fn = A->getParent())
489 return Fn->getSubprogram();
490 return nullptr;
491 }
492
493 if (BasicBlock *BB = cast<Instruction>(Val: V)->getParent()) {
494 if (auto *Fn = BB->getParent())
495 return Fn->getSubprogram();
496 return nullptr;
497 }
498
499 return nullptr;
500}
501
502ValueAsMetadata *ValueAsMetadata::get(Value *V) {
503 assert(V && "Unexpected null Value");
504
505 auto &Context = V->getContext();
506 auto *&Entry = Context.pImpl->ValuesAsMetadata[V];
507 if (!Entry) {
508 assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) &&
509 "Expected constant or function-local value");
510 assert(!V->IsUsedByMD && "Expected this to be the only metadata use");
511 V->IsUsedByMD = true;
512 if (auto *C = dyn_cast<Constant>(Val: V))
513 Entry = new ConstantAsMetadata(C);
514 else
515 Entry = new LocalAsMetadata(V);
516 }
517
518 return Entry;
519}
520
521ValueAsMetadata *ValueAsMetadata::getIfExists(Value *V) {
522 assert(V && "Unexpected null Value");
523 return V->getContext().pImpl->ValuesAsMetadata.lookup(Val: V);
524}
525
526void ValueAsMetadata::handleDeletion(Value *V) {
527 assert(V && "Expected valid value");
528
529 auto &Store = V->getType()->getContext().pImpl->ValuesAsMetadata;
530 auto I = Store.find(Val: V);
531 if (I == Store.end())
532 return;
533
534 // Remove old entry from the map.
535 ValueAsMetadata *MD = I->second;
536 assert(MD && "Expected valid metadata");
537 assert(MD->getValue() == V && "Expected valid mapping");
538 Store.erase(I);
539
540 // Delete the metadata.
541 MD->replaceAllUsesWith(MD: nullptr);
542 delete MD;
543}
544
545void ValueAsMetadata::handleRAUW(Value *From, Value *To) {
546 assert(From && "Expected valid value");
547 assert(To && "Expected valid value");
548 assert(From != To && "Expected changed value");
549 assert(&From->getContext() == &To->getContext() && "Expected same context");
550
551 LLVMContext &Context = From->getType()->getContext();
552 auto &Store = Context.pImpl->ValuesAsMetadata;
553 auto I = Store.find(Val: From);
554 if (I == Store.end()) {
555 assert(!From->IsUsedByMD && "Expected From not to be used by metadata");
556 return;
557 }
558
559 // Remove old entry from the map.
560 assert(From->IsUsedByMD && "Expected From to be used by metadata");
561 From->IsUsedByMD = false;
562 ValueAsMetadata *MD = I->second;
563 assert(MD && "Expected valid metadata");
564 assert(MD->getValue() == From && "Expected valid mapping");
565 Store.erase(I);
566
567 if (isa<LocalAsMetadata>(Val: MD)) {
568 if (auto *C = dyn_cast<Constant>(Val: To)) {
569 // Local became a constant.
570 MD->replaceAllUsesWith(MD: ConstantAsMetadata::get(C));
571 delete MD;
572 return;
573 }
574 if (getLocalFunctionMetadata(V: From) && getLocalFunctionMetadata(V: To) &&
575 getLocalFunctionMetadata(V: From) != getLocalFunctionMetadata(V: To)) {
576 // DISubprogram changed.
577 MD->replaceAllUsesWith(MD: nullptr);
578 delete MD;
579 return;
580 }
581 } else if (!isa<Constant>(Val: To)) {
582 // Changed to function-local value.
583 MD->replaceAllUsesWith(MD: nullptr);
584 delete MD;
585 return;
586 }
587
588 auto *&Entry = Store[To];
589 if (Entry) {
590 // The target already exists.
591 MD->replaceAllUsesWith(MD: Entry);
592 delete MD;
593 return;
594 }
595
596 // Update MD in place (and update the map entry).
597 assert(!To->IsUsedByMD && "Expected this to be the only metadata use");
598 To->IsUsedByMD = true;
599 MD->V = To;
600 Entry = MD;
601}
602
603//===----------------------------------------------------------------------===//
604// MDString implementation.
605//
606
607MDString *MDString::get(LLVMContext &Context, StringRef Str) {
608 auto &Store = Context.pImpl->MDStringCache;
609 auto I = Store.try_emplace(Key: Str);
610 auto &MapEntry = I.first->getValue();
611 if (!I.second)
612 return &MapEntry;
613 MapEntry.Entry = &*I.first;
614 return &MapEntry;
615}
616
617StringRef MDString::getString() const {
618 assert(Entry && "Expected to find string map entry");
619 return Entry->first();
620}
621
622//===----------------------------------------------------------------------===//
623// MDNode implementation.
624//
625
626// Assert that the MDNode types will not be unaligned by the objects
627// prepended to them.
628#define HANDLE_MDNODE_LEAF(CLASS) \
629 static_assert( \
630 alignof(uint64_t) >= alignof(CLASS), \
631 "Alignment is insufficient after objects prepended to " #CLASS);
632#include "llvm/IR/Metadata.def"
633
634void *MDNode::operator new(size_t Size, size_t NumOps, StorageType Storage) {
635 // uint64_t is the most aligned type we need support (ensured by static_assert
636 // above)
637 size_t AllocSize =
638 alignTo(Value: Header::getAllocSize(Storage, NumOps), Align: alignof(uint64_t));
639 char *Mem = reinterpret_cast<char *>(::operator new(AllocSize + Size));
640 Header *H = new (Mem + AllocSize - sizeof(Header)) Header(NumOps, Storage);
641 return reinterpret_cast<void *>(H + 1);
642}
643
644void MDNode::operator delete(void *N) {
645 Header *H = reinterpret_cast<Header *>(N) - 1;
646 void *Mem = H->getAllocation();
647 H->~Header();
648 ::operator delete(Mem);
649}
650
651MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage,
652 ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2)
653 : Metadata(ID, Storage), Context(Context) {
654 unsigned Op = 0;
655 for (Metadata *MD : Ops1)
656 setOperand(I: Op++, New: MD);
657 for (Metadata *MD : Ops2)
658 setOperand(I: Op++, New: MD);
659
660 if (!isUniqued())
661 return;
662
663 // Count the unresolved operands. If there are any, RAUW support will be
664 // added lazily on first reference.
665 countUnresolvedOperands();
666}
667
668TempMDNode MDNode::clone() const {
669 switch (getMetadataID()) {
670 default:
671 llvm_unreachable("Invalid MDNode subclass");
672#define HANDLE_MDNODE_LEAF(CLASS) \
673 case CLASS##Kind: \
674 return cast<CLASS>(this)->cloneImpl();
675#include "llvm/IR/Metadata.def"
676 }
677}
678
679MDNode::Header::Header(size_t NumOps, StorageType Storage) {
680 IsLarge = isLarge(NumOps);
681 IsResizable = isResizable(Storage);
682 SmallSize = getSmallSize(NumOps, IsResizable, IsLarge);
683 if (IsLarge) {
684 SmallNumOps = 0;
685 new (getLargePtr()) LargeStorageVector();
686 getLarge().resize(N: NumOps);
687 return;
688 }
689 SmallNumOps = NumOps;
690 MDOperand *O = reinterpret_cast<MDOperand *>(this) - SmallSize;
691 for (MDOperand *E = O + SmallSize; O != E;)
692 (void)new (O++) MDOperand();
693}
694
695MDNode::Header::~Header() {
696 if (IsLarge) {
697 getLarge().~LargeStorageVector();
698 return;
699 }
700 MDOperand *O = reinterpret_cast<MDOperand *>(this);
701 for (MDOperand *E = O - SmallSize; O != E; --O)
702 (O - 1)->~MDOperand();
703}
704
705void *MDNode::Header::getSmallPtr() {
706 static_assert(alignof(MDOperand) <= alignof(Header),
707 "MDOperand too strongly aligned");
708 return reinterpret_cast<char *>(const_cast<Header *>(this)) -
709 sizeof(MDOperand) * SmallSize;
710}
711
712void MDNode::Header::resize(size_t NumOps) {
713 assert(IsResizable && "Node is not resizable");
714 if (operands().size() == NumOps)
715 return;
716
717 if (IsLarge)
718 getLarge().resize(N: NumOps);
719 else if (NumOps <= SmallSize)
720 resizeSmall(NumOps);
721 else
722 resizeSmallToLarge(NumOps);
723}
724
725void MDNode::Header::resizeSmall(size_t NumOps) {
726 assert(!IsLarge && "Expected a small MDNode");
727 assert(NumOps <= SmallSize && "NumOps too large for small resize");
728
729 MutableArrayRef<MDOperand> ExistingOps = operands();
730 assert(NumOps != ExistingOps.size() && "Expected a different size");
731
732 int NumNew = (int)NumOps - (int)ExistingOps.size();
733 MDOperand *O = ExistingOps.end();
734 for (int I = 0, E = NumNew; I < E; ++I)
735 (O++)->reset();
736 for (int I = 0, E = NumNew; I > E; --I)
737 (--O)->reset();
738 SmallNumOps = NumOps;
739 assert(O == operands().end() && "Operands not (un)initialized until the end");
740}
741
742void MDNode::Header::resizeSmallToLarge(size_t NumOps) {
743 assert(!IsLarge && "Expected a small MDNode");
744 assert(NumOps > SmallSize && "Expected NumOps to be larger than allocation");
745 LargeStorageVector NewOps;
746 NewOps.resize(N: NumOps);
747 llvm::move(Range: operands(), Out: NewOps.begin());
748 resizeSmall(NumOps: 0);
749 new (getLargePtr()) LargeStorageVector(std::move(NewOps));
750 IsLarge = true;
751}
752
753static bool isOperandUnresolved(Metadata *Op) {
754 if (auto *N = dyn_cast_or_null<MDNode>(Val: Op))
755 return !N->isResolved();
756 return false;
757}
758
759void MDNode::countUnresolvedOperands() {
760 assert(getNumUnresolved() == 0 && "Expected unresolved ops to be uncounted");
761 assert(isUniqued() && "Expected this to be uniqued");
762 setNumUnresolved(count_if(Range: operands(), P: isOperandUnresolved));
763}
764
765void MDNode::makeUniqued() {
766 assert(isTemporary() && "Expected this to be temporary");
767 assert(!isResolved() && "Expected this to be unresolved");
768
769 // Enable uniquing callbacks.
770 for (auto &Op : mutable_operands())
771 Op.reset(MD: Op.get(), Owner: this);
772
773 // Make this 'uniqued'.
774 Storage = Uniqued;
775 countUnresolvedOperands();
776 if (!getNumUnresolved()) {
777 dropReplaceableUses();
778 assert(isResolved() && "Expected this to be resolved");
779 }
780
781 assert(isUniqued() && "Expected this to be uniqued");
782}
783
784void MDNode::makeDistinct() {
785 assert(isTemporary() && "Expected this to be temporary");
786 assert(!isResolved() && "Expected this to be unresolved");
787
788 // Drop RAUW support and store as a distinct node.
789 dropReplaceableUses();
790 storeDistinctInContext();
791
792 assert(isDistinct() && "Expected this to be distinct");
793 assert(isResolved() && "Expected this to be resolved");
794}
795
796void MDNode::resolve() {
797 assert(isUniqued() && "Expected this to be uniqued");
798 assert(!isResolved() && "Expected this to be unresolved");
799
800 setNumUnresolved(0);
801 dropReplaceableUses();
802
803 assert(isResolved() && "Expected this to be resolved");
804}
805
806void MDNode::dropReplaceableUses() {
807 assert(!getNumUnresolved() && "Unexpected unresolved operand");
808
809 // Drop any RAUW support.
810 if (Context.hasReplaceableUses())
811 Context.takeReplaceableUses()->resolveAllUses();
812}
813
814void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) {
815 assert(isUniqued() && "Expected this to be uniqued");
816 assert(getNumUnresolved() != 0 && "Expected unresolved operands");
817
818 // Check if an operand was resolved.
819 if (!isOperandUnresolved(Op: Old)) {
820 if (isOperandUnresolved(Op: New))
821 // An operand was un-resolved!
822 setNumUnresolved(getNumUnresolved() + 1);
823 } else if (!isOperandUnresolved(Op: New))
824 decrementUnresolvedOperandCount();
825}
826
827void MDNode::decrementUnresolvedOperandCount() {
828 assert(!isResolved() && "Expected this to be unresolved");
829 if (isTemporary())
830 return;
831
832 assert(isUniqued() && "Expected this to be uniqued");
833 setNumUnresolved(getNumUnresolved() - 1);
834 if (getNumUnresolved())
835 return;
836
837 // Last unresolved operand has just been resolved.
838 dropReplaceableUses();
839 assert(isResolved() && "Expected this to become resolved");
840}
841
842void MDNode::resolveCycles() {
843 if (isResolved())
844 return;
845
846 // Resolve this node immediately.
847 resolve();
848
849 // Resolve all operands.
850 for (const auto &Op : operands()) {
851 auto *N = dyn_cast_or_null<MDNode>(Val: Op);
852 if (!N)
853 continue;
854
855 assert(!N->isTemporary() &&
856 "Expected all forward declarations to be resolved");
857 if (!N->isResolved())
858 N->resolveCycles();
859 }
860}
861
862static bool hasSelfReference(MDNode *N) {
863 return llvm::is_contained(Range: N->operands(), Element: N);
864}
865
866MDNode *MDNode::replaceWithPermanentImpl() {
867 switch (getMetadataID()) {
868 default:
869 // If this type isn't uniquable, replace with a distinct node.
870 return replaceWithDistinctImpl();
871
872#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
873 case CLASS##Kind: \
874 break;
875#include "llvm/IR/Metadata.def"
876 }
877
878 // Even if this type is uniquable, self-references have to be distinct.
879 if (hasSelfReference(N: this))
880 return replaceWithDistinctImpl();
881 return replaceWithUniquedImpl();
882}
883
884MDNode *MDNode::replaceWithUniquedImpl() {
885 // Try to uniquify in place.
886 MDNode *UniquedNode = uniquify();
887
888 if (UniquedNode == this) {
889 makeUniqued();
890 return this;
891 }
892
893 // Collision, so RAUW instead.
894 replaceAllUsesWith(MD: UniquedNode);
895 deleteAsSubclass();
896 return UniquedNode;
897}
898
899MDNode *MDNode::replaceWithDistinctImpl() {
900 makeDistinct();
901 return this;
902}
903
904void MDTuple::recalculateHash() {
905 setHash(MDTupleInfo::KeyTy::calculateHash(N: this));
906}
907
908void MDNode::dropAllReferences() {
909 for (unsigned I = 0, E = getNumOperands(); I != E; ++I)
910 setOperand(I, New: nullptr);
911 if (Context.hasReplaceableUses()) {
912 Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false);
913 (void)Context.takeReplaceableUses();
914 }
915}
916
917void MDNode::handleChangedOperand(void *Ref, Metadata *New) {
918 unsigned Op = static_cast<MDOperand *>(Ref) - op_begin();
919 assert(Op < getNumOperands() && "Expected valid operand");
920
921 if (!isUniqued()) {
922 // This node is not uniqued. Just set the operand and be done with it.
923 setOperand(I: Op, New);
924 return;
925 }
926
927 // This node is uniqued.
928 eraseFromStore();
929
930 Metadata *Old = getOperand(I: Op);
931 setOperand(I: Op, New);
932
933 // Drop uniquing for self-reference cycles and deleted constants.
934 if (New == this || (!New && Old && isa<ConstantAsMetadata>(Val: Old))) {
935 if (!isResolved())
936 resolve();
937 storeDistinctInContext();
938 return;
939 }
940
941 // Re-unique the node.
942 auto *Uniqued = uniquify();
943 if (Uniqued == this) {
944 if (!isResolved())
945 resolveAfterOperandChange(Old, New);
946 return;
947 }
948
949 // Collision.
950 if (!isResolved()) {
951 // Still unresolved, so RAUW.
952 //
953 // First, clear out all operands to prevent any recursion (similar to
954 // dropAllReferences(), but we still need the use-list).
955 for (unsigned O = 0, E = getNumOperands(); O != E; ++O)
956 setOperand(I: O, New: nullptr);
957 if (Context.hasReplaceableUses())
958 Context.getReplaceableUses()->replaceAllUsesWith(MD: Uniqued);
959 deleteAsSubclass();
960 return;
961 }
962
963 // Store in non-uniqued form if RAUW isn't possible.
964 storeDistinctInContext();
965}
966
967void MDNode::deleteAsSubclass() {
968 switch (getMetadataID()) {
969 default:
970 llvm_unreachable("Invalid subclass of MDNode");
971#define HANDLE_MDNODE_LEAF(CLASS) \
972 case CLASS##Kind: \
973 delete cast<CLASS>(this); \
974 break;
975#include "llvm/IR/Metadata.def"
976 }
977}
978
979template <class T, class InfoT>
980static T *uniquifyImpl(T *N, DenseSet<T *, InfoT> &Store) {
981 if (T *U = getUniqued(Store, N))
982 return U;
983
984 Store.insert(N);
985 return N;
986}
987
988template <class NodeTy> struct MDNode::HasCachedHash {
989 using Yes = char[1];
990 using No = char[2];
991 template <class U, U Val> struct SFINAE {};
992
993 template <class U>
994 static Yes &check(SFINAE<void (U::*)(unsigned), &U::setHash> *);
995 template <class U> static No &check(...);
996
997 static const bool value = sizeof(check<NodeTy>(nullptr)) == sizeof(Yes);
998};
999
1000MDNode *MDNode::uniquify() {
1001 assert(!hasSelfReference(this) && "Cannot uniquify a self-referencing node");
1002
1003 // Try to insert into uniquing store.
1004 switch (getMetadataID()) {
1005 default:
1006 llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
1007#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
1008 case CLASS##Kind: { \
1009 CLASS *SubclassThis = cast<CLASS>(this); \
1010 std::integral_constant<bool, HasCachedHash<CLASS>::value> \
1011 ShouldRecalculateHash; \
1012 dispatchRecalculateHash(SubclassThis, ShouldRecalculateHash); \
1013 return uniquifyImpl(SubclassThis, getContext().pImpl->CLASS##s); \
1014 }
1015#include "llvm/IR/Metadata.def"
1016 }
1017}
1018
1019void MDNode::eraseFromStore() {
1020 switch (getMetadataID()) {
1021 default:
1022 llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
1023#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
1024 case CLASS##Kind: \
1025 getContext().pImpl->CLASS##s.erase(cast<CLASS>(this)); \
1026 break;
1027#include "llvm/IR/Metadata.def"
1028 }
1029}
1030
1031MDTuple *MDTuple::getImpl(LLVMContext &Context, ArrayRef<Metadata *> MDs,
1032 StorageType Storage, bool ShouldCreate) {
1033 unsigned Hash = 0;
1034 if (Storage == Uniqued) {
1035 MDTupleInfo::KeyTy Key(MDs);
1036 if (auto *N = getUniqued(Store&: Context.pImpl->MDTuples, Key))
1037 return N;
1038 if (!ShouldCreate)
1039 return nullptr;
1040 Hash = Key.getHash();
1041 } else {
1042 assert(ShouldCreate && "Expected non-uniqued nodes to always be created");
1043 }
1044
1045 return storeImpl(N: new (MDs.size(), Storage)
1046 MDTuple(Context, Storage, Hash, MDs),
1047 Storage, Store&: Context.pImpl->MDTuples);
1048}
1049
1050void MDNode::deleteTemporary(MDNode *N) {
1051 assert(N->isTemporary() && "Expected temporary node");
1052 N->replaceAllUsesWith(MD: nullptr);
1053 N->deleteAsSubclass();
1054}
1055
1056void MDNode::storeDistinctInContext() {
1057 assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses");
1058 assert(!getNumUnresolved() && "Unexpected unresolved nodes");
1059 Storage = Distinct;
1060 assert(isResolved() && "Expected this to be resolved");
1061
1062 // Reset the hash.
1063 switch (getMetadataID()) {
1064 default:
1065 llvm_unreachable("Invalid subclass of MDNode");
1066#define HANDLE_MDNODE_LEAF(CLASS) \
1067 case CLASS##Kind: { \
1068 std::integral_constant<bool, HasCachedHash<CLASS>::value> ShouldResetHash; \
1069 dispatchResetHash(cast<CLASS>(this), ShouldResetHash); \
1070 break; \
1071 }
1072#include "llvm/IR/Metadata.def"
1073 }
1074
1075 getContext().pImpl->DistinctMDNodes.push_back(x: this);
1076}
1077
1078void MDNode::replaceOperandWith(unsigned I, Metadata *New) {
1079 if (getOperand(I) == New)
1080 return;
1081
1082 if (!isUniqued()) {
1083 setOperand(I, New);
1084 return;
1085 }
1086
1087 handleChangedOperand(Ref: mutable_begin() + I, New);
1088}
1089
1090void MDNode::setOperand(unsigned I, Metadata *New) {
1091 assert(I < getNumOperands());
1092 mutable_begin()[I].reset(MD: New, Owner: isUniqued() ? this : nullptr);
1093}
1094
1095/// Get a node or a self-reference that looks like it.
1096///
1097/// Special handling for finding self-references, for use by \a
1098/// MDNode::concatenate() and \a MDNode::intersect() to maintain behaviour from
1099/// when self-referencing nodes were still uniqued. If the first operand has
1100/// the same operands as \c Ops, return the first operand instead.
1101static MDNode *getOrSelfReference(LLVMContext &Context,
1102 ArrayRef<Metadata *> Ops) {
1103 if (!Ops.empty())
1104 if (MDNode *N = dyn_cast_or_null<MDNode>(Val: Ops[0]))
1105 if (N->getNumOperands() == Ops.size() && N == N->getOperand(I: 0)) {
1106 for (unsigned I = 1, E = Ops.size(); I != E; ++I)
1107 if (Ops[I] != N->getOperand(I))
1108 return MDNode::get(Context, MDs: Ops);
1109 return N;
1110 }
1111
1112 return MDNode::get(Context, MDs: Ops);
1113}
1114
1115MDNode *MDNode::concatenate(MDNode *A, MDNode *B) {
1116 if (!A)
1117 return B;
1118 if (!B)
1119 return A;
1120
1121 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end());
1122 MDs.insert(Start: B->op_begin(), End: B->op_end());
1123
1124 // FIXME: This preserves long-standing behaviour, but is it really the right
1125 // behaviour? Or was that an unintended side-effect of node uniquing?
1126 return getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1127}
1128
1129MDNode *MDNode::intersect(MDNode *A, MDNode *B) {
1130 if (!A || !B)
1131 return nullptr;
1132
1133 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end());
1134 SmallPtrSet<Metadata *, 4> BSet(B->op_begin(), B->op_end());
1135 MDs.remove_if(P: [&](Metadata *MD) { return !BSet.count(Ptr: MD); });
1136
1137 // FIXME: This preserves long-standing behaviour, but is it really the right
1138 // behaviour? Or was that an unintended side-effect of node uniquing?
1139 return getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1140}
1141
1142MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) {
1143 if (!A || !B)
1144 return nullptr;
1145
1146 // Take the intersection of domains then union the scopes
1147 // within those domains
1148 SmallPtrSet<const MDNode *, 16> ADomains;
1149 SmallPtrSet<const MDNode *, 16> IntersectDomains;
1150 SmallSetVector<Metadata *, 4> MDs;
1151 for (const MDOperand &MDOp : A->operands())
1152 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1153 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1154 ADomains.insert(Ptr: Domain);
1155
1156 for (const MDOperand &MDOp : B->operands())
1157 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1158 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1159 if (ADomains.contains(Ptr: Domain)) {
1160 IntersectDomains.insert(Ptr: Domain);
1161 MDs.insert(X: MDOp);
1162 }
1163
1164 for (const MDOperand &MDOp : A->operands())
1165 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1166 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1167 if (IntersectDomains.contains(Ptr: Domain))
1168 MDs.insert(X: MDOp);
1169
1170 return MDs.empty() ? nullptr
1171 : getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1172}
1173
1174MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) {
1175 if (!A || !B)
1176 return nullptr;
1177
1178 APFloat AVal = mdconst::extract<ConstantFP>(MD: A->getOperand(I: 0))->getValueAPF();
1179 APFloat BVal = mdconst::extract<ConstantFP>(MD: B->getOperand(I: 0))->getValueAPF();
1180 if (AVal < BVal)
1181 return A;
1182 return B;
1183}
1184
1185// Call instructions with branch weights are only used in SamplePGO as
1186// documented in
1187/// https://llvm.org/docs/BranchWeightMetadata.html#callinst).
1188MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B,
1189 const Instruction *AInstr,
1190 const Instruction *BInstr) {
1191 assert(A && B && AInstr && BInstr && "Caller should guarantee");
1192 auto &Ctx = AInstr->getContext();
1193 MDBuilder MDHelper(Ctx);
1194
1195 // LLVM IR verifier verifies !prof metadata has at least 2 operands.
1196 assert(A->getNumOperands() >= 2 && B->getNumOperands() >= 2 &&
1197 "!prof annotations should have no less than 2 operands");
1198 MDString *AMDS = dyn_cast<MDString>(Val: A->getOperand(I: 0));
1199 MDString *BMDS = dyn_cast<MDString>(Val: B->getOperand(I: 0));
1200 // LLVM IR verfier verifies first operand is MDString.
1201 assert(AMDS != nullptr && BMDS != nullptr &&
1202 "first operand should be a non-null MDString");
1203 StringRef AProfName = AMDS->getString();
1204 StringRef BProfName = BMDS->getString();
1205 if (AProfName == MDProfLabels::BranchWeights &&
1206 BProfName == MDProfLabels::BranchWeights) {
1207 ConstantInt *AInstrWeight = mdconst::dyn_extract<ConstantInt>(
1208 MD: A->getOperand(I: getBranchWeightOffset(ProfileData: A)));
1209 ConstantInt *BInstrWeight = mdconst::dyn_extract<ConstantInt>(
1210 MD: B->getOperand(I: getBranchWeightOffset(ProfileData: B)));
1211 assert(AInstrWeight && BInstrWeight && "verified by LLVM verifier");
1212 return MDNode::get(Context&: Ctx,
1213 MDs: {MDHelper.createString(Str: MDProfLabels::BranchWeights),
1214 MDHelper.createConstant(C: ConstantInt::get(
1215 Ty: Type::getInt64Ty(C&: Ctx),
1216 V: SaturatingAdd(X: AInstrWeight->getZExtValue(),
1217 Y: BInstrWeight->getZExtValue())))});
1218 }
1219 return nullptr;
1220}
1221
1222// Pass in both instructions and nodes. Instruction information (e.g.,
1223// instruction type) helps interpret profiles and make implementation clearer.
1224MDNode *MDNode::getMergedProfMetadata(MDNode *A, MDNode *B,
1225 const Instruction *AInstr,
1226 const Instruction *BInstr) {
1227 // Check that it is legal to merge prof metadata based on the opcode.
1228 auto IsLegal = [](const Instruction &I) -> bool {
1229 switch (I.getOpcode()) {
1230 case Instruction::Invoke:
1231 case Instruction::Br:
1232 case Instruction::Switch:
1233 case Instruction::Call:
1234 case Instruction::IndirectBr:
1235 case Instruction::Select:
1236 case Instruction::CallBr:
1237 return true;
1238 default:
1239 return false;
1240 }
1241 };
1242 if (AInstr && !IsLegal(*AInstr))
1243 return nullptr;
1244 if (BInstr && !IsLegal(*BInstr))
1245 return nullptr;
1246
1247 if (!(A && B)) {
1248 return A ? A : B;
1249 }
1250
1251 assert(AInstr->getMetadata(LLVMContext::MD_prof) == A &&
1252 "Caller should guarantee");
1253 assert(BInstr->getMetadata(LLVMContext::MD_prof) == B &&
1254 "Caller should guarantee");
1255
1256 const CallInst *ACall = dyn_cast<CallInst>(Val: AInstr);
1257 const CallInst *BCall = dyn_cast<CallInst>(Val: BInstr);
1258
1259 // Both ACall and BCall are direct callsites.
1260 if (ACall && BCall && ACall->getCalledFunction() &&
1261 BCall->getCalledFunction())
1262 return mergeDirectCallProfMetadata(A, B, AInstr, BInstr);
1263
1264 // The rest of the cases are not implemented but could be added
1265 // when there are use cases.
1266 return nullptr;
1267}
1268
1269static bool isContiguous(const ConstantRange &A, const ConstantRange &B) {
1270 return A.getUpper() == B.getLower() || A.getLower() == B.getUpper();
1271}
1272
1273static bool canBeMerged(const ConstantRange &A, const ConstantRange &B) {
1274 return !A.intersectWith(CR: B).isEmptySet() || isContiguous(A, B);
1275}
1276
1277static bool tryMergeRange(SmallVectorImpl<ConstantInt *> &EndPoints,
1278 ConstantInt *Low, ConstantInt *High) {
1279 ConstantRange NewRange(Low->getValue(), High->getValue());
1280 unsigned Size = EndPoints.size();
1281 const APInt &LB = EndPoints[Size - 2]->getValue();
1282 const APInt &LE = EndPoints[Size - 1]->getValue();
1283 ConstantRange LastRange(LB, LE);
1284 if (canBeMerged(A: NewRange, B: LastRange)) {
1285 ConstantRange Union = LastRange.unionWith(CR: NewRange);
1286 Type *Ty = High->getType();
1287 EndPoints[Size - 2] =
1288 cast<ConstantInt>(Val: ConstantInt::get(Ty, V: Union.getLower()));
1289 EndPoints[Size - 1] =
1290 cast<ConstantInt>(Val: ConstantInt::get(Ty, V: Union.getUpper()));
1291 return true;
1292 }
1293 return false;
1294}
1295
1296static void addRange(SmallVectorImpl<ConstantInt *> &EndPoints,
1297 ConstantInt *Low, ConstantInt *High) {
1298 if (!EndPoints.empty())
1299 if (tryMergeRange(EndPoints, Low, High))
1300 return;
1301
1302 EndPoints.push_back(Elt: Low);
1303 EndPoints.push_back(Elt: High);
1304}
1305
1306MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) {
1307 // Given two ranges, we want to compute the union of the ranges. This
1308 // is slightly complicated by having to combine the intervals and merge
1309 // the ones that overlap.
1310
1311 if (!A || !B)
1312 return nullptr;
1313
1314 if (A == B)
1315 return A;
1316
1317 // First, walk both lists in order of the lower boundary of each interval.
1318 // At each step, try to merge the new interval to the last one we added.
1319 SmallVector<ConstantInt *, 4> EndPoints;
1320 unsigned AI = 0;
1321 unsigned BI = 0;
1322 unsigned AN = A->getNumOperands() / 2;
1323 unsigned BN = B->getNumOperands() / 2;
1324 while (AI < AN && BI < BN) {
1325 ConstantInt *ALow = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI));
1326 ConstantInt *BLow = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI));
1327
1328 if (ALow->getValue().slt(RHS: BLow->getValue())) {
1329 addRange(EndPoints, Low: ALow,
1330 High: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI + 1)));
1331 ++AI;
1332 } else {
1333 addRange(EndPoints, Low: BLow,
1334 High: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI + 1)));
1335 ++BI;
1336 }
1337 }
1338 while (AI < AN) {
1339 addRange(EndPoints, Low: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI)),
1340 High: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI + 1)));
1341 ++AI;
1342 }
1343 while (BI < BN) {
1344 addRange(EndPoints, Low: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI)),
1345 High: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI + 1)));
1346 ++BI;
1347 }
1348
1349 // We haven't handled wrap in the previous merge,
1350 // if we have at least 2 ranges (4 endpoints) we have to try to merge
1351 // the last and first ones.
1352 unsigned Size = EndPoints.size();
1353 if (Size > 2) {
1354 ConstantInt *FB = EndPoints[0];
1355 ConstantInt *FE = EndPoints[1];
1356 if (tryMergeRange(EndPoints, Low: FB, High: FE)) {
1357 for (unsigned i = 0; i < Size - 2; ++i) {
1358 EndPoints[i] = EndPoints[i + 2];
1359 }
1360 EndPoints.resize(N: Size - 2);
1361 }
1362 }
1363
1364 // If in the end we have a single range, it is possible that it is now the
1365 // full range. Just drop the metadata in that case.
1366 if (EndPoints.size() == 2) {
1367 ConstantRange Range(EndPoints[0]->getValue(), EndPoints[1]->getValue());
1368 if (Range.isFullSet())
1369 return nullptr;
1370 }
1371
1372 SmallVector<Metadata *, 4> MDs;
1373 MDs.reserve(N: EndPoints.size());
1374 for (auto *I : EndPoints)
1375 MDs.push_back(Elt: ConstantAsMetadata::get(C: I));
1376 return MDNode::get(Context&: A->getContext(), MDs);
1377}
1378
1379MDNode *MDNode::getMostGenericNoaliasAddrspace(MDNode *A, MDNode *B) {
1380 if (!A || !B)
1381 return nullptr;
1382
1383 if (A == B)
1384 return A;
1385
1386 SmallVector<ConstantRange> RangeListA, RangeListB;
1387 for (unsigned I = 0, E = A->getNumOperands() / 2; I != E; ++I) {
1388 auto *LowA = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * I + 0));
1389 auto *HighA = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * I + 1));
1390 RangeListA.push_back(Elt: ConstantRange(LowA->getValue(), HighA->getValue()));
1391 }
1392
1393 for (unsigned I = 0, E = B->getNumOperands() / 2; I != E; ++I) {
1394 auto *LowB = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * I + 0));
1395 auto *HighB = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * I + 1));
1396 RangeListB.push_back(Elt: ConstantRange(LowB->getValue(), HighB->getValue()));
1397 }
1398
1399 ConstantRangeList CRLA(RangeListA);
1400 ConstantRangeList CRLB(RangeListB);
1401 ConstantRangeList Result = CRLA.intersectWith(CRL: CRLB);
1402 if (Result.empty())
1403 return nullptr;
1404
1405 SmallVector<Metadata *> MDs;
1406 for (const ConstantRange &CR : Result) {
1407 MDs.push_back(Elt: ConstantAsMetadata::get(
1408 C: ConstantInt::get(Context&: A->getContext(), V: CR.getLower())));
1409 MDs.push_back(Elt: ConstantAsMetadata::get(
1410 C: ConstantInt::get(Context&: A->getContext(), V: CR.getUpper())));
1411 }
1412
1413 return MDNode::get(Context&: A->getContext(), MDs);
1414}
1415
1416MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) {
1417 if (!A || !B)
1418 return nullptr;
1419
1420 ConstantInt *AVal = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 0));
1421 ConstantInt *BVal = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 0));
1422 if (AVal->getZExtValue() < BVal->getZExtValue())
1423 return A;
1424 return B;
1425}
1426
1427//===----------------------------------------------------------------------===//
1428// NamedMDNode implementation.
1429//
1430
1431static SmallVector<TrackingMDRef, 4> &getNMDOps(void *Operands) {
1432 return *(SmallVector<TrackingMDRef, 4> *)Operands;
1433}
1434
1435NamedMDNode::NamedMDNode(const Twine &N)
1436 : Name(N.str()), Operands(new SmallVector<TrackingMDRef, 4>()) {}
1437
1438NamedMDNode::~NamedMDNode() {
1439 dropAllReferences();
1440 delete &getNMDOps(Operands);
1441}
1442
1443unsigned NamedMDNode::getNumOperands() const {
1444 return (unsigned)getNMDOps(Operands).size();
1445}
1446
1447MDNode *NamedMDNode::getOperand(unsigned i) const {
1448 assert(i < getNumOperands() && "Invalid Operand number!");
1449 auto *N = getNMDOps(Operands)[i].get();
1450 return cast_or_null<MDNode>(Val: N);
1451}
1452
1453void NamedMDNode::addOperand(MDNode *M) { getNMDOps(Operands).emplace_back(Args&: M); }
1454
1455void NamedMDNode::setOperand(unsigned I, MDNode *New) {
1456 assert(I < getNumOperands() && "Invalid operand number");
1457 getNMDOps(Operands)[I].reset(MD: New);
1458}
1459
1460void NamedMDNode::eraseFromParent() { getParent()->eraseNamedMetadata(NMD: this); }
1461
1462void NamedMDNode::clearOperands() { getNMDOps(Operands).clear(); }
1463
1464StringRef NamedMDNode::getName() const { return StringRef(Name); }
1465
1466//===----------------------------------------------------------------------===//
1467// Instruction Metadata method implementations.
1468//
1469
1470MDNode *MDAttachments::lookup(unsigned ID) const {
1471 for (const auto &A : Attachments)
1472 if (A.MDKind == ID)
1473 return A.Node;
1474 return nullptr;
1475}
1476
1477void MDAttachments::get(unsigned ID, SmallVectorImpl<MDNode *> &Result) const {
1478 for (const auto &A : Attachments)
1479 if (A.MDKind == ID)
1480 Result.push_back(Elt: A.Node);
1481}
1482
1483void MDAttachments::getAll(
1484 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
1485 for (const auto &A : Attachments)
1486 Result.emplace_back(Args: A.MDKind, Args: A.Node);
1487
1488 // Sort the resulting array so it is stable with respect to metadata IDs. We
1489 // need to preserve the original insertion order though.
1490 if (Result.size() > 1)
1491 llvm::stable_sort(Range&: Result, C: less_first());
1492}
1493
1494void MDAttachments::set(unsigned ID, MDNode *MD) {
1495 erase(ID);
1496 if (MD)
1497 insert(ID, MD&: *MD);
1498}
1499
1500void MDAttachments::insert(unsigned ID, MDNode &MD) {
1501 Attachments.push_back(Elt: {.MDKind: ID, .Node: TrackingMDNodeRef(&MD)});
1502}
1503
1504bool MDAttachments::erase(unsigned ID) {
1505 if (empty())
1506 return false;
1507
1508 // Common case is one value.
1509 if (Attachments.size() == 1 && Attachments.back().MDKind == ID) {
1510 Attachments.pop_back();
1511 return true;
1512 }
1513
1514 auto OldSize = Attachments.size();
1515 llvm::erase_if(C&: Attachments,
1516 P: [ID](const Attachment &A) { return A.MDKind == ID; });
1517 return OldSize != Attachments.size();
1518}
1519
1520MDNode *Value::getMetadata(StringRef Kind) const {
1521 if (!hasMetadata())
1522 return nullptr;
1523 unsigned KindID = getContext().getMDKindID(Name: Kind);
1524 return getMetadataImpl(KindID);
1525}
1526
1527MDNode *Value::getMetadataImpl(unsigned KindID) const {
1528 const LLVMContext &Ctx = getContext();
1529 const MDAttachments &Attachements = Ctx.pImpl->ValueMetadata.at(Val: this);
1530 return Attachements.lookup(ID: KindID);
1531}
1532
1533void Value::getMetadata(unsigned KindID, SmallVectorImpl<MDNode *> &MDs) const {
1534 if (hasMetadata())
1535 getContext().pImpl->ValueMetadata.at(Val: this).get(ID: KindID, Result&: MDs);
1536}
1537
1538void Value::getMetadata(StringRef Kind, SmallVectorImpl<MDNode *> &MDs) const {
1539 if (hasMetadata())
1540 getMetadata(KindID: getContext().getMDKindID(Name: Kind), MDs);
1541}
1542
1543void Value::getAllMetadata(
1544 SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const {
1545 if (hasMetadata()) {
1546 assert(getContext().pImpl->ValueMetadata.count(this) &&
1547 "bit out of sync with hash table");
1548 const MDAttachments &Info = getContext().pImpl->ValueMetadata.at(Val: this);
1549 Info.getAll(Result&: MDs);
1550 }
1551}
1552
1553void Value::setMetadata(unsigned KindID, MDNode *Node) {
1554 assert(isa<Instruction>(this) || isa<GlobalObject>(this));
1555
1556 // Handle the case when we're adding/updating metadata on a value.
1557 if (Node) {
1558 MDAttachments &Info = getContext().pImpl->ValueMetadata[this];
1559 assert(!Info.empty() == HasMetadata && "bit out of sync with hash table");
1560 if (Info.empty())
1561 HasMetadata = true;
1562 Info.set(ID: KindID, MD: Node);
1563 return;
1564 }
1565
1566 // Otherwise, we're removing metadata from an instruction.
1567 assert((HasMetadata == (getContext().pImpl->ValueMetadata.count(this) > 0)) &&
1568 "bit out of sync with hash table");
1569 if (!HasMetadata)
1570 return; // Nothing to remove!
1571 MDAttachments &Info = getContext().pImpl->ValueMetadata.find(Val: this)->second;
1572
1573 // Handle removal of an existing value.
1574 Info.erase(ID: KindID);
1575 if (!Info.empty())
1576 return;
1577 getContext().pImpl->ValueMetadata.erase(Val: this);
1578 HasMetadata = false;
1579}
1580
1581void Value::setMetadata(StringRef Kind, MDNode *Node) {
1582 if (!Node && !HasMetadata)
1583 return;
1584 setMetadata(KindID: getContext().getMDKindID(Name: Kind), Node);
1585}
1586
1587void Value::addMetadata(unsigned KindID, MDNode &MD) {
1588 assert(isa<Instruction>(this) || isa<GlobalObject>(this));
1589 if (!HasMetadata)
1590 HasMetadata = true;
1591 getContext().pImpl->ValueMetadata[this].insert(ID: KindID, MD);
1592}
1593
1594void Value::addMetadata(StringRef Kind, MDNode &MD) {
1595 addMetadata(KindID: getContext().getMDKindID(Name: Kind), MD);
1596}
1597
1598bool Value::eraseMetadata(unsigned KindID) {
1599 // Nothing to unset.
1600 if (!HasMetadata)
1601 return false;
1602
1603 MDAttachments &Store = getContext().pImpl->ValueMetadata.find(Val: this)->second;
1604 bool Changed = Store.erase(ID: KindID);
1605 if (Store.empty())
1606 clearMetadata();
1607 return Changed;
1608}
1609
1610void Value::eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred) {
1611 if (!HasMetadata)
1612 return;
1613
1614 auto &MetadataStore = getContext().pImpl->ValueMetadata;
1615 MDAttachments &Info = MetadataStore.find(Val: this)->second;
1616 assert(!Info.empty() && "bit out of sync with hash table");
1617 Info.remove_if(shouldRemove: [Pred](const MDAttachments::Attachment &I) {
1618 return Pred(I.MDKind, I.Node);
1619 });
1620
1621 if (Info.empty())
1622 clearMetadata();
1623}
1624
1625void Value::clearMetadata() {
1626 if (!HasMetadata)
1627 return;
1628 assert(getContext().pImpl->ValueMetadata.count(this) &&
1629 "bit out of sync with hash table");
1630 getContext().pImpl->ValueMetadata.erase(Val: this);
1631 HasMetadata = false;
1632}
1633
1634void Instruction::setMetadata(StringRef Kind, MDNode *Node) {
1635 if (!Node && !hasMetadata())
1636 return;
1637 setMetadata(KindID: getContext().getMDKindID(Name: Kind), Node);
1638}
1639
1640MDNode *Instruction::getMetadataImpl(StringRef Kind) const {
1641 const LLVMContext &Ctx = getContext();
1642 unsigned KindID = Ctx.getMDKindID(Name: Kind);
1643 if (KindID == LLVMContext::MD_dbg)
1644 return DbgLoc.getAsMDNode();
1645 return Value::getMetadata(KindID);
1646}
1647
1648void Instruction::eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred) {
1649 if (DbgLoc && Pred(LLVMContext::MD_dbg, DbgLoc.getAsMDNode()))
1650 DbgLoc = {};
1651
1652 Value::eraseMetadataIf(Pred);
1653}
1654
1655void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) {
1656 if (!Value::hasMetadata())
1657 return; // Nothing to remove!
1658
1659 SmallSet<unsigned, 32> KnownSet(llvm::from_range, KnownIDs);
1660
1661 // A DIAssignID attachment is debug metadata, don't drop it.
1662 KnownSet.insert(V: LLVMContext::MD_DIAssignID);
1663
1664 Value::eraseMetadataIf(Pred: [&KnownSet](unsigned MDKind, MDNode *Node) {
1665 return !KnownSet.count(V: MDKind);
1666 });
1667}
1668
1669void Instruction::updateDIAssignIDMapping(DIAssignID *ID) {
1670 auto &IDToInstrs = getContext().pImpl->AssignmentIDToInstrs;
1671 if (const DIAssignID *CurrentID =
1672 cast_or_null<DIAssignID>(Val: getMetadata(KindID: LLVMContext::MD_DIAssignID))) {
1673 // Nothing to do if the ID isn't changing.
1674 if (ID == CurrentID)
1675 return;
1676
1677 // Unmap this instruction from its current ID.
1678 auto InstrsIt = IDToInstrs.find(Val: CurrentID);
1679 assert(InstrsIt != IDToInstrs.end() &&
1680 "Expect existing attachment to be mapped");
1681
1682 auto &InstVec = InstrsIt->second;
1683 auto *InstIt = llvm::find(Range&: InstVec, Val: this);
1684 assert(InstIt != InstVec.end() &&
1685 "Expect instruction to be mapped to attachment");
1686 // The vector contains a ptr to this. If this is the only element in the
1687 // vector, remove the ID:vector entry, otherwise just remove the
1688 // instruction from the vector.
1689 if (InstVec.size() == 1)
1690 IDToInstrs.erase(I: InstrsIt);
1691 else
1692 InstVec.erase(CI: InstIt);
1693 }
1694
1695 // Map this instruction to the new ID.
1696 if (ID)
1697 IDToInstrs[ID].push_back(Elt: this);
1698}
1699
1700void Instruction::setMetadata(unsigned KindID, MDNode *Node) {
1701 if (!Node && !hasMetadata())
1702 return;
1703
1704 // Handle 'dbg' as a special case since it is not stored in the hash table.
1705 if (KindID == LLVMContext::MD_dbg) {
1706 DbgLoc = DebugLoc(Node);
1707 return;
1708 }
1709
1710 // Update DIAssignID to Instruction(s) mapping.
1711 if (KindID == LLVMContext::MD_DIAssignID) {
1712 // The DIAssignID tracking infrastructure doesn't support RAUWing temporary
1713 // nodes with DIAssignIDs. The cast_or_null below would also catch this, but
1714 // having a dedicated assert helps make this obvious.
1715 assert((!Node || !Node->isTemporary()) &&
1716 "Temporary DIAssignIDs are invalid");
1717 updateDIAssignIDMapping(ID: cast_or_null<DIAssignID>(Val: Node));
1718 }
1719
1720 Value::setMetadata(KindID, Node);
1721}
1722
1723void Instruction::addAnnotationMetadata(SmallVector<StringRef> Annotations) {
1724 SmallVector<Metadata *, 4> Names;
1725 if (auto *Existing = getMetadata(KindID: LLVMContext::MD_annotation)) {
1726 SmallSetVector<StringRef, 2> AnnotationsSet(Annotations.begin(),
1727 Annotations.end());
1728 auto *Tuple = cast<MDTuple>(Val: Existing);
1729 for (auto &N : Tuple->operands()) {
1730 if (isa<MDString>(Val: N.get())) {
1731 Names.push_back(Elt: N);
1732 continue;
1733 }
1734 auto *MDAnnotationTuple = cast<MDTuple>(Val: N);
1735 if (any_of(Range: MDAnnotationTuple->operands(), P: [&AnnotationsSet](auto &Op) {
1736 return AnnotationsSet.contains(key: cast<MDString>(Op)->getString());
1737 }))
1738 return;
1739 Names.push_back(Elt: N);
1740 }
1741 }
1742
1743 MDBuilder MDB(getContext());
1744 SmallVector<Metadata *> MDAnnotationStrings;
1745 for (StringRef Annotation : Annotations)
1746 MDAnnotationStrings.push_back(Elt: MDB.createString(Str: Annotation));
1747 MDNode *InfoTuple = MDTuple::get(Context&: getContext(), MDs: MDAnnotationStrings);
1748 Names.push_back(Elt: InfoTuple);
1749 MDNode *MD = MDTuple::get(Context&: getContext(), MDs: Names);
1750 setMetadata(KindID: LLVMContext::MD_annotation, Node: MD);
1751}
1752
1753void Instruction::addAnnotationMetadata(StringRef Name) {
1754 SmallVector<Metadata *, 4> Names;
1755 if (auto *Existing = getMetadata(KindID: LLVMContext::MD_annotation)) {
1756 auto *Tuple = cast<MDTuple>(Val: Existing);
1757 for (auto &N : Tuple->operands()) {
1758 if (isa<MDString>(Val: N.get()) &&
1759 cast<MDString>(Val: N.get())->getString() == Name)
1760 return;
1761 Names.push_back(Elt: N.get());
1762 }
1763 }
1764
1765 MDBuilder MDB(getContext());
1766 Names.push_back(Elt: MDB.createString(Str: Name));
1767 MDNode *MD = MDTuple::get(Context&: getContext(), MDs: Names);
1768 setMetadata(KindID: LLVMContext::MD_annotation, Node: MD);
1769}
1770
1771AAMDNodes Instruction::getAAMetadata() const {
1772 AAMDNodes Result;
1773 // Not using Instruction::hasMetadata() because we're not interested in
1774 // DebugInfoMetadata.
1775 if (Value::hasMetadata()) {
1776 const MDAttachments &Info = getContext().pImpl->ValueMetadata.at(Val: this);
1777 Result.TBAA = Info.lookup(ID: LLVMContext::MD_tbaa);
1778 Result.TBAAStruct = Info.lookup(ID: LLVMContext::MD_tbaa_struct);
1779 Result.Scope = Info.lookup(ID: LLVMContext::MD_alias_scope);
1780 Result.NoAlias = Info.lookup(ID: LLVMContext::MD_noalias);
1781 }
1782 return Result;
1783}
1784
1785void Instruction::setAAMetadata(const AAMDNodes &N) {
1786 setMetadata(KindID: LLVMContext::MD_tbaa, Node: N.TBAA);
1787 setMetadata(KindID: LLVMContext::MD_tbaa_struct, Node: N.TBAAStruct);
1788 setMetadata(KindID: LLVMContext::MD_alias_scope, Node: N.Scope);
1789 setMetadata(KindID: LLVMContext::MD_noalias, Node: N.NoAlias);
1790}
1791
1792void Instruction::setNoSanitizeMetadata() {
1793 setMetadata(KindID: llvm::LLVMContext::MD_nosanitize,
1794 Node: llvm::MDNode::get(Context&: getContext(), MDs: {}));
1795}
1796
1797void Instruction::getAllMetadataImpl(
1798 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
1799 Result.clear();
1800
1801 // Handle 'dbg' as a special case since it is not stored in the hash table.
1802 if (DbgLoc) {
1803 Result.push_back(
1804 Elt: std::make_pair(x: (unsigned)LLVMContext::MD_dbg, y: DbgLoc.getAsMDNode()));
1805 }
1806 Value::getAllMetadata(MDs&: Result);
1807}
1808
1809bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) const {
1810 assert(
1811 (getOpcode() == Instruction::Br || getOpcode() == Instruction::Select ||
1812 getOpcode() == Instruction::Call || getOpcode() == Instruction::Invoke ||
1813 getOpcode() == Instruction::IndirectBr ||
1814 getOpcode() == Instruction::Switch) &&
1815 "Looking for branch weights on something besides branch");
1816
1817 return ::extractProfTotalWeight(I: *this, TotalWeights&: TotalVal);
1818}
1819
1820void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) {
1821 SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
1822 Other->getAllMetadata(MDs);
1823 for (auto &MD : MDs) {
1824 // We need to adjust the type metadata offset.
1825 if (Offset != 0 && MD.first == LLVMContext::MD_type) {
1826 auto *OffsetConst = cast<ConstantInt>(
1827 Val: cast<ConstantAsMetadata>(Val: MD.second->getOperand(I: 0))->getValue());
1828 Metadata *TypeId = MD.second->getOperand(I: 1);
1829 auto *NewOffsetMD = ConstantAsMetadata::get(C: ConstantInt::get(
1830 Ty: OffsetConst->getType(), V: OffsetConst->getValue() + Offset));
1831 addMetadata(KindID: LLVMContext::MD_type,
1832 MD&: *MDNode::get(Context&: getContext(), MDs: {NewOffsetMD, TypeId}));
1833 continue;
1834 }
1835 // If an offset adjustment was specified we need to modify the DIExpression
1836 // to prepend the adjustment:
1837 // !DIExpression(DW_OP_plus, Offset, [original expr])
1838 auto *Attachment = MD.second;
1839 if (Offset != 0 && MD.first == LLVMContext::MD_dbg) {
1840 DIGlobalVariable *GV = dyn_cast<DIGlobalVariable>(Val: Attachment);
1841 DIExpression *E = nullptr;
1842 if (!GV) {
1843 auto *GVE = cast<DIGlobalVariableExpression>(Val: Attachment);
1844 GV = GVE->getVariable();
1845 E = GVE->getExpression();
1846 }
1847 ArrayRef<uint64_t> OrigElements;
1848 if (E)
1849 OrigElements = E->getElements();
1850 std::vector<uint64_t> Elements(OrigElements.size() + 2);
1851 Elements[0] = dwarf::DW_OP_plus_uconst;
1852 Elements[1] = Offset;
1853 llvm::copy(Range&: OrigElements, Out: Elements.begin() + 2);
1854 E = DIExpression::get(Context&: getContext(), Elements);
1855 Attachment = DIGlobalVariableExpression::get(Context&: getContext(), Variable: GV, Expression: E);
1856 }
1857 addMetadata(KindID: MD.first, MD&: *Attachment);
1858 }
1859}
1860
1861void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) {
1862 addMetadata(
1863 KindID: LLVMContext::MD_type,
1864 MD&: *MDTuple::get(Context&: getContext(),
1865 MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1866 Ty: Type::getInt64Ty(C&: getContext()), V: Offset)),
1867 TypeID}));
1868}
1869
1870void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) {
1871 // Remove any existing vcall visibility metadata first in case we are
1872 // updating.
1873 eraseMetadata(KindID: LLVMContext::MD_vcall_visibility);
1874 addMetadata(KindID: LLVMContext::MD_vcall_visibility,
1875 MD&: *MDNode::get(Context&: getContext(),
1876 MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1877 Ty: Type::getInt64Ty(C&: getContext()), V: Visibility))}));
1878}
1879
1880GlobalObject::VCallVisibility GlobalObject::getVCallVisibility() const {
1881 if (MDNode *MD = getMetadata(KindID: LLVMContext::MD_vcall_visibility)) {
1882 uint64_t Val = cast<ConstantInt>(
1883 Val: cast<ConstantAsMetadata>(Val: MD->getOperand(I: 0))->getValue())
1884 ->getZExtValue();
1885 assert(Val <= 2 && "unknown vcall visibility!");
1886 return (VCallVisibility)Val;
1887 }
1888 return VCallVisibility::VCallVisibilityPublic;
1889}
1890
1891void Function::setSubprogram(DISubprogram *SP) {
1892 setMetadata(KindID: LLVMContext::MD_dbg, Node: SP);
1893}
1894
1895DISubprogram *Function::getSubprogram() const {
1896 return cast_or_null<DISubprogram>(Val: getMetadata(KindID: LLVMContext::MD_dbg));
1897}
1898
1899bool Function::shouldEmitDebugInfoForProfiling() const {
1900 if (DISubprogram *SP = getSubprogram()) {
1901 if (DICompileUnit *CU = SP->getUnit()) {
1902 return CU->getDebugInfoForProfiling();
1903 }
1904 }
1905 return false;
1906}
1907
1908void GlobalVariable::addDebugInfo(DIGlobalVariableExpression *GV) {
1909 addMetadata(KindID: LLVMContext::MD_dbg, MD&: *GV);
1910}
1911
1912void GlobalVariable::getDebugInfo(
1913 SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const {
1914 SmallVector<MDNode *, 1> MDs;
1915 getMetadata(KindID: LLVMContext::MD_dbg, MDs);
1916 for (MDNode *MD : MDs)
1917 GVs.push_back(Elt: cast<DIGlobalVariableExpression>(Val: MD));
1918}
1919