1//===- Metadata.cpp - Implement Metadata classes --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Metadata classes.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Metadata.h"
14#include "LLVMContextImpl.h"
15#include "MetadataImpl.h"
16#include "llvm/ADT/APFloat.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/DenseSet.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/SmallString.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/StringMap.h"
27#include "llvm/ADT/StringRef.h"
28#include "llvm/ADT/Twine.h"
29#include "llvm/IR/Argument.h"
30#include "llvm/IR/BasicBlock.h"
31#include "llvm/IR/Constant.h"
32#include "llvm/IR/ConstantRange.h"
33#include "llvm/IR/ConstantRangeList.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DebugInfoMetadata.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/DebugProgramInstruction.h"
38#include "llvm/IR/Function.h"
39#include "llvm/IR/GlobalObject.h"
40#include "llvm/IR/GlobalVariable.h"
41#include "llvm/IR/Instruction.h"
42#include "llvm/IR/LLVMContext.h"
43#include "llvm/IR/MDBuilder.h"
44#include "llvm/IR/Module.h"
45#include "llvm/IR/ProfDataUtils.h"
46#include "llvm/IR/TrackingMDRef.h"
47#include "llvm/IR/Type.h"
48#include "llvm/IR/Value.h"
49#include "llvm/Support/Casting.h"
50#include "llvm/Support/CommandLine.h"
51
52#include "llvm/Support/ErrorHandling.h"
53#include "llvm/Support/MathExtras.h"
54#include "llvm/Support/ModRef.h"
55#include <cassert>
56#include <cstddef>
57#include <cstdint>
58#include <type_traits>
59#include <utility>
60#include <vector>
61
62using namespace llvm;
63
64MetadataAsValue::MetadataAsValue(Type *Ty, Metadata *MD)
65 : Value(Ty, MetadataAsValueVal), MD(MD) {
66 track();
67}
68
69MetadataAsValue::~MetadataAsValue() {
70 getType()->getContext().pImpl->MetadataAsValues.erase(Val: MD);
71 untrack();
72}
73
74/// Canonicalize metadata arguments to intrinsics.
75///
76/// To support bitcode upgrades (and assembly semantic sugar) for \a
77/// MetadataAsValue, we need to canonicalize certain metadata.
78///
79/// - nullptr is replaced by an empty MDNode.
80/// - An MDNode with a single null operand is replaced by an empty MDNode.
81/// - An MDNode whose only operand is a \a ConstantAsMetadata gets skipped.
82///
83/// This maintains readability of bitcode from when metadata was a type of
84/// value, and these bridges were unnecessary.
85static Metadata *canonicalizeMetadataForValue(LLVMContext &Context,
86 Metadata *MD) {
87 if (!MD)
88 // !{}
89 return MDNode::get(Context, MDs: {});
90
91 // Return early if this isn't a single-operand MDNode.
92 auto *N = dyn_cast<MDNode>(Val: MD);
93 if (!N || N->getNumOperands() != 1)
94 return MD;
95
96 if (!N->getOperand(I: 0))
97 // !{}
98 return MDNode::get(Context, MDs: {});
99
100 if (auto *C = dyn_cast<ConstantAsMetadata>(Val: N->getOperand(I: 0)))
101 // Look through the MDNode.
102 return C;
103
104 return MD;
105}
106
107MetadataAsValue *MetadataAsValue::get(LLVMContext &Context, Metadata *MD) {
108 MD = canonicalizeMetadataForValue(Context, MD);
109 auto *&Entry = Context.pImpl->MetadataAsValues[MD];
110 if (!Entry)
111 Entry = new MetadataAsValue(Type::getMetadataTy(C&: Context), MD);
112 return Entry;
113}
114
115MetadataAsValue *MetadataAsValue::getIfExists(LLVMContext &Context,
116 Metadata *MD) {
117 MD = canonicalizeMetadataForValue(Context, MD);
118 auto &Store = Context.pImpl->MetadataAsValues;
119 return Store.lookup(Val: MD);
120}
121
122void MetadataAsValue::handleChangedMetadata(Metadata *MD) {
123 LLVMContext &Context = getContext();
124 MD = canonicalizeMetadataForValue(Context, MD);
125 auto &Store = Context.pImpl->MetadataAsValues;
126
127 // Stop tracking the old metadata.
128 Store.erase(Val: this->MD);
129 untrack();
130 this->MD = nullptr;
131
132 // Start tracking MD, or RAUW if necessary.
133 auto *&Entry = Store[MD];
134 if (Entry) {
135 replaceAllUsesWith(V: Entry);
136 delete this;
137 return;
138 }
139
140 this->MD = MD;
141 track();
142 Entry = this;
143}
144
145void MetadataAsValue::track() {
146 if (MD)
147 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
148}
149
150void MetadataAsValue::untrack() {
151 if (MD)
152 MetadataTracking::untrack(MD);
153}
154
155DbgVariableRecord *DebugValueUser::getUser() {
156 return static_cast<DbgVariableRecord *>(this);
157}
158const DbgVariableRecord *DebugValueUser::getUser() const {
159 return static_cast<const DbgVariableRecord *>(this);
160}
161
162void DebugValueUser::handleChangedValue(void *Old, Metadata *New) {
163 // NOTE: We could inform the "owner" that a value has changed through
164 // getOwner, if needed.
165 auto OldMD = static_cast<Metadata **>(Old);
166 ptrdiff_t Idx = std::distance(first: &*DebugValues.begin(), last: OldMD);
167 // If replacing a ValueAsMetadata with a nullptr, replace it with a
168 // PoisonValue instead.
169 if (OldMD && isa<ValueAsMetadata>(Val: *OldMD) && !New) {
170 auto *OldVAM = cast<ValueAsMetadata>(Val: *OldMD);
171 New = ValueAsMetadata::get(V: PoisonValue::get(T: OldVAM->getValue()->getType()));
172 }
173 resetDebugValue(Idx, DebugValue: New);
174}
175
176void DebugValueUser::trackDebugValue(size_t Idx) {
177 assert(Idx < 3 && "Invalid debug value index.");
178 Metadata *&MD = DebugValues[Idx];
179 if (!MD)
180 return;
181 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
182 if (auto *ID = Idx == AssignIDIdx ? dyn_cast<DIAssignID>(Val: MD) : nullptr)
183 ID->Records.push_back(NewVal: getUser());
184}
185
186void DebugValueUser::trackDebugValues() {
187 for (size_t I = 0, E = DebugValues.size(); I != E; ++I)
188 trackDebugValue(Idx: I);
189}
190
191void DebugValueUser::untrackDebugValue(size_t Idx) {
192 assert(Idx < 3 && "Invalid debug value index.");
193 Metadata *&MD = DebugValues[Idx];
194 if (!MD)
195 return;
196 MetadataTracking::untrack(MD);
197 if (auto *ID = Idx == AssignIDIdx ? dyn_cast<DIAssignID>(Val: MD) : nullptr)
198 ID->Records.erase(I: llvm::find(Range&: ID->Records, Val: getUser()));
199}
200
201void DebugValueUser::untrackDebugValues() {
202 for (size_t I = 0, E = DebugValues.size(); I != E; ++I)
203 untrackDebugValue(Idx: I);
204}
205
206void DebugValueUser::retrackDebugValues(DebugValueUser &X) {
207 assert(DebugValueUser::operator==(X) && "Expected values to match");
208 for (const auto &[MD, XMD] : zip(t&: DebugValues, u&: X.DebugValues))
209 if (XMD)
210 MetadataTracking::retrack(MD&: XMD, New&: MD);
211 if (auto *ID = dyn_cast_or_null<DIAssignID>(Val: DebugValues[AssignIDIdx]))
212 *llvm::find(Range&: ID->Records, Val: X.getUser()) = getUser();
213 X.DebugValues.fill(u: nullptr);
214}
215
216bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) {
217 assert(Ref && "Expected live reference");
218 assert((Owner || *static_cast<Metadata **>(Ref) == &MD) &&
219 "Reference without owner must be direct");
220 if (auto *R = ReplaceableUses::getOrCreate(MD)) {
221 R->addRef(Ref, Owner);
222 return true;
223 }
224 if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(Val: &MD)) {
225 assert(!PH->Use && "Placeholders can only be used once");
226 assert(!Owner && "Unexpected callback to owner");
227 PH->Use = static_cast<Metadata **>(Ref);
228 return true;
229 }
230 return false;
231}
232
233void MetadataTracking::untrack(void *Ref, Metadata &MD) {
234 assert(Ref && "Expected live reference");
235 if (auto *R = ReplaceableUses::getIfExists(MD))
236 R->dropRef(Ref);
237 else if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(Val: &MD))
238 PH->Use = nullptr;
239}
240
241bool MetadataTracking::retrack(void *Ref, Metadata &MD, void *New) {
242 assert(Ref && "Expected live reference");
243 assert(New && "Expected live reference");
244 assert(Ref != New && "Expected change");
245 if (auto *R = ReplaceableUses::getIfExists(MD)) {
246 R->moveRef(Ref, New, MD);
247 return true;
248 }
249 assert(!isa<DistinctMDOperandPlaceholder>(MD) &&
250 "Unexpected move of an MDOperand");
251 assert(!isReplaceable(MD) &&
252 "Expected un-replaceable metadata, since we didn't move a reference");
253 return false;
254}
255
256bool MetadataTracking::isReplaceable(const Metadata &MD) {
257 return ReplaceableUses::isReplaceable(MD);
258}
259
260SmallVector<Metadata *> ReplaceableUses::getAllArgListUsers() {
261 SmallVector<std::pair<OwnerTy, uint64_t> *> MDUsersWithID;
262 for (auto Pair : UseMap) {
263 OwnerTy Owner = Pair.second.first;
264 if (Owner.isNull())
265 continue;
266 if (!isa<Metadata *>(Val: Owner))
267 continue;
268 Metadata *OwnerMD = cast<Metadata *>(Val&: Owner);
269 if (OwnerMD->getMetadataID() == Metadata::DIArgListKind)
270 MDUsersWithID.push_back(Elt: &UseMap[Pair.first]);
271 }
272 llvm::sort(C&: MDUsersWithID, Comp: [](auto UserA, auto UserB) {
273 return UserA->second < UserB->second;
274 });
275 SmallVector<Metadata *> MDUsers;
276 for (auto *UserWithID : MDUsersWithID)
277 MDUsers.push_back(Elt: cast<Metadata *>(Val&: UserWithID->first));
278 return MDUsers;
279}
280
281SmallVector<DbgVariableRecord *>
282ReplaceableUses::getAllDbgVariableRecordUsers() {
283 SmallVector<std::pair<OwnerTy, uint64_t> *> DVRUsersWithID;
284 for (auto Pair : UseMap) {
285 OwnerTy Owner = Pair.second.first;
286 if (Owner.isNull())
287 continue;
288 if (!isa<DebugValueUser *>(Val: Owner))
289 continue;
290 DVRUsersWithID.push_back(Elt: &UseMap[Pair.first]);
291 }
292 // Order DbgVariableRecord users in reverse-creation order. Normal dbg.value
293 // users of MetadataAsValues are ordered by their UseList, i.e. reverse order
294 // of when they were added: we need to replicate that here. The structure of
295 // debug-info output depends on the ordering of intrinsics, thus we need
296 // to keep them consistent for comparisons sake.
297 llvm::sort(C&: DVRUsersWithID, Comp: [](auto UserA, auto UserB) {
298 return UserA->second > UserB->second;
299 });
300 SmallVector<DbgVariableRecord *> DVRUsers;
301 for (auto UserWithID : DVRUsersWithID)
302 DVRUsers.push_back(Elt: cast<DebugValueUser *>(Val&: UserWithID->first)->getUser());
303 return DVRUsers;
304}
305
306void ReplaceableUses::addRef(void *Ref, OwnerTy Owner) {
307 bool WasInserted =
308 UseMap.insert(KV: std::make_pair(x&: Ref, y: std::make_pair(x&: Owner, y&: NextIndex)))
309 .second;
310 (void)WasInserted;
311 assert(WasInserted && "Expected to add a reference");
312
313 ++NextIndex;
314 assert(NextIndex != 0 && "Unexpected overflow");
315}
316
317void ReplaceableUses::dropRef(void *Ref) {
318 bool WasErased = UseMap.erase(Val: Ref);
319 (void)WasErased;
320 assert(WasErased && "Expected to drop a reference");
321}
322
323void ReplaceableUses::moveRef(void *Ref, void *New, const Metadata &MD) {
324 auto I = UseMap.find(Val: Ref);
325 assert(I != UseMap.end() && "Expected to move a reference");
326 auto OwnerAndIndex = I->second;
327 UseMap.erase(I);
328 bool WasInserted = UseMap.insert(KV: std::make_pair(x&: New, y&: OwnerAndIndex)).second;
329 (void)WasInserted;
330 assert(WasInserted && "Expected to add a reference");
331
332 // Check that the references are direct if there's no owner.
333 (void)MD;
334 assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) &&
335 "Reference without owner must be direct");
336 assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) &&
337 "Reference without owner must be direct");
338}
339
340void ReplaceableUses::SalvageDebugInfo(const Constant &C) {
341 if (!C.isUsedByMetadata()) {
342 return;
343 }
344
345 LLVMContext &Context = C.getType()->getContext();
346 auto &Store = Context.pImpl->ValuesAsMetadata;
347 auto I = Store.find(Val: &C);
348 ValueAsMetadata *MD = I->second;
349 using UseTy =
350 std::pair<void *, std::pair<MetadataTracking::OwnerTy, uint64_t>>;
351 // Copy out uses and update value of Constant used by debug info metadata with
352 // poison below
353 SmallVector<UseTy, 8> Uses(MD->UseMap.begin(), MD->UseMap.end());
354
355 for (const auto &Pair : Uses) {
356 MetadataTracking::OwnerTy Owner = Pair.second.first;
357 if (!Owner)
358 continue;
359 // Check for MetadataAsValue.
360 if (isa<MetadataAsValue *>(Val: Owner)) {
361 cast<MetadataAsValue *>(Val&: Owner)->handleChangedMetadata(
362 MD: ValueAsMetadata::get(V: PoisonValue::get(T: C.getType())));
363 continue;
364 }
365 if (!isa<Metadata *>(Val: Owner))
366 continue;
367 auto *OwnerMD = dyn_cast_if_present<MDNode>(Val: cast<Metadata *>(Val&: Owner));
368 if (!OwnerMD)
369 continue;
370 if (isa<DINode>(Val: OwnerMD)) {
371 OwnerMD->handleChangedOperand(
372 Ref: Pair.first, New: ValueAsMetadata::get(V: PoisonValue::get(T: C.getType())));
373 }
374 }
375}
376
377void ReplaceableUses::replaceAllUsesWith(Metadata *MD) {
378 if (UseMap.empty())
379 return;
380
381 // Copy out uses since UseMap will get touched below.
382 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
383 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
384 llvm::sort(C&: Uses, Comp: [](const UseTy &L, const UseTy &R) {
385 return L.second.second < R.second.second;
386 });
387 for (const auto &Pair : Uses) {
388 // Check that this Ref hasn't disappeared after RAUW (when updating a
389 // previous Ref).
390 if (!UseMap.count(Val: Pair.first))
391 continue;
392
393 OwnerTy Owner = Pair.second.first;
394 if (!Owner) {
395 // Update unowned tracking references directly.
396 Metadata *&Ref = *static_cast<Metadata **>(Pair.first);
397 Ref = MD;
398 if (MD)
399 MetadataTracking::track(MD&: Ref);
400 UseMap.erase(Val: Pair.first);
401 continue;
402 }
403
404 // Check for MetadataAsValue.
405 if (isa<MetadataAsValue *>(Val: Owner)) {
406 cast<MetadataAsValue *>(Val&: Owner)->handleChangedMetadata(MD);
407 continue;
408 }
409
410 if (auto *DVU = dyn_cast<DebugValueUser *>(Val&: Owner)) {
411 DVU->handleChangedValue(Old: Pair.first, New: MD);
412 continue;
413 }
414
415 // There's a Metadata owner -- dispatch.
416 Metadata *OwnerMD = cast<Metadata *>(Val&: Owner);
417 switch (OwnerMD->getMetadataID()) {
418#define HANDLE_METADATA_LEAF(CLASS) \
419 case Metadata::CLASS##Kind: \
420 cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \
421 continue;
422#include "llvm/IR/Metadata.def"
423 default:
424 llvm_unreachable("Invalid metadata subclass");
425 }
426 }
427 assert(UseMap.empty() && "Expected all uses to be replaced");
428}
429
430void ReplaceableUses::resolveAllUses(bool ResolveUsers) {
431 if (UseMap.empty())
432 return;
433
434 if (!ResolveUsers) {
435 UseMap.clear();
436 return;
437 }
438
439 // Copy out uses since UseMap could get touched below.
440 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
441 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
442 llvm::sort(C&: Uses, Comp: [](const UseTy &L, const UseTy &R) {
443 return L.second.second < R.second.second;
444 });
445 UseMap.clear();
446 for (const auto &Pair : Uses) {
447 auto Owner = Pair.second.first;
448 if (!Owner)
449 continue;
450 if (!isa<Metadata *>(Val: Owner))
451 continue;
452
453 // Resolve MDNodes that point at this.
454 auto *OwnerMD = dyn_cast_if_present<MDNode>(Val: cast<Metadata *>(Val&: Owner));
455 if (!OwnerMD)
456 continue;
457 if (OwnerMD->isResolved())
458 continue;
459 OwnerMD->decrementUnresolvedOperandCount();
460 }
461}
462
463// A value without a use list (e.g. ConstantData) is never RAUW'd, so don't
464// create a ReplaceableUses instance for it.
465static bool isTrackedValue(const Metadata &MD) {
466 auto *VAM = dyn_cast<ValueAsMetadata>(Val: &MD);
467 return VAM && VAM->getValue()->hasUseList();
468}
469
470// Special handing of DIArgList is required in the RemoveDIs project, see
471// commentry in DIArgList::handleChangedOperand for details. Hidden behind
472// conditional compilation to avoid a compile time regression.
473ReplaceableUses *ReplaceableUses::getOrCreate(Metadata &MD) {
474 if (auto *N = dyn_cast<MDNode>(Val: &MD)) {
475 return N->isResolved() ? nullptr : N->Context.getOrCreateReplaceableUses();
476 }
477 if (auto ArgList = dyn_cast<DIArgList>(Val: &MD))
478 return ArgList;
479 return isTrackedValue(MD) ? cast<ValueAsMetadata>(Val: &MD) : nullptr;
480}
481
482ReplaceableUses *ReplaceableUses::getIfExists(Metadata &MD) {
483 if (auto *N = dyn_cast<MDNode>(Val: &MD)) {
484 return N->isResolved() ? nullptr : N->Context.getReplaceableUses();
485 }
486 if (auto ArgList = dyn_cast<DIArgList>(Val: &MD))
487 return ArgList;
488 return isTrackedValue(MD) ? cast<ValueAsMetadata>(Val: &MD) : nullptr;
489}
490
491bool ReplaceableUses::isReplaceable(const Metadata &MD) {
492 if (auto *N = dyn_cast<MDNode>(Val: &MD))
493 return !N->isResolved();
494 return isTrackedValue(MD) || isa<DIArgList>(Val: &MD);
495}
496
497static DISubprogram *getLocalFunctionMetadata(Value *V) {
498 assert(V && "Expected value");
499 if (auto *A = dyn_cast<Argument>(Val: V)) {
500 if (auto *Fn = A->getParent())
501 return Fn->getSubprogram();
502 return nullptr;
503 }
504
505 if (BasicBlock *BB = cast<Instruction>(Val: V)->getParent()) {
506 if (auto *Fn = BB->getParent())
507 return Fn->getSubprogram();
508 return nullptr;
509 }
510
511 return nullptr;
512}
513
514ValueAsMetadata *ValueAsMetadata::get(Value *V) {
515 assert(V && "Unexpected null Value");
516
517 auto &Context = V->getContext();
518 auto *&Entry = Context.pImpl->ValuesAsMetadata[V];
519 if (!Entry) {
520 assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) &&
521 "Expected constant or function-local value");
522 assert(!V->IsUsedByMD && "Expected this to be the only metadata use");
523 V->IsUsedByMD = true;
524 if (auto *C = dyn_cast<Constant>(Val: V))
525 Entry = new ConstantAsMetadata(C);
526 else
527 Entry = new LocalAsMetadata(V);
528 }
529
530 return Entry;
531}
532
533ValueAsMetadata *ValueAsMetadata::getIfExists(Value *V) {
534 assert(V && "Unexpected null Value");
535 return V->getContext().pImpl->ValuesAsMetadata.lookup(Val: V);
536}
537
538void ValueAsMetadata::handleDeletion(Value *V) {
539 assert(V && "Expected valid value");
540
541 auto &Store = V->getType()->getContext().pImpl->ValuesAsMetadata;
542 auto I = Store.find(Val: V);
543 if (I == Store.end())
544 return;
545
546 // Remove old entry from the map.
547 ValueAsMetadata *MD = I->second;
548 assert(MD && "Expected valid metadata");
549 assert(MD->getValue() == V && "Expected valid mapping");
550 Store.erase(I);
551
552 // Delete the metadata.
553 MD->replaceAllUsesWith(MD: nullptr);
554 delete MD;
555}
556
557void ValueAsMetadata::handleRAUW(Value *From, Value *To) {
558 assert(From && "Expected valid value");
559 assert(To && "Expected valid value");
560 assert(From != To && "Expected changed value");
561 assert(&From->getContext() == &To->getContext() && "Expected same context");
562 assert(From->hasUseList() && "Must have use list");
563
564 auto &Store = From->getContext().pImpl->ValuesAsMetadata;
565 auto I = Store.find(Val: From);
566 if (I == Store.end()) {
567 assert(!From->IsUsedByMD && "Expected From not to be used by metadata");
568 return;
569 }
570
571 assert(From->IsUsedByMD && "Expected From to be used by metadata");
572 From->IsUsedByMD = false;
573 ValueAsMetadata *MD = I->second;
574 assert(MD && "Expected valid metadata");
575 assert(MD->getValue() == From && "Expected valid mapping");
576 Store.erase(I);
577
578 // Move the uses to To's node. Uses of a function-local value are dropped if
579 // it becomes a local of another function or replaces a constant.
580 Metadata *New = nullptr;
581 if (isa<Constant>(Val: To)) {
582 New = ValueAsMetadata::get(V: To);
583 } else if (isa<LocalAsMetadata>(Val: MD)) {
584 DISubprogram *FromSP = getLocalFunctionMetadata(V: From);
585 DISubprogram *ToSP = FromSP ? getLocalFunctionMetadata(V: To) : nullptr;
586 if (!FromSP || !ToSP || FromSP == ToSP)
587 New = ValueAsMetadata::get(V: To);
588 }
589 MD->replaceAllUsesWith(MD: New);
590 delete MD;
591}
592
593//===----------------------------------------------------------------------===//
594// MDString implementation.
595//
596
597MDString *MDString::get(LLVMContext &Context, StringRef Str) {
598 auto &Store = Context.pImpl->MDStringCache;
599 auto I = Store.try_emplace(Key: Str);
600 auto &MapEntry = I.first->getValue();
601 if (!I.second)
602 return &MapEntry;
603 MapEntry.Entry = &*I.first;
604 return &MapEntry;
605}
606
607MDString *MDString::getIfExists(LLVMContext &Context, StringRef Str) {
608 auto &Store = Context.pImpl->MDStringCache;
609 auto I = Store.find(Key: Str);
610 if (I == Store.end())
611 return nullptr;
612 return &I->getValue();
613}
614
615StringRef MDString::getString() const {
616 assert(Entry && "Expected to find string map entry");
617 return Entry->first();
618}
619
620//===----------------------------------------------------------------------===//
621// MDNode implementation.
622//
623
624// Assert that the MDNode types will not be unaligned by the objects
625// prepended to them.
626#define HANDLE_MDNODE_LEAF(CLASS) \
627 static_assert( \
628 alignof(uint64_t) >= alignof(CLASS), \
629 "Alignment is insufficient after objects prepended to " #CLASS);
630#include "llvm/IR/Metadata.def"
631
632void *MDNode::operator new(size_t Size, size_t NumOps, StorageType Storage) {
633 // uint64_t is the most aligned type we need support (ensured by static_assert
634 // above)
635 static_assert(sizeof(Header) == sizeof(size_t) + 2 * sizeof(uint32_t),
636 "MDNode header fields poorly packed");
637 size_t AllocSize =
638 alignTo(Value: Header::getAllocSize(Storage, NumOps), Align: alignof(uint64_t));
639 char *Mem = reinterpret_cast<char *>(::operator new(AllocSize + Size));
640 Header *H = new (Mem + AllocSize - sizeof(Header)) Header(NumOps, Storage);
641 return reinterpret_cast<void *>(H + 1);
642}
643
644void MDNode::operator delete(void *N) {
645 Header *H = reinterpret_cast<Header *>(N) - 1;
646 void *Mem = H->getAllocation();
647 H->~Header();
648 ::operator delete(Mem);
649}
650
651MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage,
652 ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2)
653 : Metadata(ID, Storage), Context(Context) {
654 getHeader().MetadataPrintID = Context.pImpl->allocateMetadataPrintID();
655
656 unsigned Op = 0;
657 for (Metadata *MD : Ops1)
658 setOperand(I: Op++, New: MD);
659 for (Metadata *MD : Ops2)
660 setOperand(I: Op++, New: MD);
661
662 if (!isUniqued())
663 return;
664
665 // Count the unresolved operands. If there are any, RAUW support will be
666 // added lazily on first reference.
667 countUnresolvedOperands();
668}
669
670TempMDNode MDNode::clone() const {
671 switch (getMetadataID()) {
672 default:
673 llvm_unreachable("Invalid MDNode subclass");
674#define HANDLE_MDNODE_LEAF(CLASS) \
675 case CLASS##Kind: \
676 return cast<CLASS>(this)->cloneImpl();
677#include "llvm/IR/Metadata.def"
678 }
679}
680
681MDNode::Header::Header(size_t NumOps, StorageType Storage) {
682 IsLarge = isLarge(NumOps);
683 IsResizable = isResizable(Storage);
684 SmallSize = getSmallSize(NumOps, IsResizable, IsLarge);
685 if (IsLarge) {
686 SmallNumOps = 0;
687 new (getLargePtr()) LargeStorageVector();
688 getLarge().resize(N: NumOps);
689 return;
690 }
691 SmallNumOps = NumOps;
692 MDOperand *O = reinterpret_cast<MDOperand *>(this) - SmallSize;
693 for (MDOperand *E = O + SmallSize; O != E;)
694 (void)new (O++) MDOperand();
695}
696
697MDNode::Header::~Header() {
698 if (IsLarge) {
699 getLarge().~LargeStorageVector();
700 return;
701 }
702 MDOperand *O = reinterpret_cast<MDOperand *>(this);
703 for (MDOperand *E = O - SmallSize; O != E; --O)
704 (O - 1)->~MDOperand();
705}
706
707void *MDNode::Header::getSmallPtr() {
708 static_assert(alignof(MDOperand) <= alignof(Header),
709 "MDOperand too strongly aligned");
710 return reinterpret_cast<char *>(const_cast<Header *>(this)) -
711 sizeof(MDOperand) * SmallSize;
712}
713
714void MDNode::Header::resize(size_t NumOps) {
715 assert(IsResizable && "Node is not resizable");
716 if (operands().size() == NumOps)
717 return;
718
719 if (IsLarge)
720 getLarge().resize(N: NumOps);
721 else if (NumOps <= SmallSize)
722 resizeSmall(NumOps);
723 else
724 resizeSmallToLarge(NumOps);
725}
726
727void MDNode::Header::resizeSmall(size_t NumOps) {
728 assert(!IsLarge && "Expected a small MDNode");
729 assert(NumOps <= SmallSize && "NumOps too large for small resize");
730
731 MutableArrayRef<MDOperand> ExistingOps = operands();
732 assert(NumOps != ExistingOps.size() && "Expected a different size");
733
734 int NumNew = (int)NumOps - (int)ExistingOps.size();
735 MDOperand *O = ExistingOps.end();
736 for (int I = 0, E = NumNew; I < E; ++I)
737 (O++)->reset();
738 for (int I = 0, E = NumNew; I > E; --I)
739 (--O)->reset();
740 SmallNumOps = NumOps;
741 assert(O == operands().end() && "Operands not (un)initialized until the end");
742}
743
744void MDNode::Header::resizeSmallToLarge(size_t NumOps) {
745 assert(!IsLarge && "Expected a small MDNode");
746 assert(NumOps > SmallSize && "Expected NumOps to be larger than allocation");
747 LargeStorageVector NewOps;
748 NewOps.resize(N: NumOps);
749 llvm::move(Range: operands(), Out: NewOps.begin());
750 resizeSmall(NumOps: 0);
751 new (getLargePtr()) LargeStorageVector(std::move(NewOps));
752 IsLarge = true;
753}
754
755static bool isOperandUnresolved(Metadata *Op) {
756 if (auto *N = dyn_cast_or_null<MDNode>(Val: Op))
757 return !N->isResolved();
758 return false;
759}
760
761void MDNode::countUnresolvedOperands() {
762 assert(getNumUnresolved() == 0 && "Expected unresolved ops to be uncounted");
763 assert(isUniqued() && "Expected this to be uniqued");
764 setNumUnresolved(count_if(Range: operands(), P: isOperandUnresolved));
765}
766
767void MDNode::makeUniqued() {
768 assert(isTemporary() && "Expected this to be temporary");
769 assert(!isResolved() && "Expected this to be unresolved");
770 bool WasTracked = getContext().pImpl->TemporaryMDNodes.erase(V: this);
771 assert(WasTracked && "Temporary node not tracked");
772 (void)WasTracked;
773
774 // Enable uniquing callbacks.
775 for (auto &Op : mutable_operands())
776 Op.reset(MD: Op.get(), Owner: this);
777
778 // Make this 'uniqued'.
779 Storage = Uniqued;
780 countUnresolvedOperands();
781 if (!getNumUnresolved()) {
782 dropReplaceableUses();
783 assert(isResolved() && "Expected this to be resolved");
784 }
785
786 assert(isUniqued() && "Expected this to be uniqued");
787}
788
789void MDNode::makeDistinct() {
790 assert(isTemporary() && "Expected this to be temporary");
791 assert(!isResolved() && "Expected this to be unresolved");
792
793 // Drop RAUW support and store as a distinct node.
794 dropReplaceableUses();
795 storeDistinctInContext();
796
797 assert(isDistinct() && "Expected this to be distinct");
798 assert(isResolved() && "Expected this to be resolved");
799}
800
801void MDNode::resolve() {
802 assert(isUniqued() && "Expected this to be uniqued");
803 assert(!isResolved() && "Expected this to be unresolved");
804
805 setNumUnresolved(0);
806 dropReplaceableUses();
807
808 assert(isResolved() && "Expected this to be resolved");
809}
810
811void MDNode::dropReplaceableUses() {
812 assert(!getNumUnresolved() && "Unexpected unresolved operand");
813
814 // Drop any RAUW support.
815 if (Context.hasReplaceableUses())
816 Context.takeReplaceableUses()->resolveAllUses();
817}
818
819void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) {
820 assert(isUniqued() && "Expected this to be uniqued");
821 assert(getNumUnresolved() != 0 && "Expected unresolved operands");
822
823 // Check if an operand was resolved.
824 if (!isOperandUnresolved(Op: Old)) {
825 if (isOperandUnresolved(Op: New))
826 // An operand was un-resolved!
827 setNumUnresolved(getNumUnresolved() + 1);
828 } else if (!isOperandUnresolved(Op: New))
829 decrementUnresolvedOperandCount();
830}
831
832void MDNode::decrementUnresolvedOperandCount() {
833 assert(!isResolved() && "Expected this to be unresolved");
834 if (isTemporary())
835 return;
836
837 assert(isUniqued() && "Expected this to be uniqued");
838 setNumUnresolved(getNumUnresolved() - 1);
839 if (getNumUnresolved())
840 return;
841
842 // Last unresolved operand has just been resolved.
843 dropReplaceableUses();
844 assert(isResolved() && "Expected this to become resolved");
845}
846
847void MDNode::resolveCycles() {
848 if (isResolved())
849 return;
850
851 // Resolve this node immediately.
852 resolve();
853
854 // Resolve all operands.
855 for (const auto &Op : operands()) {
856 auto *N = dyn_cast_or_null<MDNode>(Val: Op);
857 if (!N)
858 continue;
859
860 assert(!N->isTemporary() &&
861 "Expected all forward declarations to be resolved");
862 if (!N->isResolved())
863 N->resolveCycles();
864 }
865}
866
867static bool hasSelfReference(MDNode *N) {
868 return llvm::is_contained(Range: N->operands(), Element: N);
869}
870
871MDNode *MDNode::replaceWithPermanentImpl() {
872 switch (getMetadataID()) {
873 default:
874 // If this type isn't uniquable, replace with a distinct node.
875 return replaceWithDistinctImpl();
876
877#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
878 case CLASS##Kind: \
879 break;
880#include "llvm/IR/Metadata.def"
881 }
882
883 // Even if this type is uniquable, self-references have to be distinct.
884 if (hasSelfReference(N: this))
885 return replaceWithDistinctImpl();
886 return replaceWithUniquedImpl();
887}
888
889MDNode *MDNode::replaceWithUniquedImpl() {
890 // Try to uniquify in place.
891 MDNode *UniquedNode = uniquify();
892
893 if (UniquedNode == this) {
894 makeUniqued();
895 return this;
896 }
897
898 // Collision, so RAUW instead.
899 replaceAllUsesWith(MD: UniquedNode);
900 deleteAsSubclass();
901 return UniquedNode;
902}
903
904MDNode *MDNode::replaceWithDistinctImpl() {
905 makeDistinct();
906 return this;
907}
908
909void MDTuple::recalculateHash() {
910 setHash(MDTupleInfo::KeyTy::calculateHash(N: this));
911}
912
913void MDNode::dropAllReferences() {
914 for (unsigned I = 0, E = getNumOperands(); I != E; ++I)
915 setOperand(I, New: nullptr);
916 if (Context.hasReplaceableUses()) {
917 Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false);
918 (void)Context.takeReplaceableUses();
919 }
920}
921
922void MDNode::handleChangedOperand(void *Ref, Metadata *New) {
923 unsigned Op = static_cast<MDOperand *>(Ref) - op_begin();
924 assert(Op < getNumOperands() && "Expected valid operand");
925
926 if (!isUniqued()) {
927 // This node is not uniqued. Just set the operand and be done with it.
928 setOperand(I: Op, New);
929 return;
930 }
931
932 // This node is uniqued.
933 eraseFromStore();
934
935 Metadata *Old = getOperand(I: Op);
936 setOperand(I: Op, New);
937
938 // Drop uniquing for self-reference cycles and deleted constants.
939 if (New == this || (!New && Old && isa<ConstantAsMetadata>(Val: Old))) {
940 if (!isResolved())
941 resolve();
942 storeDistinctInContext();
943 return;
944 }
945
946 // Re-unique the node.
947 auto *Uniqued = uniquify();
948 if (Uniqued == this) {
949 if (!isResolved())
950 resolveAfterOperandChange(Old, New);
951 return;
952 }
953
954 // Collision.
955 if (!isResolved()) {
956 // Still unresolved, so RAUW.
957 //
958 // First, clear out all operands to prevent any recursion (similar to
959 // dropAllReferences(), but we still need the use-list).
960 for (unsigned O = 0, E = getNumOperands(); O != E; ++O)
961 setOperand(I: O, New: nullptr);
962 if (Context.hasReplaceableUses())
963 Context.getReplaceableUses()->replaceAllUsesWith(MD: Uniqued);
964 deleteAsSubclass();
965 return;
966 }
967
968 // Store in non-uniqued form if RAUW isn't possible.
969 storeDistinctInContext();
970}
971
972void MDNode::deleteAsSubclass() {
973 if (isTemporary()) {
974 bool WasTracked = getContext().pImpl->TemporaryMDNodes.erase(V: this);
975 assert(WasTracked && "Temporary node not tracked");
976 (void)WasTracked;
977 }
978 switch (getMetadataID()) {
979 default:
980 llvm_unreachable("Invalid subclass of MDNode");
981#define HANDLE_MDNODE_LEAF(CLASS) \
982 case CLASS##Kind: \
983 delete cast<CLASS>(this); \
984 break;
985#include "llvm/IR/Metadata.def"
986 }
987}
988
989template <class T, class InfoT>
990static T *uniquifyImpl(T *N, DenseSet<T *, InfoT> &Store) {
991 if (T *U = getUniqued(Store, N))
992 return U;
993
994 Store.insert(N);
995 return N;
996}
997
998template <class NodeTy> struct MDNode::HasCachedHash {
999 template <class U>
1000 static std::true_type check(SameType<void (U::*)(unsigned), &U::setHash> *);
1001 template <class U> static std::false_type check(...);
1002
1003 static constexpr bool value = decltype(check<NodeTy>(nullptr))::value;
1004};
1005
1006MDNode *MDNode::uniquify() {
1007 assert(!hasSelfReference(this) && "Cannot uniquify a self-referencing node");
1008
1009 // Try to insert into uniquing store.
1010 switch (getMetadataID()) {
1011 default:
1012 llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
1013#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
1014 case CLASS##Kind: { \
1015 CLASS *SubclassThis = cast<CLASS>(this); \
1016 dispatchRecalculateHash(SubclassThis); \
1017 return uniquifyImpl(SubclassThis, getContext().pImpl->CLASS##s); \
1018 }
1019#include "llvm/IR/Metadata.def"
1020 }
1021}
1022
1023void MDNode::eraseFromStore() {
1024 switch (getMetadataID()) {
1025 default:
1026 llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
1027#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
1028 case CLASS##Kind: \
1029 getContext().pImpl->CLASS##s.erase(cast<CLASS>(this)); \
1030 break;
1031#include "llvm/IR/Metadata.def"
1032 }
1033}
1034
1035MDTuple *MDTuple::getImpl(LLVMContext &Context, ArrayRef<Metadata *> MDs,
1036 StorageType Storage, bool ShouldCreate) {
1037 unsigned Hash = 0;
1038 if (Storage == Uniqued) {
1039 MDTupleInfo::KeyTy Key(MDs);
1040 if (auto *N = getUniqued(Store&: Context.pImpl->MDTuples, Key))
1041 return N;
1042 if (!ShouldCreate)
1043 return nullptr;
1044 Hash = Key.getHash();
1045 } else {
1046 assert(ShouldCreate && "Expected non-uniqued nodes to always be created");
1047 }
1048
1049 return storeImpl(N: new (MDs.size(), Storage)
1050 MDTuple(Context, Storage, Hash, MDs),
1051 Storage, Store&: Context.pImpl->MDTuples);
1052}
1053
1054void MDNode::deleteTemporary(MDNode *N) {
1055 assert(N->isTemporary() && "Expected temporary node");
1056 N->replaceAllUsesWith(MD: nullptr);
1057 N->deleteAsSubclass();
1058}
1059
1060void MDNode::storeDistinctInContext() {
1061 assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses");
1062 assert(!getNumUnresolved() && "Unexpected unresolved nodes");
1063 if (isTemporary()) {
1064 bool WasTracked = getContext().pImpl->TemporaryMDNodes.erase(V: this);
1065 assert(WasTracked && "Temporary node not tracked");
1066 (void)WasTracked;
1067 }
1068 Storage = Distinct;
1069 assert(isResolved() && "Expected this to be resolved");
1070
1071 // Reset the hash.
1072 switch (getMetadataID()) {
1073 default:
1074 llvm_unreachable("Invalid subclass of MDNode");
1075#define HANDLE_MDNODE_LEAF(CLASS) \
1076 case CLASS##Kind: { \
1077 dispatchResetHash(cast<CLASS>(this)); \
1078 break; \
1079 }
1080#include "llvm/IR/Metadata.def"
1081 }
1082
1083 getContext().pImpl->DistinctMDNodes.push_back(x: this);
1084}
1085
1086void MDNode::replaceOperandWith(unsigned I, Metadata *New) {
1087 if (getOperand(I) == New)
1088 return;
1089
1090 if (!isUniqued()) {
1091 setOperand(I, New);
1092 return;
1093 }
1094
1095 handleChangedOperand(Ref: mutable_begin() + I, New);
1096}
1097
1098void MDNode::setOperand(unsigned I, Metadata *New) {
1099 assert(I < getNumOperands());
1100 mutable_begin()[I].reset(MD: New, Owner: isUniqued() ? this : nullptr);
1101}
1102
1103/// Get a node or a self-reference that looks like it.
1104///
1105/// Special handling for finding self-references, for use by \a
1106/// MDNode::concatenate() and \a MDNode::intersect() to maintain behaviour from
1107/// when self-referencing nodes were still uniqued. If the first operand has
1108/// the same operands as \c Ops, return the first operand instead.
1109static MDNode *getOrSelfReference(LLVMContext &Context,
1110 ArrayRef<Metadata *> Ops) {
1111 if (!Ops.empty())
1112 if (MDNode *N = dyn_cast_or_null<MDNode>(Val: Ops[0]))
1113 if (N->getNumOperands() == Ops.size() && N == N->getOperand(I: 0)) {
1114 for (unsigned I = 1, E = Ops.size(); I != E; ++I)
1115 if (Ops[I] != N->getOperand(I))
1116 return MDNode::get(Context, MDs: Ops);
1117 return N;
1118 }
1119
1120 return MDNode::get(Context, MDs: Ops);
1121}
1122
1123MDNode *MDNode::concatenate(MDNode *A, MDNode *B) {
1124 if (!A)
1125 return B;
1126 if (!B)
1127 return A;
1128
1129 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end());
1130 MDs.insert(Start: B->op_begin(), End: B->op_end());
1131
1132 // FIXME: This preserves long-standing behaviour, but is it really the right
1133 // behaviour? Or was that an unintended side-effect of node uniquing?
1134 return getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1135}
1136
1137MDNode *MDNode::intersect(MDNode *A, MDNode *B) {
1138 if (!A || !B)
1139 return nullptr;
1140
1141 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end());
1142 SmallPtrSet<Metadata *, 4> BSet(B->op_begin(), B->op_end());
1143 MDs.remove_if(P: [&](Metadata *MD) { return !BSet.count(Ptr: MD); });
1144
1145 // FIXME: This preserves long-standing behaviour, but is it really the right
1146 // behaviour? Or was that an unintended side-effect of node uniquing?
1147 return getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1148}
1149
1150MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) {
1151 if (!A || !B)
1152 return nullptr;
1153
1154 // Take the intersection of domains then union the scopes
1155 // within those domains
1156 SmallPtrSet<const MDNode *, 16> ADomains;
1157 SmallPtrSet<const MDNode *, 16> IntersectDomains;
1158 SmallSetVector<Metadata *, 4> MDs;
1159 for (const MDOperand &MDOp : A->operands())
1160 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1161 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1162 ADomains.insert(Ptr: Domain);
1163
1164 for (const MDOperand &MDOp : B->operands())
1165 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1166 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1167 if (ADomains.contains(Ptr: Domain)) {
1168 IntersectDomains.insert(Ptr: Domain);
1169 MDs.insert(X: MDOp);
1170 }
1171
1172 for (const MDOperand &MDOp : A->operands())
1173 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1174 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1175 if (IntersectDomains.contains(Ptr: Domain))
1176 MDs.insert(X: MDOp);
1177
1178 return MDs.empty() ? nullptr
1179 : getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1180}
1181
1182MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) {
1183 if (!A || !B)
1184 return nullptr;
1185
1186 APFloat AVal = mdconst::extract<ConstantFP>(MD: A->getOperand(I: 0))->getValueAPF();
1187 APFloat BVal = mdconst::extract<ConstantFP>(MD: B->getOperand(I: 0))->getValueAPF();
1188 if (AVal < BVal)
1189 return A;
1190 return B;
1191}
1192
1193// Call instructions with branch weights are only used in SamplePGO as
1194// documented in
1195/// https://llvm.org/docs/BranchWeightMetadata.html#callinst).
1196MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B,
1197 const Instruction *AInstr,
1198 const Instruction *BInstr) {
1199 assert(A && B && AInstr && BInstr && "Caller should guarantee");
1200 auto &Ctx = AInstr->getContext();
1201 MDBuilder MDHelper(Ctx);
1202
1203 // LLVM IR verifier verifies !prof metadata has at least 2 operands.
1204 assert(A->getNumOperands() >= 2 && B->getNumOperands() >= 2 &&
1205 "!prof annotations should have no less than 2 operands");
1206 MDString *AMDS = dyn_cast<MDString>(Val: A->getOperand(I: 0));
1207 MDString *BMDS = dyn_cast<MDString>(Val: B->getOperand(I: 0));
1208 // LLVM IR verfier verifies first operand is MDString.
1209 assert(AMDS != nullptr && BMDS != nullptr &&
1210 "first operand should be a non-null MDString");
1211 StringRef AProfName = AMDS->getString();
1212 StringRef BProfName = BMDS->getString();
1213 if (AProfName == MDProfLabels::BranchWeights &&
1214 BProfName == MDProfLabels::BranchWeights) {
1215 ConstantInt *AInstrWeight = mdconst::dyn_extract<ConstantInt>(
1216 MD: A->getOperand(I: getBranchWeightOffset(ProfileData: A)));
1217 ConstantInt *BInstrWeight = mdconst::dyn_extract<ConstantInt>(
1218 MD: B->getOperand(I: getBranchWeightOffset(ProfileData: B)));
1219 assert(AInstrWeight && BInstrWeight && "verified by LLVM verifier");
1220 return MDNode::get(Context&: Ctx,
1221 MDs: {MDHelper.createString(Str: MDProfLabels::BranchWeights),
1222 MDHelper.createConstant(C: ConstantInt::get(
1223 Ty: Type::getInt64Ty(C&: Ctx),
1224 V: SaturatingAdd(X: AInstrWeight->getZExtValue(),
1225 Y: BInstrWeight->getZExtValue())))});
1226 }
1227 return nullptr;
1228}
1229
1230// Pass in both instructions and nodes. Instruction information (e.g.,
1231// instruction type) helps interpret profiles and make implementation clearer.
1232MDNode *MDNode::getMergedProfMetadata(MDNode *A, MDNode *B,
1233 const Instruction *AInstr,
1234 const Instruction *BInstr) {
1235 // Check that it is legal to merge prof metadata based on the opcode.
1236 auto IsLegal = [](const Instruction &I) -> bool {
1237 switch (I.getOpcode()) {
1238 case Instruction::Invoke:
1239 case Instruction::CondBr:
1240 case Instruction::Switch:
1241 case Instruction::Call:
1242 case Instruction::IndirectBr:
1243 case Instruction::Select:
1244 case Instruction::CallBr:
1245 return true;
1246 default:
1247 return false;
1248 }
1249 };
1250 if (AInstr && !IsLegal(*AInstr))
1251 return nullptr;
1252 if (BInstr && !IsLegal(*BInstr))
1253 return nullptr;
1254
1255 if (!(A && B)) {
1256 return A ? A : B;
1257 }
1258
1259 assert(AInstr->getMetadata(LLVMContext::MD_prof) == A &&
1260 "Caller should guarantee");
1261 assert(BInstr->getMetadata(LLVMContext::MD_prof) == B &&
1262 "Caller should guarantee");
1263
1264 const CallInst *ACall = dyn_cast<CallInst>(Val: AInstr);
1265 const CallInst *BCall = dyn_cast<CallInst>(Val: BInstr);
1266
1267 // Both ACall and BCall are direct callsites.
1268 if (ACall && BCall && ACall->getCalledFunction() &&
1269 BCall->getCalledFunction())
1270 return mergeDirectCallProfMetadata(A, B, AInstr, BInstr);
1271
1272 if (A == B)
1273 return A;
1274
1275 // The rest of the cases are not implemented but could be added
1276 // when there are use cases.
1277 return nullptr;
1278}
1279
1280static bool isContiguous(const ConstantRange &A, const ConstantRange &B) {
1281 return A.getUpper() == B.getLower() || A.getLower() == B.getUpper();
1282}
1283
1284static bool canBeMerged(const ConstantRange &A, const ConstantRange &B) {
1285 return !A.intersectWith(CR: B).isEmptySet() || isContiguous(A, B);
1286}
1287
1288static bool tryMergeRange(SmallVectorImpl<ConstantInt *> &EndPoints,
1289 ConstantInt *Low, ConstantInt *High) {
1290 ConstantRange NewRange(Low->getValue(), High->getValue());
1291 unsigned Size = EndPoints.size();
1292 const APInt &LB = EndPoints[Size - 2]->getValue();
1293 const APInt &LE = EndPoints[Size - 1]->getValue();
1294 ConstantRange LastRange(LB, LE);
1295 if (canBeMerged(A: NewRange, B: LastRange)) {
1296 ConstantRange Union = LastRange.unionWith(CR: NewRange);
1297 Type *Ty = High->getType();
1298 EndPoints[Size - 2] =
1299 cast<ConstantInt>(Val: ConstantInt::get(Ty, V: Union.getLower()));
1300 EndPoints[Size - 1] =
1301 cast<ConstantInt>(Val: ConstantInt::get(Ty, V: Union.getUpper()));
1302 return true;
1303 }
1304 return false;
1305}
1306
1307static void addRange(SmallVectorImpl<ConstantInt *> &EndPoints,
1308 ConstantInt *Low, ConstantInt *High) {
1309 if (!EndPoints.empty())
1310 if (tryMergeRange(EndPoints, Low, High))
1311 return;
1312
1313 EndPoints.push_back(Elt: Low);
1314 EndPoints.push_back(Elt: High);
1315}
1316
1317MDNode *MDNode::getMergedCalleeTypeMetadata(const MDNode *A, const MDNode *B) {
1318 // Drop the callee_type metadata if either of the call instructions do not
1319 // have it.
1320 if (!A || !B)
1321 return nullptr;
1322 SmallVector<Metadata *, 8> AB;
1323 SmallPtrSet<Metadata *, 8> MergedCallees;
1324 auto AddUniqueCallees = [&AB, &MergedCallees](const MDNode *N) {
1325 for (Metadata *MD : N->operands()) {
1326 if (MergedCallees.insert(Ptr: MD).second)
1327 AB.push_back(Elt: MD);
1328 }
1329 };
1330 AddUniqueCallees(A);
1331 AddUniqueCallees(B);
1332 return MDNode::get(Context&: A->getContext(), MDs: AB);
1333}
1334
1335MDNode *MDNode::getMergedAllocTokenMetadata(const MDNode *A, const MDNode *B) {
1336 // Drop !alloc_token metadata if either instruction lacks it to avoid mis-
1337 // classifying unclassified allocations, where the fallback token must be
1338 // used instead.
1339 if (!A || !B)
1340 return nullptr;
1341 if (A == B)
1342 return const_cast<MDNode *>(A);
1343 if (A->getNumOperands() != 2 || B->getNumOperands() != 2)
1344 return nullptr;
1345 auto *CIA = mdconst::dyn_extract_or_null<ConstantInt>(MD: A->getOperand(I: 1));
1346 auto *CIB = mdconst::dyn_extract_or_null<ConstantInt>(MD: B->getOperand(I: 1));
1347 if (!CIA || !CIB)
1348 return nullptr;
1349
1350 MDString *NameA = dyn_cast<MDString>(Val: A->getOperand(I: 0));
1351 MDString *NameB = dyn_cast<MDString>(Val: B->getOperand(I: 0));
1352 if (!NameA || !NameB)
1353 return nullptr;
1354
1355 if (NameA == NameB)
1356 return CIA->isOne() ? const_cast<MDNode *>(A) : const_cast<MDNode *>(B);
1357
1358 LLVMContext &Ctx = A->getContext();
1359 StringRef StrA = NameA->getString();
1360 StringRef StrB = NameB->getString();
1361
1362 SmallString<64> Buffer;
1363 Buffer.reserve(N: StrA.size() + 1 + StrB.size());
1364 Buffer.append(RHS: StrA);
1365 Buffer.push_back(Elt: '|');
1366 Buffer.append(RHS: StrB);
1367
1368 bool MergedContainsPointer = CIA->isOne() || CIB->isOne();
1369 Metadata *Ops[] = {MDString::get(Context&: Ctx, Str: Buffer),
1370 ConstantAsMetadata::get(C: ConstantInt::get(
1371 Ty: Type::getInt1Ty(C&: Ctx), V: MergedContainsPointer))};
1372 return MDNode::get(Context&: Ctx, MDs: Ops);
1373}
1374
1375MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) {
1376 // Given two ranges, we want to compute the union of the ranges. This
1377 // is slightly complicated by having to combine the intervals and merge
1378 // the ones that overlap.
1379
1380 if (!A || !B)
1381 return nullptr;
1382
1383 if (A == B)
1384 return A;
1385
1386 // First, walk both lists in order of the lower boundary of each interval.
1387 // At each step, try to merge the new interval to the last one we added.
1388 SmallVector<ConstantInt *, 4> EndPoints;
1389 unsigned AI = 0;
1390 unsigned BI = 0;
1391 unsigned AN = A->getNumOperands() / 2;
1392 unsigned BN = B->getNumOperands() / 2;
1393 while (AI < AN && BI < BN) {
1394 ConstantInt *ALow = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI));
1395 ConstantInt *BLow = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI));
1396
1397 if (ALow->getValue().slt(RHS: BLow->getValue())) {
1398 addRange(EndPoints, Low: ALow,
1399 High: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI + 1)));
1400 ++AI;
1401 } else {
1402 addRange(EndPoints, Low: BLow,
1403 High: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI + 1)));
1404 ++BI;
1405 }
1406 }
1407 while (AI < AN) {
1408 addRange(EndPoints, Low: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI)),
1409 High: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI + 1)));
1410 ++AI;
1411 }
1412 while (BI < BN) {
1413 addRange(EndPoints, Low: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI)),
1414 High: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI + 1)));
1415 ++BI;
1416 }
1417
1418 // We haven't handled wrap in the previous merge,
1419 // if we have at least 2 ranges (4 endpoints) we have to try to merge
1420 // the last and first ones.
1421 unsigned Size = EndPoints.size();
1422 if (Size > 2) {
1423 ConstantInt *FB = EndPoints[0];
1424 ConstantInt *FE = EndPoints[1];
1425 if (tryMergeRange(EndPoints, Low: FB, High: FE)) {
1426 for (unsigned i = 0; i < Size - 2; ++i) {
1427 EndPoints[i] = EndPoints[i + 2];
1428 }
1429 EndPoints.resize(N: Size - 2);
1430 }
1431 }
1432
1433 // If in the end we have a single range, it is possible that it is now the
1434 // full range. Just drop the metadata in that case.
1435 if (EndPoints.size() == 2) {
1436 ConstantRange Range(EndPoints[0]->getValue(), EndPoints[1]->getValue());
1437 if (Range.isFullSet())
1438 return nullptr;
1439 }
1440
1441 SmallVector<Metadata *, 4> MDs;
1442 MDs.reserve(N: EndPoints.size());
1443 for (auto *I : EndPoints)
1444 MDs.push_back(Elt: ConstantAsMetadata::get(C: I));
1445 return MDNode::get(Context&: A->getContext(), MDs);
1446}
1447
1448MDNode *MDNode::getMostGenericNoFPClass(MDNode *A, MDNode *B) {
1449 if (!A || !B)
1450 return nullptr;
1451
1452 if (A == B)
1453 return A;
1454
1455 ConstantInt *AVal = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 0));
1456 ConstantInt *BVal = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 0));
1457 unsigned Intersect = AVal->getZExtValue() & BVal->getZExtValue();
1458 if (Intersect == 0)
1459 return nullptr;
1460
1461 return MDNode::get(Context&: A->getContext(), MDs: ConstantAsMetadata::get(C: ConstantInt::get(
1462 Ty: AVal->getType(), V: Intersect)));
1463}
1464
1465MDNode *MDNode::getMostGenericNoaliasAddrspace(MDNode *A, MDNode *B) {
1466 if (!A || !B)
1467 return nullptr;
1468
1469 if (A == B)
1470 return A;
1471
1472 SmallVector<ConstantRange> RangeListA, RangeListB;
1473 for (unsigned I = 0, E = A->getNumOperands() / 2; I != E; ++I) {
1474 auto *LowA = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * I + 0));
1475 auto *HighA = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * I + 1));
1476 RangeListA.push_back(Elt: ConstantRange(LowA->getValue(), HighA->getValue()));
1477 }
1478
1479 for (unsigned I = 0, E = B->getNumOperands() / 2; I != E; ++I) {
1480 auto *LowB = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * I + 0));
1481 auto *HighB = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * I + 1));
1482 RangeListB.push_back(Elt: ConstantRange(LowB->getValue(), HighB->getValue()));
1483 }
1484
1485 ConstantRangeList CRLA(RangeListA);
1486 ConstantRangeList CRLB(RangeListB);
1487 ConstantRangeList Result = CRLA.intersectWith(CRL: CRLB);
1488 if (Result.empty())
1489 return nullptr;
1490
1491 SmallVector<Metadata *> MDs;
1492 for (const ConstantRange &CR : Result) {
1493 MDs.push_back(Elt: ConstantAsMetadata::get(
1494 C: ConstantInt::get(Context&: A->getContext(), V: CR.getLower())));
1495 MDs.push_back(Elt: ConstantAsMetadata::get(
1496 C: ConstantInt::get(Context&: A->getContext(), V: CR.getUpper())));
1497 }
1498
1499 return MDNode::get(Context&: A->getContext(), MDs);
1500}
1501
1502MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) {
1503 if (!A || !B)
1504 return nullptr;
1505
1506 ConstantInt *AVal = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 0));
1507 ConstantInt *BVal = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 0));
1508 if (AVal->getZExtValue() < BVal->getZExtValue())
1509 return A;
1510 return B;
1511}
1512
1513CaptureComponents MDNode::toCaptureComponents(const MDNode *MD) {
1514 if (!MD)
1515 return CaptureComponents::All;
1516
1517 CaptureComponents CC = CaptureComponents::None;
1518 for (Metadata *Op : MD->operands()) {
1519 CaptureComponents Component =
1520 StringSwitch<CaptureComponents>(cast<MDString>(Val: Op)->getString())
1521 .Case(S: "address", Value: CaptureComponents::Address)
1522 .Case(S: "address_is_null", Value: CaptureComponents::AddressIsNull)
1523 .Case(S: "provenance", Value: CaptureComponents::Provenance)
1524 .Case(S: "read_provenance", Value: CaptureComponents::ReadProvenance);
1525 CC |= Component;
1526 }
1527 return CC;
1528}
1529
1530MDNode *MDNode::fromCaptureComponents(LLVMContext &Ctx, CaptureComponents CC) {
1531 assert(!capturesNothing(CC) && "Can't encode captures(none)");
1532 if (capturesAll(CC))
1533 return nullptr;
1534
1535 SmallVector<Metadata *> Components;
1536 if (capturesAddressIsNullOnly(CC))
1537 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "address_is_null"));
1538 else if (capturesAddress(CC))
1539 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "address"));
1540 if (capturesReadProvenanceOnly(CC))
1541 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "read_provenance"));
1542 else if (capturesFullProvenance(CC))
1543 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "provenance"));
1544 return MDNode::get(Context&: Ctx, MDs: Components);
1545}
1546
1547//===----------------------------------------------------------------------===//
1548// NamedMDNode implementation.
1549//
1550
1551static SmallVector<TrackingMDRef, 4> &getNMDOps(void *Operands) {
1552 return *(SmallVector<TrackingMDRef, 4> *)Operands;
1553}
1554
1555NamedMDNode::NamedMDNode(const Twine &N)
1556 : Name(N.str()), Operands(new SmallVector<TrackingMDRef, 4>()) {}
1557
1558NamedMDNode::~NamedMDNode() {
1559 dropAllReferences();
1560 delete &getNMDOps(Operands);
1561}
1562
1563unsigned NamedMDNode::getNumOperands() const {
1564 return (unsigned)getNMDOps(Operands).size();
1565}
1566
1567MDNode *NamedMDNode::getOperand(unsigned i) const {
1568 assert(i < getNumOperands() && "Invalid Operand number!");
1569 auto *N = getNMDOps(Operands)[i].get();
1570 return cast_or_null<MDNode>(Val: N);
1571}
1572
1573void NamedMDNode::addOperand(MDNode *M) { getNMDOps(Operands).emplace_back(Args&: M); }
1574
1575void NamedMDNode::setOperand(unsigned I, MDNode *New) {
1576 assert(I < getNumOperands() && "Invalid operand number");
1577 getNMDOps(Operands)[I].reset(MD: New);
1578}
1579
1580void NamedMDNode::eraseFromParent() { getParent()->eraseNamedMetadata(NMD: this); }
1581
1582void NamedMDNode::clearOperands() { getNMDOps(Operands).clear(); }
1583
1584StringRef NamedMDNode::getName() const { return StringRef(Name); }
1585
1586//===----------------------------------------------------------------------===//
1587// Instruction Metadata method implementations.
1588//
1589
1590unsigned &Value::getMetadataIndex() {
1591 if (auto *I = dyn_cast<Instruction>(Val: this))
1592 return I->MetadataIndex;
1593 return cast<GlobalObject>(Val: this)->MetadataIndex;
1594}
1595
1596unsigned Value::getMetadataIndex() const {
1597 return const_cast<Value *>(this)->getMetadataIndex();
1598}
1599
1600MDNode *Value::getMetadata(StringRef Kind) const {
1601 unsigned KindID = getContext().getMDKindID(Name: Kind);
1602 return getMetadataImpl(KindID);
1603}
1604
1605MDNode *Value::getMetadataImpl(unsigned KindID) const {
1606 const LLVMContext &Ctx = getContext();
1607 unsigned Idx = getMetadataIndex();
1608 while (Idx) {
1609 const MDAttachment &A = Ctx.pImpl->Metadatas[Idx];
1610 if (A.MDKind == KindID)
1611 return A.Node;
1612 Idx = A.Next;
1613 }
1614 return nullptr;
1615}
1616
1617void GlobalObject::getMetadata(unsigned KindID,
1618 SmallVectorImpl<MDNode *> &MDs) const {
1619 const LLVMContext &Ctx = getContext();
1620 unsigned Idx = MetadataIndex;
1621 while (Idx) {
1622 const MDAttachment &A = Ctx.pImpl->Metadatas[Idx];
1623 if (A.MDKind == KindID)
1624 MDs.push_back(Elt: A.Node);
1625 Idx = A.Next;
1626 }
1627 // We store metadata in reverse order, so reverse for output.
1628 std::reverse(first: MDs.begin(), last: MDs.end());
1629}
1630
1631void GlobalObject::getMetadata(StringRef Kind,
1632 SmallVectorImpl<MDNode *> &MDs) const {
1633 getMetadata(KindID: getContext().getMDKindID(Name: Kind), MDs);
1634}
1635
1636void Value::getAllMetadata(
1637 SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const {
1638 const LLVMContext &Ctx = getContext();
1639 unsigned Idx = getMetadataIndex();
1640 while (Idx) {
1641 const MDAttachment &A = Ctx.pImpl->Metadatas[Idx];
1642 MDs.emplace_back(Args: A.MDKind, Args: A.Node);
1643 Idx = A.Next;
1644 }
1645 // We store metadata in reverse order, so reverse for output in insertion
1646 // order. Sort by metadata ID for stable output.
1647 if (MDs.size() > 1) {
1648 std::reverse(first: MDs.begin(), last: MDs.end());
1649 llvm::stable_sort(Range&: MDs, C: less_first());
1650 }
1651}
1652
1653void Value::setMetadata(unsigned KindID, MDNode *Node) {
1654 assert(isa<Instruction>(this) || isa<GlobalObject>(this));
1655
1656 if (getMetadataIndex() != 0)
1657 eraseMetadata(KindID);
1658 if (Node)
1659 addMetadata(KindID, MD&: *Node);
1660}
1661
1662void Value::setMetadata(StringRef Kind, MDNode *Node) {
1663 if (!Node && getMetadataIndex() == 0)
1664 return;
1665 setMetadata(KindID: getContext().getMDKindID(Name: Kind), Node);
1666}
1667
1668void Value::addMetadata(unsigned KindID, MDNode &MD) {
1669 const LLVMContext &Ctx = getContext();
1670 unsigned &Idx = getMetadataIndex();
1671 unsigned NewIdx = Ctx.pImpl->MetadataRecycleHead;
1672 if (NewIdx == 0) {
1673 NewIdx = Ctx.pImpl->Metadatas.size();
1674 if (NewIdx == 0)
1675 NewIdx = 1;
1676 Ctx.pImpl->Metadatas.resize(N: NewIdx + 1);
1677 } else {
1678 Ctx.pImpl->MetadataRecycleHead = Ctx.pImpl->Metadatas[NewIdx].Next;
1679#ifndef NDEBUG
1680 Ctx.pImpl->MetadataRecycleSize -= 1;
1681#endif
1682 }
1683 Ctx.pImpl->Metadatas[NewIdx] =
1684 MDAttachment{.Next: Idx, .MDKind: KindID, .Node: TrackingMDNodeRef(&MD)};
1685 Idx = NewIdx;
1686}
1687
1688void Value::addMetadata(StringRef Kind, MDNode &MD) {
1689 addMetadata(KindID: getContext().getMDKindID(Name: Kind), MD);
1690}
1691
1692bool Value::eraseMetadata(unsigned KindID) {
1693 bool Changed = false;
1694 eraseMetadataIf(Pred: [&Changed, KindID](unsigned MDKind, MDNode *) {
1695 Changed |= MDKind == KindID;
1696 return MDKind == KindID;
1697 });
1698 return Changed;
1699}
1700
1701void Value::eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred) {
1702 unsigned *Idx = &getMetadataIndex();
1703 const LLVMContext &Ctx = getContext();
1704 while (*Idx) {
1705 MDAttachment &A = Ctx.pImpl->Metadatas[*Idx];
1706 if (Pred(A.MDKind, A.Node)) {
1707 A.Node.reset();
1708 unsigned FreeIdx = *Idx;
1709 *Idx = A.Next;
1710 A.Next = Ctx.pImpl->MetadataRecycleHead;
1711 Ctx.pImpl->MetadataRecycleHead = FreeIdx;
1712#ifndef NDEBUG
1713 Ctx.pImpl->MetadataRecycleSize += 1;
1714#endif
1715 } else {
1716 Idx = &A.Next;
1717 }
1718 }
1719}
1720
1721void Value::clearMetadata() {
1722 eraseMetadataIf(Pred: [](unsigned, MDNode *) { return true; });
1723}
1724
1725void Instruction::setMetadata(StringRef Kind, MDNode *Node) {
1726 if (!Node && MetadataIndex == 0)
1727 return;
1728 setMetadata(KindID: getContext().getMDKindID(Name: Kind), Node);
1729}
1730
1731MDNode *Instruction::getMetadataImpl(StringRef Kind) const {
1732 const LLVMContext &Ctx = getContext();
1733 unsigned KindID = Ctx.getMDKindID(Name: Kind);
1734 if (KindID == LLVMContext::MD_dbg)
1735 return DbgLoc.getAsMDNode();
1736 return Value::getMetadataImpl(KindID);
1737}
1738
1739void Instruction::eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred) {
1740 if (DbgLoc && Pred(LLVMContext::MD_dbg, DbgLoc.getAsMDNode()))
1741 DbgLoc = {};
1742
1743 Value::eraseMetadataIf(Pred);
1744}
1745
1746void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) {
1747 if (!hasMetadataOtherThanDebugLoc())
1748 return; // Nothing to remove!
1749
1750 SmallSet<unsigned, 32> KnownSet(llvm::from_range, KnownIDs);
1751
1752 // A DIAssignID attachment is debug metadata, don't drop it.
1753 KnownSet.insert(V: LLVMContext::MD_DIAssignID);
1754
1755 Value::eraseMetadataIf(Pred: [&KnownSet](unsigned MDKind, MDNode *Node) {
1756 return !KnownSet.count(V: MDKind);
1757 });
1758}
1759
1760void Instruction::updateDIAssignIDMapping(DIAssignID *ID) {
1761 if (auto *CurrentID =
1762 cast_or_null<DIAssignID>(Val: getMetadata(KindID: LLVMContext::MD_DIAssignID))) {
1763 if (ID == CurrentID)
1764 return;
1765 CurrentID->Instrs.erase(I: llvm::find(Range&: CurrentID->Instrs, Val: this));
1766 }
1767 if (ID)
1768 ID->Instrs.push_back(NewVal: this);
1769}
1770
1771void Instruction::setMetadata(unsigned KindID, MDNode *Node) {
1772 if (!Node && !hasMetadata())
1773 return;
1774
1775 // Handle 'dbg' as a special case since it is not stored in the hash table.
1776 if (KindID == LLVMContext::MD_dbg) {
1777 DbgLoc = DebugLoc(cast_or_null<DILocation>(Val: Node));
1778 return;
1779 }
1780
1781 // Update DIAssignID to Instruction(s) mapping.
1782 if (KindID == LLVMContext::MD_DIAssignID) {
1783 // The DIAssignID tracking infrastructure doesn't support RAUWing temporary
1784 // nodes with DIAssignIDs. The cast_or_null below would also catch this, but
1785 // having a dedicated assert helps make this obvious.
1786 assert((!Node || !Node->isTemporary()) &&
1787 "Temporary DIAssignIDs are invalid");
1788 updateDIAssignIDMapping(ID: cast_or_null<DIAssignID>(Val: Node));
1789 }
1790
1791 Value::setMetadata(KindID, Node);
1792}
1793
1794void Instruction::addAnnotationMetadata(SmallVector<StringRef> Annotations) {
1795 SmallVector<Metadata *, 4> Names;
1796 if (auto *Existing = getMetadata(KindID: LLVMContext::MD_annotation)) {
1797 SmallSetVector<StringRef, 2> AnnotationsSet(Annotations.begin(),
1798 Annotations.end());
1799 auto *Tuple = cast<MDTuple>(Val: Existing);
1800 for (auto &N : Tuple->operands()) {
1801 if (isa<MDString>(Val: N.get())) {
1802 Names.push_back(Elt: N);
1803 continue;
1804 }
1805 auto *MDAnnotationTuple = cast<MDTuple>(Val: N);
1806 if (any_of(Range: MDAnnotationTuple->operands(), P: [&AnnotationsSet](auto &Op) {
1807 return AnnotationsSet.contains(key: cast<MDString>(Op)->getString());
1808 }))
1809 return;
1810 Names.push_back(Elt: N);
1811 }
1812 }
1813
1814 MDBuilder MDB(getContext());
1815 SmallVector<Metadata *> MDAnnotationStrings;
1816 for (StringRef Annotation : Annotations)
1817 MDAnnotationStrings.push_back(Elt: MDB.createString(Str: Annotation));
1818 MDNode *InfoTuple = MDTuple::get(Context&: getContext(), MDs: MDAnnotationStrings);
1819 Names.push_back(Elt: InfoTuple);
1820 MDNode *MD = MDTuple::get(Context&: getContext(), MDs: Names);
1821 setMetadata(KindID: LLVMContext::MD_annotation, Node: MD);
1822}
1823
1824void Instruction::addAnnotationMetadata(StringRef Name) {
1825 SmallVector<Metadata *, 4> Names;
1826 if (auto *Existing = getMetadata(KindID: LLVMContext::MD_annotation)) {
1827 auto *Tuple = cast<MDTuple>(Val: Existing);
1828 for (auto &N : Tuple->operands()) {
1829 if (isa<MDString>(Val: N.get()) &&
1830 cast<MDString>(Val: N.get())->getString() == Name)
1831 return;
1832 Names.push_back(Elt: N.get());
1833 }
1834 }
1835
1836 MDBuilder MDB(getContext());
1837 Names.push_back(Elt: MDB.createString(Str: Name));
1838 MDNode *MD = MDTuple::get(Context&: getContext(), MDs: Names);
1839 setMetadata(KindID: LLVMContext::MD_annotation, Node: MD);
1840}
1841
1842AAMDNodes Instruction::getAAMetadata() const {
1843 AAMDNodes Result;
1844 if (hasMetadataOtherThanDebugLoc()) {
1845 unsigned Idx = MetadataIndex;
1846 const auto &Metadatas = getContext().pImpl->Metadatas;
1847 while (Idx) {
1848 const MDAttachment &A = Metadatas[Idx];
1849 switch (A.MDKind) {
1850 case LLVMContext::MD_tbaa:
1851 Result.TBAA = A.Node;
1852 break;
1853 case LLVMContext::MD_tbaa_struct:
1854 Result.TBAAStruct = A.Node;
1855 break;
1856 case LLVMContext::MD_alias_scope:
1857 Result.Scope = A.Node;
1858 break;
1859 case LLVMContext::MD_noalias:
1860 Result.NoAlias = A.Node;
1861 break;
1862 case LLVMContext::MD_noalias_addrspace:
1863 Result.NoAliasAddrSpace = A.Node;
1864 break;
1865 }
1866 Idx = A.Next;
1867 }
1868 }
1869 return Result;
1870}
1871
1872void Instruction::setAAMetadata(const AAMDNodes &N) {
1873 setMetadata(KindID: LLVMContext::MD_tbaa, Node: N.TBAA);
1874 setMetadata(KindID: LLVMContext::MD_tbaa_struct, Node: N.TBAAStruct);
1875 setMetadata(KindID: LLVMContext::MD_alias_scope, Node: N.Scope);
1876 setMetadata(KindID: LLVMContext::MD_noalias, Node: N.NoAlias);
1877 setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: N.NoAliasAddrSpace);
1878}
1879
1880void Instruction::setNoSanitizeMetadata() {
1881 setMetadata(KindID: llvm::LLVMContext::MD_nosanitize,
1882 Node: llvm::MDNode::get(Context&: getContext(), MDs: {}));
1883}
1884
1885void Instruction::getAllMetadataImpl(
1886 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
1887 Result.clear();
1888
1889 // Handle 'dbg' as a special case since it is not stored in the hash table.
1890 if (DbgLoc) {
1891 Result.push_back(
1892 Elt: std::make_pair(x: (unsigned)LLVMContext::MD_dbg, y: DbgLoc.getAsMDNode()));
1893 }
1894 Value::getAllMetadata(MDs&: Result);
1895}
1896
1897bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) const {
1898 assert((getOpcode() == Instruction::CondBr ||
1899 getOpcode() == Instruction::Select ||
1900 getOpcode() == Instruction::Call ||
1901 getOpcode() == Instruction::Invoke ||
1902 getOpcode() == Instruction::IndirectBr ||
1903 getOpcode() == Instruction::Switch) &&
1904 "Looking for branch weights on something besides branch");
1905
1906 return ::extractProfTotalWeight(I: *this, TotalWeights&: TotalVal);
1907}
1908
1909void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) {
1910 SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
1911 Other->getAllMetadata(MDs);
1912 for (auto &MD : MDs) {
1913 // We need to adjust the type metadata offset.
1914 if (Offset != 0 && MD.first == LLVMContext::MD_type) {
1915 auto *OffsetConst = cast<ConstantInt>(
1916 Val: cast<ConstantAsMetadata>(Val: MD.second->getOperand(I: 0))->getValue());
1917 Metadata *TypeId = MD.second->getOperand(I: 1);
1918 auto *NewOffsetMD = ConstantAsMetadata::get(C: ConstantInt::get(
1919 Ty: OffsetConst->getType(), V: OffsetConst->getValue() + Offset));
1920 addMetadata(KindID: LLVMContext::MD_type,
1921 MD&: *MDNode::get(Context&: getContext(), MDs: {NewOffsetMD, TypeId}));
1922 continue;
1923 }
1924 // If an offset adjustment was specified we need to modify the DIExpression
1925 // to prepend the adjustment:
1926 // !DIExpression(DW_OP_plus, Offset, [original expr])
1927 auto *Attachment = MD.second;
1928 if (Offset != 0 && MD.first == LLVMContext::MD_dbg) {
1929 DIGlobalVariable *GV = dyn_cast<DIGlobalVariable>(Val: Attachment);
1930 DIExpression *E = nullptr;
1931 if (!GV) {
1932 auto *GVE = cast<DIGlobalVariableExpression>(Val: Attachment);
1933 GV = GVE->getVariable();
1934 E = GVE->getExpression();
1935 }
1936 ArrayRef<uint64_t> OrigElements;
1937 if (E)
1938 OrigElements = E->getElements();
1939 std::vector<uint64_t> Elements(OrigElements.size() + 2);
1940 Elements[0] = dwarf::DW_OP_plus_uconst;
1941 Elements[1] = Offset;
1942 llvm::copy(Range&: OrigElements, Out: Elements.begin() + 2);
1943 E = DIExpression::get(Context&: getContext(), Elements);
1944 Attachment = DIGlobalVariableExpression::get(Context&: getContext(), Variable: GV, Expression: E);
1945 }
1946 addMetadata(KindID: MD.first, MD&: *Attachment);
1947 }
1948}
1949
1950void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) {
1951 addMetadata(
1952 KindID: LLVMContext::MD_type,
1953 MD&: *MDTuple::get(Context&: getContext(),
1954 MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1955 Ty: Type::getInt64Ty(C&: getContext()), V: Offset)),
1956 TypeID}));
1957}
1958
1959void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) {
1960 // Remove any existing vcall visibility metadata first in case we are
1961 // updating.
1962 eraseMetadata(KindID: LLVMContext::MD_vcall_visibility);
1963 addMetadata(KindID: LLVMContext::MD_vcall_visibility,
1964 MD&: *MDNode::get(Context&: getContext(),
1965 MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1966 Ty: Type::getInt64Ty(C&: getContext()), V: Visibility))}));
1967}
1968
1969GlobalObject::VCallVisibility GlobalObject::getVCallVisibility() const {
1970 if (MDNode *MD = getMetadata(KindID: LLVMContext::MD_vcall_visibility)) {
1971 uint64_t Val = cast<ConstantInt>(
1972 Val: cast<ConstantAsMetadata>(Val: MD->getOperand(I: 0))->getValue())
1973 ->getZExtValue();
1974 assert(Val <= 2 && "unknown vcall visibility!");
1975 return (VCallVisibility)Val;
1976 }
1977 return VCallVisibility::VCallVisibilityPublic;
1978}
1979
1980void Function::setSubprogram(DISubprogram *SP) {
1981 setMetadata(KindID: LLVMContext::MD_dbg, Node: SP);
1982}
1983
1984DISubprogram *Function::getSubprogram() const {
1985 return cast_or_null<DISubprogram>(Val: getMetadata(KindID: LLVMContext::MD_dbg));
1986}
1987
1988bool Function::shouldEmitDebugInfoForProfiling() const {
1989 if (DISubprogram *SP = getSubprogram()) {
1990 if (DICompileUnit *CU = SP->getUnit()) {
1991 return CU->getDebugInfoForProfiling();
1992 }
1993 }
1994 return false;
1995}
1996
1997void GlobalVariable::addDebugInfo(DIGlobalVariableExpression *GV) {
1998 addMetadata(KindID: LLVMContext::MD_dbg, MD&: *GV);
1999}
2000
2001void GlobalVariable::getDebugInfo(
2002 SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const {
2003 SmallVector<MDNode *, 1> MDs;
2004 getMetadata(KindID: LLVMContext::MD_dbg, MDs);
2005 for (MDNode *MD : MDs)
2006 GVs.push_back(Elt: cast<DIGlobalVariableExpression>(Val: MD));
2007}
2008