1//===- Metadata.cpp - Implement Metadata classes --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Metadata classes.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Metadata.h"
14#include "LLVMContextImpl.h"
15#include "MetadataImpl.h"
16#include "llvm/ADT/APFloat.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/DenseSet.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/ADT/Twine.h"
28#include "llvm/IR/Argument.h"
29#include "llvm/IR/BasicBlock.h"
30#include "llvm/IR/Constant.h"
31#include "llvm/IR/ConstantRange.h"
32#include "llvm/IR/ConstantRangeList.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DebugInfoMetadata.h"
35#include "llvm/IR/DebugLoc.h"
36#include "llvm/IR/DebugProgramInstruction.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/GlobalObject.h"
39#include "llvm/IR/GlobalVariable.h"
40#include "llvm/IR/Instruction.h"
41#include "llvm/IR/LLVMContext.h"
42#include "llvm/IR/MDBuilder.h"
43#include "llvm/IR/Module.h"
44#include "llvm/IR/ProfDataUtils.h"
45#include "llvm/IR/TrackingMDRef.h"
46#include "llvm/IR/Type.h"
47#include "llvm/IR/Value.h"
48#include "llvm/Support/Casting.h"
49#include "llvm/Support/CommandLine.h"
50
51#include "llvm/Support/ErrorHandling.h"
52#include "llvm/Support/MathExtras.h"
53#include "llvm/Support/ModRef.h"
54#include <cassert>
55#include <cstddef>
56#include <cstdint>
57#include <type_traits>
58#include <utility>
59#include <vector>
60
61using namespace llvm;
62
63namespace llvm {
64extern cl::opt<bool> ProfcheckDisableMetadataFixes;
65}
66
67MetadataAsValue::MetadataAsValue(Type *Ty, Metadata *MD)
68 : Value(Ty, MetadataAsValueVal), MD(MD) {
69 track();
70}
71
72MetadataAsValue::~MetadataAsValue() {
73 getType()->getContext().pImpl->MetadataAsValues.erase(Val: MD);
74 untrack();
75}
76
77/// Canonicalize metadata arguments to intrinsics.
78///
79/// To support bitcode upgrades (and assembly semantic sugar) for \a
80/// MetadataAsValue, we need to canonicalize certain metadata.
81///
82/// - nullptr is replaced by an empty MDNode.
83/// - An MDNode with a single null operand is replaced by an empty MDNode.
84/// - An MDNode whose only operand is a \a ConstantAsMetadata gets skipped.
85///
86/// This maintains readability of bitcode from when metadata was a type of
87/// value, and these bridges were unnecessary.
88static Metadata *canonicalizeMetadataForValue(LLVMContext &Context,
89 Metadata *MD) {
90 if (!MD)
91 // !{}
92 return MDNode::get(Context, MDs: {});
93
94 // Return early if this isn't a single-operand MDNode.
95 auto *N = dyn_cast<MDNode>(Val: MD);
96 if (!N || N->getNumOperands() != 1)
97 return MD;
98
99 if (!N->getOperand(I: 0))
100 // !{}
101 return MDNode::get(Context, MDs: {});
102
103 if (auto *C = dyn_cast<ConstantAsMetadata>(Val: N->getOperand(I: 0)))
104 // Look through the MDNode.
105 return C;
106
107 return MD;
108}
109
110MetadataAsValue *MetadataAsValue::get(LLVMContext &Context, Metadata *MD) {
111 MD = canonicalizeMetadataForValue(Context, MD);
112 auto *&Entry = Context.pImpl->MetadataAsValues[MD];
113 if (!Entry)
114 Entry = new MetadataAsValue(Type::getMetadataTy(C&: Context), MD);
115 return Entry;
116}
117
118MetadataAsValue *MetadataAsValue::getIfExists(LLVMContext &Context,
119 Metadata *MD) {
120 MD = canonicalizeMetadataForValue(Context, MD);
121 auto &Store = Context.pImpl->MetadataAsValues;
122 return Store.lookup(Val: MD);
123}
124
125void MetadataAsValue::handleChangedMetadata(Metadata *MD) {
126 LLVMContext &Context = getContext();
127 MD = canonicalizeMetadataForValue(Context, MD);
128 auto &Store = Context.pImpl->MetadataAsValues;
129
130 // Stop tracking the old metadata.
131 Store.erase(Val: this->MD);
132 untrack();
133 this->MD = nullptr;
134
135 // Start tracking MD, or RAUW if necessary.
136 auto *&Entry = Store[MD];
137 if (Entry) {
138 replaceAllUsesWith(V: Entry);
139 delete this;
140 return;
141 }
142
143 this->MD = MD;
144 track();
145 Entry = this;
146}
147
148void MetadataAsValue::track() {
149 if (MD)
150 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
151}
152
153void MetadataAsValue::untrack() {
154 if (MD)
155 MetadataTracking::untrack(MD);
156}
157
158DbgVariableRecord *DebugValueUser::getUser() {
159 return static_cast<DbgVariableRecord *>(this);
160}
161const DbgVariableRecord *DebugValueUser::getUser() const {
162 return static_cast<const DbgVariableRecord *>(this);
163}
164
165void DebugValueUser::handleChangedValue(void *Old, Metadata *New) {
166 // NOTE: We could inform the "owner" that a value has changed through
167 // getOwner, if needed.
168 auto OldMD = static_cast<Metadata **>(Old);
169 ptrdiff_t Idx = std::distance(first: &*DebugValues.begin(), last: OldMD);
170 // If replacing a ValueAsMetadata with a nullptr, replace it with a
171 // PoisonValue instead.
172 if (OldMD && isa<ValueAsMetadata>(Val: *OldMD) && !New) {
173 auto *OldVAM = cast<ValueAsMetadata>(Val: *OldMD);
174 New = ValueAsMetadata::get(V: PoisonValue::get(T: OldVAM->getValue()->getType()));
175 }
176 resetDebugValue(Idx, DebugValue: New);
177}
178
179void DebugValueUser::trackDebugValue(size_t Idx) {
180 assert(Idx < 3 && "Invalid debug value index.");
181 Metadata *&MD = DebugValues[Idx];
182 if (MD)
183 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
184}
185
186void DebugValueUser::trackDebugValues() {
187 for (Metadata *&MD : DebugValues)
188 if (MD)
189 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
190}
191
192void DebugValueUser::untrackDebugValue(size_t Idx) {
193 assert(Idx < 3 && "Invalid debug value index.");
194 Metadata *&MD = DebugValues[Idx];
195 if (MD)
196 MetadataTracking::untrack(MD);
197}
198
199void DebugValueUser::untrackDebugValues() {
200 for (Metadata *&MD : DebugValues)
201 if (MD)
202 MetadataTracking::untrack(MD);
203}
204
205void DebugValueUser::retrackDebugValues(DebugValueUser &X) {
206 assert(DebugValueUser::operator==(X) && "Expected values to match");
207 for (const auto &[MD, XMD] : zip(t&: DebugValues, u&: X.DebugValues))
208 if (XMD)
209 MetadataTracking::retrack(MD&: XMD, New&: MD);
210 X.DebugValues.fill(u: nullptr);
211}
212
213bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) {
214 assert(Ref && "Expected live reference");
215 assert((Owner || *static_cast<Metadata **>(Ref) == &MD) &&
216 "Reference without owner must be direct");
217 if (auto *R = ReplaceableMetadataImpl::getOrCreate(MD)) {
218 R->addRef(Ref, Owner);
219 return true;
220 }
221 if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(Val: &MD)) {
222 assert(!PH->Use && "Placeholders can only be used once");
223 assert(!Owner && "Unexpected callback to owner");
224 PH->Use = static_cast<Metadata **>(Ref);
225 return true;
226 }
227 return false;
228}
229
230void MetadataTracking::untrack(void *Ref, Metadata &MD) {
231 assert(Ref && "Expected live reference");
232 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD))
233 R->dropRef(Ref);
234 else if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(Val: &MD))
235 PH->Use = nullptr;
236}
237
238bool MetadataTracking::retrack(void *Ref, Metadata &MD, void *New) {
239 assert(Ref && "Expected live reference");
240 assert(New && "Expected live reference");
241 assert(Ref != New && "Expected change");
242 if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) {
243 R->moveRef(Ref, New, MD);
244 return true;
245 }
246 assert(!isa<DistinctMDOperandPlaceholder>(MD) &&
247 "Unexpected move of an MDOperand");
248 assert(!isReplaceable(MD) &&
249 "Expected un-replaceable metadata, since we didn't move a reference");
250 return false;
251}
252
253bool MetadataTracking::isReplaceable(const Metadata &MD) {
254 return ReplaceableMetadataImpl::isReplaceable(MD);
255}
256
257SmallVector<Metadata *> ReplaceableMetadataImpl::getAllArgListUsers() {
258 SmallVector<std::pair<OwnerTy, uint64_t> *> MDUsersWithID;
259 for (auto Pair : UseMap) {
260 OwnerTy Owner = Pair.second.first;
261 if (Owner.isNull())
262 continue;
263 if (!isa<Metadata *>(Val: Owner))
264 continue;
265 Metadata *OwnerMD = cast<Metadata *>(Val&: Owner);
266 if (OwnerMD->getMetadataID() == Metadata::DIArgListKind)
267 MDUsersWithID.push_back(Elt: &UseMap[Pair.first]);
268 }
269 llvm::sort(C&: MDUsersWithID, Comp: [](auto UserA, auto UserB) {
270 return UserA->second < UserB->second;
271 });
272 SmallVector<Metadata *> MDUsers;
273 for (auto *UserWithID : MDUsersWithID)
274 MDUsers.push_back(Elt: cast<Metadata *>(Val&: UserWithID->first));
275 return MDUsers;
276}
277
278SmallVector<DbgVariableRecord *>
279ReplaceableMetadataImpl::getAllDbgVariableRecordUsers() {
280 SmallVector<std::pair<OwnerTy, uint64_t> *> DVRUsersWithID;
281 for (auto Pair : UseMap) {
282 OwnerTy Owner = Pair.second.first;
283 if (Owner.isNull())
284 continue;
285 if (!isa<DebugValueUser *>(Val: Owner))
286 continue;
287 DVRUsersWithID.push_back(Elt: &UseMap[Pair.first]);
288 }
289 // Order DbgVariableRecord users in reverse-creation order. Normal dbg.value
290 // users of MetadataAsValues are ordered by their UseList, i.e. reverse order
291 // of when they were added: we need to replicate that here. The structure of
292 // debug-info output depends on the ordering of intrinsics, thus we need
293 // to keep them consistent for comparisons sake.
294 llvm::sort(C&: DVRUsersWithID, Comp: [](auto UserA, auto UserB) {
295 return UserA->second > UserB->second;
296 });
297 SmallVector<DbgVariableRecord *> DVRUsers;
298 for (auto UserWithID : DVRUsersWithID)
299 DVRUsers.push_back(Elt: cast<DebugValueUser *>(Val&: UserWithID->first)->getUser());
300 return DVRUsers;
301}
302
303void ReplaceableMetadataImpl::addRef(void *Ref, OwnerTy Owner) {
304 bool WasInserted =
305 UseMap.insert(KV: std::make_pair(x&: Ref, y: std::make_pair(x&: Owner, y&: NextIndex)))
306 .second;
307 (void)WasInserted;
308 assert(WasInserted && "Expected to add a reference");
309
310 ++NextIndex;
311 assert(NextIndex != 0 && "Unexpected overflow");
312}
313
314void ReplaceableMetadataImpl::dropRef(void *Ref) {
315 bool WasErased = UseMap.erase(Val: Ref);
316 (void)WasErased;
317 assert(WasErased && "Expected to drop a reference");
318}
319
320void ReplaceableMetadataImpl::moveRef(void *Ref, void *New,
321 const Metadata &MD) {
322 auto I = UseMap.find(Val: Ref);
323 assert(I != UseMap.end() && "Expected to move a reference");
324 auto OwnerAndIndex = I->second;
325 UseMap.erase(I);
326 bool WasInserted = UseMap.insert(KV: std::make_pair(x&: New, y&: OwnerAndIndex)).second;
327 (void)WasInserted;
328 assert(WasInserted && "Expected to add a reference");
329
330 // Check that the references are direct if there's no owner.
331 (void)MD;
332 assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) &&
333 "Reference without owner must be direct");
334 assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) &&
335 "Reference without owner must be direct");
336}
337
338void ReplaceableMetadataImpl::SalvageDebugInfo(const Constant &C) {
339 if (!C.isUsedByMetadata()) {
340 return;
341 }
342
343 LLVMContext &Context = C.getType()->getContext();
344 auto &Store = Context.pImpl->ValuesAsMetadata;
345 auto I = Store.find(Val: &C);
346 ValueAsMetadata *MD = I->second;
347 using UseTy =
348 std::pair<void *, std::pair<MetadataTracking::OwnerTy, uint64_t>>;
349 // Copy out uses and update value of Constant used by debug info metadata with
350 // poison below
351 SmallVector<UseTy, 8> Uses(MD->UseMap.begin(), MD->UseMap.end());
352
353 for (const auto &Pair : Uses) {
354 MetadataTracking::OwnerTy Owner = Pair.second.first;
355 if (!Owner)
356 continue;
357 // Check for MetadataAsValue.
358 if (isa<MetadataAsValue *>(Val: Owner)) {
359 cast<MetadataAsValue *>(Val&: Owner)->handleChangedMetadata(
360 MD: ValueAsMetadata::get(V: PoisonValue::get(T: C.getType())));
361 continue;
362 }
363 if (!isa<Metadata *>(Val: Owner))
364 continue;
365 auto *OwnerMD = dyn_cast_if_present<MDNode>(Val: cast<Metadata *>(Val&: Owner));
366 if (!OwnerMD)
367 continue;
368 if (isa<DINode>(Val: OwnerMD)) {
369 OwnerMD->handleChangedOperand(
370 Ref: Pair.first, New: ValueAsMetadata::get(V: PoisonValue::get(T: C.getType())));
371 }
372 }
373}
374
375void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) {
376 if (UseMap.empty())
377 return;
378
379 // Copy out uses since UseMap will get touched below.
380 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
381 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
382 llvm::sort(C&: Uses, Comp: [](const UseTy &L, const UseTy &R) {
383 return L.second.second < R.second.second;
384 });
385 for (const auto &Pair : Uses) {
386 // Check that this Ref hasn't disappeared after RAUW (when updating a
387 // previous Ref).
388 if (!UseMap.count(Val: Pair.first))
389 continue;
390
391 OwnerTy Owner = Pair.second.first;
392 if (!Owner) {
393 // Update unowned tracking references directly.
394 Metadata *&Ref = *static_cast<Metadata **>(Pair.first);
395 Ref = MD;
396 if (MD)
397 MetadataTracking::track(MD&: Ref);
398 UseMap.erase(Val: Pair.first);
399 continue;
400 }
401
402 // Check for MetadataAsValue.
403 if (isa<MetadataAsValue *>(Val: Owner)) {
404 cast<MetadataAsValue *>(Val&: Owner)->handleChangedMetadata(MD);
405 continue;
406 }
407
408 if (auto *DVU = dyn_cast<DebugValueUser *>(Val&: Owner)) {
409 DVU->handleChangedValue(Old: Pair.first, New: MD);
410 continue;
411 }
412
413 // There's a Metadata owner -- dispatch.
414 Metadata *OwnerMD = cast<Metadata *>(Val&: Owner);
415 switch (OwnerMD->getMetadataID()) {
416#define HANDLE_METADATA_LEAF(CLASS) \
417 case Metadata::CLASS##Kind: \
418 cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \
419 continue;
420#include "llvm/IR/Metadata.def"
421 default:
422 llvm_unreachable("Invalid metadata subclass");
423 }
424 }
425 assert(UseMap.empty() && "Expected all uses to be replaced");
426}
427
428void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) {
429 if (UseMap.empty())
430 return;
431
432 if (!ResolveUsers) {
433 UseMap.clear();
434 return;
435 }
436
437 // Copy out uses since UseMap could get touched below.
438 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
439 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
440 llvm::sort(C&: Uses, Comp: [](const UseTy &L, const UseTy &R) {
441 return L.second.second < R.second.second;
442 });
443 UseMap.clear();
444 for (const auto &Pair : Uses) {
445 auto Owner = Pair.second.first;
446 if (!Owner)
447 continue;
448 if (!isa<Metadata *>(Val: Owner))
449 continue;
450
451 // Resolve MDNodes that point at this.
452 auto *OwnerMD = dyn_cast_if_present<MDNode>(Val: cast<Metadata *>(Val&: Owner));
453 if (!OwnerMD)
454 continue;
455 if (OwnerMD->isResolved())
456 continue;
457 OwnerMD->decrementUnresolvedOperandCount();
458 }
459}
460
461// Special handing of DIArgList is required in the RemoveDIs project, see
462// commentry in DIArgList::handleChangedOperand for details. Hidden behind
463// conditional compilation to avoid a compile time regression.
464ReplaceableMetadataImpl *ReplaceableMetadataImpl::getOrCreate(Metadata &MD) {
465 if (auto *N = dyn_cast<MDNode>(Val: &MD)) {
466 return !N->isResolved() || N->isAlwaysReplaceable()
467 ? N->Context.getOrCreateReplaceableUses()
468 : nullptr;
469 }
470 if (auto ArgList = dyn_cast<DIArgList>(Val: &MD))
471 return ArgList;
472 return dyn_cast<ValueAsMetadata>(Val: &MD);
473}
474
475ReplaceableMetadataImpl *ReplaceableMetadataImpl::getIfExists(Metadata &MD) {
476 if (auto *N = dyn_cast<MDNode>(Val: &MD)) {
477 return !N->isResolved() || N->isAlwaysReplaceable()
478 ? N->Context.getReplaceableUses()
479 : nullptr;
480 }
481 if (auto ArgList = dyn_cast<DIArgList>(Val: &MD))
482 return ArgList;
483 return dyn_cast<ValueAsMetadata>(Val: &MD);
484}
485
486bool ReplaceableMetadataImpl::isReplaceable(const Metadata &MD) {
487 if (auto *N = dyn_cast<MDNode>(Val: &MD))
488 return !N->isResolved() || N->isAlwaysReplaceable();
489 return isa<ValueAsMetadata>(Val: &MD) || isa<DIArgList>(Val: &MD);
490}
491
492static DISubprogram *getLocalFunctionMetadata(Value *V) {
493 assert(V && "Expected value");
494 if (auto *A = dyn_cast<Argument>(Val: V)) {
495 if (auto *Fn = A->getParent())
496 return Fn->getSubprogram();
497 return nullptr;
498 }
499
500 if (BasicBlock *BB = cast<Instruction>(Val: V)->getParent()) {
501 if (auto *Fn = BB->getParent())
502 return Fn->getSubprogram();
503 return nullptr;
504 }
505
506 return nullptr;
507}
508
509ValueAsMetadata *ValueAsMetadata::get(Value *V) {
510 assert(V && "Unexpected null Value");
511
512 auto &Context = V->getContext();
513 auto *&Entry = Context.pImpl->ValuesAsMetadata[V];
514 if (!Entry) {
515 assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) &&
516 "Expected constant or function-local value");
517 assert(!V->IsUsedByMD && "Expected this to be the only metadata use");
518 V->IsUsedByMD = true;
519 if (auto *C = dyn_cast<Constant>(Val: V))
520 Entry = new ConstantAsMetadata(C);
521 else
522 Entry = new LocalAsMetadata(V);
523 }
524
525 return Entry;
526}
527
528ValueAsMetadata *ValueAsMetadata::getIfExists(Value *V) {
529 assert(V && "Unexpected null Value");
530 return V->getContext().pImpl->ValuesAsMetadata.lookup(Val: V);
531}
532
533void ValueAsMetadata::handleDeletion(Value *V) {
534 assert(V && "Expected valid value");
535
536 auto &Store = V->getType()->getContext().pImpl->ValuesAsMetadata;
537 auto I = Store.find(Val: V);
538 if (I == Store.end())
539 return;
540
541 // Remove old entry from the map.
542 ValueAsMetadata *MD = I->second;
543 assert(MD && "Expected valid metadata");
544 assert(MD->getValue() == V && "Expected valid mapping");
545 Store.erase(I);
546
547 // Delete the metadata.
548 MD->replaceAllUsesWith(MD: nullptr);
549 delete MD;
550}
551
552void ValueAsMetadata::handleRAUW(Value *From, Value *To) {
553 assert(From && "Expected valid value");
554 assert(To && "Expected valid value");
555 assert(From != To && "Expected changed value");
556 assert(&From->getContext() == &To->getContext() && "Expected same context");
557
558 LLVMContext &Context = From->getType()->getContext();
559 auto &Store = Context.pImpl->ValuesAsMetadata;
560 auto I = Store.find(Val: From);
561 if (I == Store.end()) {
562 assert(!From->IsUsedByMD && "Expected From not to be used by metadata");
563 return;
564 }
565
566 // Remove old entry from the map.
567 assert(From->IsUsedByMD && "Expected From to be used by metadata");
568 From->IsUsedByMD = false;
569 ValueAsMetadata *MD = I->second;
570 assert(MD && "Expected valid metadata");
571 assert(MD->getValue() == From && "Expected valid mapping");
572 Store.erase(I);
573
574 if (isa<LocalAsMetadata>(Val: MD)) {
575 if (auto *C = dyn_cast<Constant>(Val: To)) {
576 // Local became a constant.
577 MD->replaceAllUsesWith(MD: ConstantAsMetadata::get(C));
578 delete MD;
579 return;
580 }
581 if (getLocalFunctionMetadata(V: From) && getLocalFunctionMetadata(V: To) &&
582 getLocalFunctionMetadata(V: From) != getLocalFunctionMetadata(V: To)) {
583 // DISubprogram changed.
584 MD->replaceAllUsesWith(MD: nullptr);
585 delete MD;
586 return;
587 }
588 } else if (!isa<Constant>(Val: To)) {
589 // Changed to function-local value.
590 MD->replaceAllUsesWith(MD: nullptr);
591 delete MD;
592 return;
593 }
594
595 auto *&Entry = Store[To];
596 if (Entry) {
597 // The target already exists.
598 MD->replaceAllUsesWith(MD: Entry);
599 delete MD;
600 return;
601 }
602
603 // Update MD in place (and update the map entry).
604 assert(!To->IsUsedByMD && "Expected this to be the only metadata use");
605 To->IsUsedByMD = true;
606 MD->V = To;
607 Entry = MD;
608}
609
610//===----------------------------------------------------------------------===//
611// MDString implementation.
612//
613
614MDString *MDString::get(LLVMContext &Context, StringRef Str) {
615 auto &Store = Context.pImpl->MDStringCache;
616 auto I = Store.try_emplace(Key: Str);
617 auto &MapEntry = I.first->getValue();
618 if (!I.second)
619 return &MapEntry;
620 MapEntry.Entry = &*I.first;
621 return &MapEntry;
622}
623
624MDString *MDString::getIfExists(LLVMContext &Context, StringRef Str) {
625 auto &Store = Context.pImpl->MDStringCache;
626 auto I = Store.find(Key: Str);
627 if (I == Store.end())
628 return nullptr;
629 return &I->getValue();
630}
631
632StringRef MDString::getString() const {
633 assert(Entry && "Expected to find string map entry");
634 return Entry->first();
635}
636
637//===----------------------------------------------------------------------===//
638// MDNode implementation.
639//
640
641// Assert that the MDNode types will not be unaligned by the objects
642// prepended to them.
643#define HANDLE_MDNODE_LEAF(CLASS) \
644 static_assert( \
645 alignof(uint64_t) >= alignof(CLASS), \
646 "Alignment is insufficient after objects prepended to " #CLASS);
647#include "llvm/IR/Metadata.def"
648
649void *MDNode::operator new(size_t Size, size_t NumOps, StorageType Storage) {
650 // uint64_t is the most aligned type we need support (ensured by static_assert
651 // above)
652 size_t AllocSize =
653 alignTo(Value: Header::getAllocSize(Storage, NumOps), Align: alignof(uint64_t));
654 char *Mem = reinterpret_cast<char *>(::operator new(AllocSize + Size));
655 Header *H = new (Mem + AllocSize - sizeof(Header)) Header(NumOps, Storage);
656 return reinterpret_cast<void *>(H + 1);
657}
658
659void MDNode::operator delete(void *N) {
660 Header *H = reinterpret_cast<Header *>(N) - 1;
661 void *Mem = H->getAllocation();
662 H->~Header();
663 ::operator delete(Mem);
664}
665
666MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage,
667 ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2)
668 : Metadata(ID, Storage), Context(Context) {
669 unsigned Op = 0;
670 for (Metadata *MD : Ops1)
671 setOperand(I: Op++, New: MD);
672 for (Metadata *MD : Ops2)
673 setOperand(I: Op++, New: MD);
674
675 if (!isUniqued())
676 return;
677
678 // Count the unresolved operands. If there are any, RAUW support will be
679 // added lazily on first reference.
680 countUnresolvedOperands();
681}
682
683TempMDNode MDNode::clone() const {
684 switch (getMetadataID()) {
685 default:
686 llvm_unreachable("Invalid MDNode subclass");
687#define HANDLE_MDNODE_LEAF(CLASS) \
688 case CLASS##Kind: \
689 return cast<CLASS>(this)->cloneImpl();
690#include "llvm/IR/Metadata.def"
691 }
692}
693
694MDNode::Header::Header(size_t NumOps, StorageType Storage) {
695 IsLarge = isLarge(NumOps);
696 IsResizable = isResizable(Storage);
697 SmallSize = getSmallSize(NumOps, IsResizable, IsLarge);
698 if (IsLarge) {
699 SmallNumOps = 0;
700 new (getLargePtr()) LargeStorageVector();
701 getLarge().resize(N: NumOps);
702 return;
703 }
704 SmallNumOps = NumOps;
705 MDOperand *O = reinterpret_cast<MDOperand *>(this) - SmallSize;
706 for (MDOperand *E = O + SmallSize; O != E;)
707 (void)new (O++) MDOperand();
708}
709
710MDNode::Header::~Header() {
711 if (IsLarge) {
712 getLarge().~LargeStorageVector();
713 return;
714 }
715 MDOperand *O = reinterpret_cast<MDOperand *>(this);
716 for (MDOperand *E = O - SmallSize; O != E; --O)
717 (O - 1)->~MDOperand();
718}
719
720void *MDNode::Header::getSmallPtr() {
721 static_assert(alignof(MDOperand) <= alignof(Header),
722 "MDOperand too strongly aligned");
723 return reinterpret_cast<char *>(const_cast<Header *>(this)) -
724 sizeof(MDOperand) * SmallSize;
725}
726
727void MDNode::Header::resize(size_t NumOps) {
728 assert(IsResizable && "Node is not resizable");
729 if (operands().size() == NumOps)
730 return;
731
732 if (IsLarge)
733 getLarge().resize(N: NumOps);
734 else if (NumOps <= SmallSize)
735 resizeSmall(NumOps);
736 else
737 resizeSmallToLarge(NumOps);
738}
739
740void MDNode::Header::resizeSmall(size_t NumOps) {
741 assert(!IsLarge && "Expected a small MDNode");
742 assert(NumOps <= SmallSize && "NumOps too large for small resize");
743
744 MutableArrayRef<MDOperand> ExistingOps = operands();
745 assert(NumOps != ExistingOps.size() && "Expected a different size");
746
747 int NumNew = (int)NumOps - (int)ExistingOps.size();
748 MDOperand *O = ExistingOps.end();
749 for (int I = 0, E = NumNew; I < E; ++I)
750 (O++)->reset();
751 for (int I = 0, E = NumNew; I > E; --I)
752 (--O)->reset();
753 SmallNumOps = NumOps;
754 assert(O == operands().end() && "Operands not (un)initialized until the end");
755}
756
757void MDNode::Header::resizeSmallToLarge(size_t NumOps) {
758 assert(!IsLarge && "Expected a small MDNode");
759 assert(NumOps > SmallSize && "Expected NumOps to be larger than allocation");
760 LargeStorageVector NewOps;
761 NewOps.resize(N: NumOps);
762 llvm::move(Range: operands(), Out: NewOps.begin());
763 resizeSmall(NumOps: 0);
764 new (getLargePtr()) LargeStorageVector(std::move(NewOps));
765 IsLarge = true;
766}
767
768static bool isOperandUnresolved(Metadata *Op) {
769 if (auto *N = dyn_cast_or_null<MDNode>(Val: Op))
770 return !N->isResolved();
771 return false;
772}
773
774void MDNode::countUnresolvedOperands() {
775 assert(getNumUnresolved() == 0 && "Expected unresolved ops to be uncounted");
776 assert(isUniqued() && "Expected this to be uniqued");
777 setNumUnresolved(count_if(Range: operands(), P: isOperandUnresolved));
778}
779
780void MDNode::makeUniqued() {
781 assert(isTemporary() && "Expected this to be temporary");
782 assert(!isResolved() && "Expected this to be unresolved");
783
784 // Enable uniquing callbacks.
785 for (auto &Op : mutable_operands())
786 Op.reset(MD: Op.get(), Owner: this);
787
788 // Make this 'uniqued'.
789 Storage = Uniqued;
790 countUnresolvedOperands();
791 if (!getNumUnresolved()) {
792 dropReplaceableUses();
793 assert(isResolved() && "Expected this to be resolved");
794 }
795
796 assert(isUniqued() && "Expected this to be uniqued");
797}
798
799void MDNode::makeDistinct() {
800 assert(isTemporary() && "Expected this to be temporary");
801 assert(!isResolved() && "Expected this to be unresolved");
802
803 // Drop RAUW support and store as a distinct node.
804 dropReplaceableUses();
805 storeDistinctInContext();
806
807 assert(isDistinct() && "Expected this to be distinct");
808 assert(isResolved() && "Expected this to be resolved");
809}
810
811void MDNode::resolve() {
812 assert(isUniqued() && "Expected this to be uniqued");
813 assert(!isResolved() && "Expected this to be unresolved");
814
815 setNumUnresolved(0);
816 dropReplaceableUses();
817
818 assert(isResolved() && "Expected this to be resolved");
819}
820
821void MDNode::dropReplaceableUses() {
822 assert(!getNumUnresolved() && "Unexpected unresolved operand");
823
824 // Drop any RAUW support.
825 if (Context.hasReplaceableUses())
826 Context.takeReplaceableUses()->resolveAllUses();
827}
828
829void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) {
830 assert(isUniqued() && "Expected this to be uniqued");
831 assert(getNumUnresolved() != 0 && "Expected unresolved operands");
832
833 // Check if an operand was resolved.
834 if (!isOperandUnresolved(Op: Old)) {
835 if (isOperandUnresolved(Op: New))
836 // An operand was un-resolved!
837 setNumUnresolved(getNumUnresolved() + 1);
838 } else if (!isOperandUnresolved(Op: New))
839 decrementUnresolvedOperandCount();
840}
841
842void MDNode::decrementUnresolvedOperandCount() {
843 assert(!isResolved() && "Expected this to be unresolved");
844 if (isTemporary())
845 return;
846
847 assert(isUniqued() && "Expected this to be uniqued");
848 setNumUnresolved(getNumUnresolved() - 1);
849 if (getNumUnresolved())
850 return;
851
852 // Last unresolved operand has just been resolved.
853 dropReplaceableUses();
854 assert(isResolved() && "Expected this to become resolved");
855}
856
857void MDNode::resolveCycles() {
858 if (isResolved())
859 return;
860
861 // Resolve this node immediately.
862 resolve();
863
864 // Resolve all operands.
865 for (const auto &Op : operands()) {
866 auto *N = dyn_cast_or_null<MDNode>(Val: Op);
867 if (!N)
868 continue;
869
870 assert(!N->isTemporary() &&
871 "Expected all forward declarations to be resolved");
872 if (!N->isResolved())
873 N->resolveCycles();
874 }
875}
876
877static bool hasSelfReference(MDNode *N) {
878 return llvm::is_contained(Range: N->operands(), Element: N);
879}
880
881MDNode *MDNode::replaceWithPermanentImpl() {
882 switch (getMetadataID()) {
883 default:
884 // If this type isn't uniquable, replace with a distinct node.
885 return replaceWithDistinctImpl();
886
887#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
888 case CLASS##Kind: \
889 break;
890#include "llvm/IR/Metadata.def"
891 }
892
893 // Even if this type is uniquable, self-references have to be distinct.
894 if (hasSelfReference(N: this))
895 return replaceWithDistinctImpl();
896 return replaceWithUniquedImpl();
897}
898
899MDNode *MDNode::replaceWithUniquedImpl() {
900 // Try to uniquify in place.
901 MDNode *UniquedNode = uniquify();
902
903 if (UniquedNode == this) {
904 makeUniqued();
905 return this;
906 }
907
908 // Collision, so RAUW instead.
909 replaceAllUsesWith(MD: UniquedNode);
910 deleteAsSubclass();
911 return UniquedNode;
912}
913
914MDNode *MDNode::replaceWithDistinctImpl() {
915 makeDistinct();
916 return this;
917}
918
919void MDTuple::recalculateHash() {
920 setHash(MDTupleInfo::KeyTy::calculateHash(N: this));
921}
922
923void MDNode::dropAllReferences() {
924 for (unsigned I = 0, E = getNumOperands(); I != E; ++I)
925 setOperand(I, New: nullptr);
926 if (Context.hasReplaceableUses()) {
927 Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false);
928 (void)Context.takeReplaceableUses();
929 }
930}
931
932void MDNode::handleChangedOperand(void *Ref, Metadata *New) {
933 unsigned Op = static_cast<MDOperand *>(Ref) - op_begin();
934 assert(Op < getNumOperands() && "Expected valid operand");
935
936 if (!isUniqued()) {
937 // This node is not uniqued. Just set the operand and be done with it.
938 setOperand(I: Op, New);
939 return;
940 }
941
942 // This node is uniqued.
943 eraseFromStore();
944
945 Metadata *Old = getOperand(I: Op);
946 setOperand(I: Op, New);
947
948 // Drop uniquing for self-reference cycles and deleted constants.
949 if (New == this || (!New && Old && isa<ConstantAsMetadata>(Val: Old))) {
950 if (!isResolved())
951 resolve();
952 storeDistinctInContext();
953 return;
954 }
955
956 // Re-unique the node.
957 auto *Uniqued = uniquify();
958 if (Uniqued == this) {
959 if (!isResolved())
960 resolveAfterOperandChange(Old, New);
961 return;
962 }
963
964 // Collision.
965 if (!isResolved()) {
966 // Still unresolved, so RAUW.
967 //
968 // First, clear out all operands to prevent any recursion (similar to
969 // dropAllReferences(), but we still need the use-list).
970 for (unsigned O = 0, E = getNumOperands(); O != E; ++O)
971 setOperand(I: O, New: nullptr);
972 if (Context.hasReplaceableUses())
973 Context.getReplaceableUses()->replaceAllUsesWith(MD: Uniqued);
974 deleteAsSubclass();
975 return;
976 }
977
978 // Store in non-uniqued form if RAUW isn't possible.
979 storeDistinctInContext();
980}
981
982void MDNode::deleteAsSubclass() {
983 switch (getMetadataID()) {
984 default:
985 llvm_unreachable("Invalid subclass of MDNode");
986#define HANDLE_MDNODE_LEAF(CLASS) \
987 case CLASS##Kind: \
988 delete cast<CLASS>(this); \
989 break;
990#include "llvm/IR/Metadata.def"
991 }
992}
993
994template <class T, class InfoT>
995static T *uniquifyImpl(T *N, DenseSet<T *, InfoT> &Store) {
996 if (T *U = getUniqued(Store, N))
997 return U;
998
999 Store.insert(N);
1000 return N;
1001}
1002
1003template <class NodeTy> struct MDNode::HasCachedHash {
1004 template <class U>
1005 static std::true_type check(SameType<void (U::*)(unsigned), &U::setHash> *);
1006 template <class U> static std::false_type check(...);
1007
1008 static constexpr bool value = decltype(check<NodeTy>(nullptr))::value;
1009};
1010
1011MDNode *MDNode::uniquify() {
1012 assert(!hasSelfReference(this) && "Cannot uniquify a self-referencing node");
1013
1014 // Try to insert into uniquing store.
1015 switch (getMetadataID()) {
1016 default:
1017 llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
1018#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
1019 case CLASS##Kind: { \
1020 CLASS *SubclassThis = cast<CLASS>(this); \
1021 dispatchRecalculateHash(SubclassThis); \
1022 return uniquifyImpl(SubclassThis, getContext().pImpl->CLASS##s); \
1023 }
1024#include "llvm/IR/Metadata.def"
1025 }
1026}
1027
1028void MDNode::eraseFromStore() {
1029 switch (getMetadataID()) {
1030 default:
1031 llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
1032#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
1033 case CLASS##Kind: \
1034 getContext().pImpl->CLASS##s.erase(cast<CLASS>(this)); \
1035 break;
1036#include "llvm/IR/Metadata.def"
1037 }
1038}
1039
1040MDTuple *MDTuple::getImpl(LLVMContext &Context, ArrayRef<Metadata *> MDs,
1041 StorageType Storage, bool ShouldCreate) {
1042 unsigned Hash = 0;
1043 if (Storage == Uniqued) {
1044 MDTupleInfo::KeyTy Key(MDs);
1045 if (auto *N = getUniqued(Store&: Context.pImpl->MDTuples, Key))
1046 return N;
1047 if (!ShouldCreate)
1048 return nullptr;
1049 Hash = Key.getHash();
1050 } else {
1051 assert(ShouldCreate && "Expected non-uniqued nodes to always be created");
1052 }
1053
1054 return storeImpl(N: new (MDs.size(), Storage)
1055 MDTuple(Context, Storage, Hash, MDs),
1056 Storage, Store&: Context.pImpl->MDTuples);
1057}
1058
1059void MDNode::deleteTemporary(MDNode *N) {
1060 assert(N->isTemporary() && "Expected temporary node");
1061 N->replaceAllUsesWith(MD: nullptr);
1062 N->deleteAsSubclass();
1063}
1064
1065void MDNode::storeDistinctInContext() {
1066 assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses");
1067 assert(!getNumUnresolved() && "Unexpected unresolved nodes");
1068 Storage = Distinct;
1069 assert(isResolved() && "Expected this to be resolved");
1070
1071 // Reset the hash.
1072 switch (getMetadataID()) {
1073 default:
1074 llvm_unreachable("Invalid subclass of MDNode");
1075#define HANDLE_MDNODE_LEAF(CLASS) \
1076 case CLASS##Kind: { \
1077 dispatchResetHash(cast<CLASS>(this)); \
1078 break; \
1079 }
1080#include "llvm/IR/Metadata.def"
1081 }
1082
1083 getContext().pImpl->DistinctMDNodes.push_back(x: this);
1084}
1085
1086void MDNode::replaceOperandWith(unsigned I, Metadata *New) {
1087 if (getOperand(I) == New)
1088 return;
1089
1090 if (!isUniqued()) {
1091 setOperand(I, New);
1092 return;
1093 }
1094
1095 handleChangedOperand(Ref: mutable_begin() + I, New);
1096}
1097
1098void MDNode::setOperand(unsigned I, Metadata *New) {
1099 assert(I < getNumOperands());
1100 mutable_begin()[I].reset(MD: New, Owner: isUniqued() ? this : nullptr);
1101}
1102
1103/// Get a node or a self-reference that looks like it.
1104///
1105/// Special handling for finding self-references, for use by \a
1106/// MDNode::concatenate() and \a MDNode::intersect() to maintain behaviour from
1107/// when self-referencing nodes were still uniqued. If the first operand has
1108/// the same operands as \c Ops, return the first operand instead.
1109static MDNode *getOrSelfReference(LLVMContext &Context,
1110 ArrayRef<Metadata *> Ops) {
1111 if (!Ops.empty())
1112 if (MDNode *N = dyn_cast_or_null<MDNode>(Val: Ops[0]))
1113 if (N->getNumOperands() == Ops.size() && N == N->getOperand(I: 0)) {
1114 for (unsigned I = 1, E = Ops.size(); I != E; ++I)
1115 if (Ops[I] != N->getOperand(I))
1116 return MDNode::get(Context, MDs: Ops);
1117 return N;
1118 }
1119
1120 return MDNode::get(Context, MDs: Ops);
1121}
1122
1123MDNode *MDNode::concatenate(MDNode *A, MDNode *B) {
1124 if (!A)
1125 return B;
1126 if (!B)
1127 return A;
1128
1129 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end());
1130 MDs.insert(Start: B->op_begin(), End: B->op_end());
1131
1132 // FIXME: This preserves long-standing behaviour, but is it really the right
1133 // behaviour? Or was that an unintended side-effect of node uniquing?
1134 return getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1135}
1136
1137MDNode *MDNode::intersect(MDNode *A, MDNode *B) {
1138 if (!A || !B)
1139 return nullptr;
1140
1141 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end());
1142 SmallPtrSet<Metadata *, 4> BSet(B->op_begin(), B->op_end());
1143 MDs.remove_if(P: [&](Metadata *MD) { return !BSet.count(Ptr: MD); });
1144
1145 // FIXME: This preserves long-standing behaviour, but is it really the right
1146 // behaviour? Or was that an unintended side-effect of node uniquing?
1147 return getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1148}
1149
1150MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) {
1151 if (!A || !B)
1152 return nullptr;
1153
1154 // Take the intersection of domains then union the scopes
1155 // within those domains
1156 SmallPtrSet<const MDNode *, 16> ADomains;
1157 SmallPtrSet<const MDNode *, 16> IntersectDomains;
1158 SmallSetVector<Metadata *, 4> MDs;
1159 for (const MDOperand &MDOp : A->operands())
1160 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1161 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1162 ADomains.insert(Ptr: Domain);
1163
1164 for (const MDOperand &MDOp : B->operands())
1165 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1166 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1167 if (ADomains.contains(Ptr: Domain)) {
1168 IntersectDomains.insert(Ptr: Domain);
1169 MDs.insert(X: MDOp);
1170 }
1171
1172 for (const MDOperand &MDOp : A->operands())
1173 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1174 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1175 if (IntersectDomains.contains(Ptr: Domain))
1176 MDs.insert(X: MDOp);
1177
1178 return MDs.empty() ? nullptr
1179 : getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1180}
1181
1182MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) {
1183 if (!A || !B)
1184 return nullptr;
1185
1186 APFloat AVal = mdconst::extract<ConstantFP>(MD: A->getOperand(I: 0))->getValueAPF();
1187 APFloat BVal = mdconst::extract<ConstantFP>(MD: B->getOperand(I: 0))->getValueAPF();
1188 if (AVal < BVal)
1189 return A;
1190 return B;
1191}
1192
1193// Call instructions with branch weights are only used in SamplePGO as
1194// documented in
1195/// https://llvm.org/docs/BranchWeightMetadata.html#callinst).
1196MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B,
1197 const Instruction *AInstr,
1198 const Instruction *BInstr) {
1199 assert(A && B && AInstr && BInstr && "Caller should guarantee");
1200 auto &Ctx = AInstr->getContext();
1201 MDBuilder MDHelper(Ctx);
1202
1203 // LLVM IR verifier verifies !prof metadata has at least 2 operands.
1204 assert(A->getNumOperands() >= 2 && B->getNumOperands() >= 2 &&
1205 "!prof annotations should have no less than 2 operands");
1206 MDString *AMDS = dyn_cast<MDString>(Val: A->getOperand(I: 0));
1207 MDString *BMDS = dyn_cast<MDString>(Val: B->getOperand(I: 0));
1208 // LLVM IR verfier verifies first operand is MDString.
1209 assert(AMDS != nullptr && BMDS != nullptr &&
1210 "first operand should be a non-null MDString");
1211 StringRef AProfName = AMDS->getString();
1212 StringRef BProfName = BMDS->getString();
1213 if (AProfName == MDProfLabels::BranchWeights &&
1214 BProfName == MDProfLabels::BranchWeights) {
1215 ConstantInt *AInstrWeight = mdconst::dyn_extract<ConstantInt>(
1216 MD: A->getOperand(I: getBranchWeightOffset(ProfileData: A)));
1217 ConstantInt *BInstrWeight = mdconst::dyn_extract<ConstantInt>(
1218 MD: B->getOperand(I: getBranchWeightOffset(ProfileData: B)));
1219 assert(AInstrWeight && BInstrWeight && "verified by LLVM verifier");
1220 return MDNode::get(Context&: Ctx,
1221 MDs: {MDHelper.createString(Str: MDProfLabels::BranchWeights),
1222 MDHelper.createConstant(C: ConstantInt::get(
1223 Ty: Type::getInt64Ty(C&: Ctx),
1224 V: SaturatingAdd(X: AInstrWeight->getZExtValue(),
1225 Y: BInstrWeight->getZExtValue())))});
1226 }
1227 return nullptr;
1228}
1229
1230// Pass in both instructions and nodes. Instruction information (e.g.,
1231// instruction type) helps interpret profiles and make implementation clearer.
1232MDNode *MDNode::getMergedProfMetadata(MDNode *A, MDNode *B,
1233 const Instruction *AInstr,
1234 const Instruction *BInstr) {
1235 // Check that it is legal to merge prof metadata based on the opcode.
1236 auto IsLegal = [](const Instruction &I) -> bool {
1237 switch (I.getOpcode()) {
1238 case Instruction::Invoke:
1239 case Instruction::Br:
1240 case Instruction::Switch:
1241 case Instruction::Call:
1242 case Instruction::IndirectBr:
1243 case Instruction::Select:
1244 case Instruction::CallBr:
1245 return true;
1246 default:
1247 return false;
1248 }
1249 };
1250 if (AInstr && !IsLegal(*AInstr))
1251 return nullptr;
1252 if (BInstr && !IsLegal(*BInstr))
1253 return nullptr;
1254
1255 if (!(A && B)) {
1256 return A ? A : B;
1257 }
1258
1259 assert(AInstr->getMetadata(LLVMContext::MD_prof) == A &&
1260 "Caller should guarantee");
1261 assert(BInstr->getMetadata(LLVMContext::MD_prof) == B &&
1262 "Caller should guarantee");
1263
1264 const CallInst *ACall = dyn_cast<CallInst>(Val: AInstr);
1265 const CallInst *BCall = dyn_cast<CallInst>(Val: BInstr);
1266
1267 // Both ACall and BCall are direct callsites.
1268 if (ACall && BCall && ACall->getCalledFunction() &&
1269 BCall->getCalledFunction())
1270 return mergeDirectCallProfMetadata(A, B, AInstr, BInstr);
1271
1272 if (A == B && !ProfcheckDisableMetadataFixes)
1273 return A;
1274
1275 // The rest of the cases are not implemented but could be added
1276 // when there are use cases.
1277 return nullptr;
1278}
1279
1280static bool isContiguous(const ConstantRange &A, const ConstantRange &B) {
1281 return A.getUpper() == B.getLower() || A.getLower() == B.getUpper();
1282}
1283
1284static bool canBeMerged(const ConstantRange &A, const ConstantRange &B) {
1285 return !A.intersectWith(CR: B).isEmptySet() || isContiguous(A, B);
1286}
1287
1288static bool tryMergeRange(SmallVectorImpl<ConstantInt *> &EndPoints,
1289 ConstantInt *Low, ConstantInt *High) {
1290 ConstantRange NewRange(Low->getValue(), High->getValue());
1291 unsigned Size = EndPoints.size();
1292 const APInt &LB = EndPoints[Size - 2]->getValue();
1293 const APInt &LE = EndPoints[Size - 1]->getValue();
1294 ConstantRange LastRange(LB, LE);
1295 if (canBeMerged(A: NewRange, B: LastRange)) {
1296 ConstantRange Union = LastRange.unionWith(CR: NewRange);
1297 Type *Ty = High->getType();
1298 EndPoints[Size - 2] =
1299 cast<ConstantInt>(Val: ConstantInt::get(Ty, V: Union.getLower()));
1300 EndPoints[Size - 1] =
1301 cast<ConstantInt>(Val: ConstantInt::get(Ty, V: Union.getUpper()));
1302 return true;
1303 }
1304 return false;
1305}
1306
1307static void addRange(SmallVectorImpl<ConstantInt *> &EndPoints,
1308 ConstantInt *Low, ConstantInt *High) {
1309 if (!EndPoints.empty())
1310 if (tryMergeRange(EndPoints, Low, High))
1311 return;
1312
1313 EndPoints.push_back(Elt: Low);
1314 EndPoints.push_back(Elt: High);
1315}
1316
1317MDNode *MDNode::getMergedCalleeTypeMetadata(const MDNode *A, const MDNode *B) {
1318 // Drop the callee_type metadata if either of the call instructions do not
1319 // have it.
1320 if (!A || !B)
1321 return nullptr;
1322 SmallVector<Metadata *, 8> AB;
1323 SmallPtrSet<Metadata *, 8> MergedCallees;
1324 auto AddUniqueCallees = [&AB, &MergedCallees](const MDNode *N) {
1325 for (Metadata *MD : N->operands()) {
1326 if (MergedCallees.insert(Ptr: MD).second)
1327 AB.push_back(Elt: MD);
1328 }
1329 };
1330 AddUniqueCallees(A);
1331 AddUniqueCallees(B);
1332 return MDNode::get(Context&: A->getContext(), MDs: AB);
1333}
1334
1335MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) {
1336 // Given two ranges, we want to compute the union of the ranges. This
1337 // is slightly complicated by having to combine the intervals and merge
1338 // the ones that overlap.
1339
1340 if (!A || !B)
1341 return nullptr;
1342
1343 if (A == B)
1344 return A;
1345
1346 // First, walk both lists in order of the lower boundary of each interval.
1347 // At each step, try to merge the new interval to the last one we added.
1348 SmallVector<ConstantInt *, 4> EndPoints;
1349 unsigned AI = 0;
1350 unsigned BI = 0;
1351 unsigned AN = A->getNumOperands() / 2;
1352 unsigned BN = B->getNumOperands() / 2;
1353 while (AI < AN && BI < BN) {
1354 ConstantInt *ALow = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI));
1355 ConstantInt *BLow = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI));
1356
1357 if (ALow->getValue().slt(RHS: BLow->getValue())) {
1358 addRange(EndPoints, Low: ALow,
1359 High: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI + 1)));
1360 ++AI;
1361 } else {
1362 addRange(EndPoints, Low: BLow,
1363 High: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI + 1)));
1364 ++BI;
1365 }
1366 }
1367 while (AI < AN) {
1368 addRange(EndPoints, Low: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI)),
1369 High: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI + 1)));
1370 ++AI;
1371 }
1372 while (BI < BN) {
1373 addRange(EndPoints, Low: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI)),
1374 High: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI + 1)));
1375 ++BI;
1376 }
1377
1378 // We haven't handled wrap in the previous merge,
1379 // if we have at least 2 ranges (4 endpoints) we have to try to merge
1380 // the last and first ones.
1381 unsigned Size = EndPoints.size();
1382 if (Size > 2) {
1383 ConstantInt *FB = EndPoints[0];
1384 ConstantInt *FE = EndPoints[1];
1385 if (tryMergeRange(EndPoints, Low: FB, High: FE)) {
1386 for (unsigned i = 0; i < Size - 2; ++i) {
1387 EndPoints[i] = EndPoints[i + 2];
1388 }
1389 EndPoints.resize(N: Size - 2);
1390 }
1391 }
1392
1393 // If in the end we have a single range, it is possible that it is now the
1394 // full range. Just drop the metadata in that case.
1395 if (EndPoints.size() == 2) {
1396 ConstantRange Range(EndPoints[0]->getValue(), EndPoints[1]->getValue());
1397 if (Range.isFullSet())
1398 return nullptr;
1399 }
1400
1401 SmallVector<Metadata *, 4> MDs;
1402 MDs.reserve(N: EndPoints.size());
1403 for (auto *I : EndPoints)
1404 MDs.push_back(Elt: ConstantAsMetadata::get(C: I));
1405 return MDNode::get(Context&: A->getContext(), MDs);
1406}
1407
1408MDNode *MDNode::getMostGenericNoFPClass(MDNode *A, MDNode *B) {
1409 if (!A || !B)
1410 return nullptr;
1411
1412 if (A == B)
1413 return A;
1414
1415 ConstantInt *AVal = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 0));
1416 ConstantInt *BVal = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 0));
1417 unsigned Intersect = AVal->getZExtValue() & BVal->getZExtValue();
1418 if (Intersect == 0)
1419 return nullptr;
1420
1421 return MDNode::get(Context&: A->getContext(), MDs: ConstantAsMetadata::get(C: ConstantInt::get(
1422 Ty: AVal->getType(), V: Intersect)));
1423}
1424
1425MDNode *MDNode::getMostGenericNoaliasAddrspace(MDNode *A, MDNode *B) {
1426 if (!A || !B)
1427 return nullptr;
1428
1429 if (A == B)
1430 return A;
1431
1432 SmallVector<ConstantRange> RangeListA, RangeListB;
1433 for (unsigned I = 0, E = A->getNumOperands() / 2; I != E; ++I) {
1434 auto *LowA = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * I + 0));
1435 auto *HighA = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * I + 1));
1436 RangeListA.push_back(Elt: ConstantRange(LowA->getValue(), HighA->getValue()));
1437 }
1438
1439 for (unsigned I = 0, E = B->getNumOperands() / 2; I != E; ++I) {
1440 auto *LowB = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * I + 0));
1441 auto *HighB = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * I + 1));
1442 RangeListB.push_back(Elt: ConstantRange(LowB->getValue(), HighB->getValue()));
1443 }
1444
1445 ConstantRangeList CRLA(RangeListA);
1446 ConstantRangeList CRLB(RangeListB);
1447 ConstantRangeList Result = CRLA.intersectWith(CRL: CRLB);
1448 if (Result.empty())
1449 return nullptr;
1450
1451 SmallVector<Metadata *> MDs;
1452 for (const ConstantRange &CR : Result) {
1453 MDs.push_back(Elt: ConstantAsMetadata::get(
1454 C: ConstantInt::get(Context&: A->getContext(), V: CR.getLower())));
1455 MDs.push_back(Elt: ConstantAsMetadata::get(
1456 C: ConstantInt::get(Context&: A->getContext(), V: CR.getUpper())));
1457 }
1458
1459 return MDNode::get(Context&: A->getContext(), MDs);
1460}
1461
1462MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) {
1463 if (!A || !B)
1464 return nullptr;
1465
1466 ConstantInt *AVal = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 0));
1467 ConstantInt *BVal = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 0));
1468 if (AVal->getZExtValue() < BVal->getZExtValue())
1469 return A;
1470 return B;
1471}
1472
1473CaptureComponents MDNode::toCaptureComponents(const MDNode *MD) {
1474 if (!MD)
1475 return CaptureComponents::All;
1476
1477 CaptureComponents CC = CaptureComponents::None;
1478 for (Metadata *Op : MD->operands()) {
1479 CaptureComponents Component =
1480 StringSwitch<CaptureComponents>(cast<MDString>(Val: Op)->getString())
1481 .Case(S: "address", Value: CaptureComponents::Address)
1482 .Case(S: "address_is_null", Value: CaptureComponents::AddressIsNull)
1483 .Case(S: "provenance", Value: CaptureComponents::Provenance)
1484 .Case(S: "read_provenance", Value: CaptureComponents::ReadProvenance);
1485 CC |= Component;
1486 }
1487 return CC;
1488}
1489
1490MDNode *MDNode::fromCaptureComponents(LLVMContext &Ctx, CaptureComponents CC) {
1491 assert(!capturesNothing(CC) && "Can't encode captures(none)");
1492 if (capturesAll(CC))
1493 return nullptr;
1494
1495 SmallVector<Metadata *> Components;
1496 if (capturesAddressIsNullOnly(CC))
1497 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "address_is_null"));
1498 else if (capturesAddress(CC))
1499 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "address"));
1500 if (capturesReadProvenanceOnly(CC))
1501 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "read_provenance"));
1502 else if (capturesFullProvenance(CC))
1503 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "provenance"));
1504 return MDNode::get(Context&: Ctx, MDs: Components);
1505}
1506
1507//===----------------------------------------------------------------------===//
1508// NamedMDNode implementation.
1509//
1510
1511static SmallVector<TrackingMDRef, 4> &getNMDOps(void *Operands) {
1512 return *(SmallVector<TrackingMDRef, 4> *)Operands;
1513}
1514
1515NamedMDNode::NamedMDNode(const Twine &N)
1516 : Name(N.str()), Operands(new SmallVector<TrackingMDRef, 4>()) {}
1517
1518NamedMDNode::~NamedMDNode() {
1519 dropAllReferences();
1520 delete &getNMDOps(Operands);
1521}
1522
1523unsigned NamedMDNode::getNumOperands() const {
1524 return (unsigned)getNMDOps(Operands).size();
1525}
1526
1527MDNode *NamedMDNode::getOperand(unsigned i) const {
1528 assert(i < getNumOperands() && "Invalid Operand number!");
1529 auto *N = getNMDOps(Operands)[i].get();
1530 return cast_or_null<MDNode>(Val: N);
1531}
1532
1533void NamedMDNode::addOperand(MDNode *M) { getNMDOps(Operands).emplace_back(Args&: M); }
1534
1535void NamedMDNode::setOperand(unsigned I, MDNode *New) {
1536 assert(I < getNumOperands() && "Invalid operand number");
1537 getNMDOps(Operands)[I].reset(MD: New);
1538}
1539
1540void NamedMDNode::eraseFromParent() { getParent()->eraseNamedMetadata(NMD: this); }
1541
1542void NamedMDNode::clearOperands() { getNMDOps(Operands).clear(); }
1543
1544StringRef NamedMDNode::getName() const { return StringRef(Name); }
1545
1546//===----------------------------------------------------------------------===//
1547// Instruction Metadata method implementations.
1548//
1549
1550MDNode *MDAttachments::lookup(unsigned ID) const {
1551 for (const auto &A : Attachments)
1552 if (A.MDKind == ID)
1553 return A.Node;
1554 return nullptr;
1555}
1556
1557void MDAttachments::get(unsigned ID, SmallVectorImpl<MDNode *> &Result) const {
1558 for (const auto &A : Attachments)
1559 if (A.MDKind == ID)
1560 Result.push_back(Elt: A.Node);
1561}
1562
1563void MDAttachments::getAll(
1564 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
1565 for (const auto &A : Attachments)
1566 Result.emplace_back(Args: A.MDKind, Args: A.Node);
1567
1568 // Sort the resulting array so it is stable with respect to metadata IDs. We
1569 // need to preserve the original insertion order though.
1570 if (Result.size() > 1)
1571 llvm::stable_sort(Range&: Result, C: less_first());
1572}
1573
1574void MDAttachments::set(unsigned ID, MDNode *MD) {
1575 erase(ID);
1576 if (MD)
1577 insert(ID, MD&: *MD);
1578}
1579
1580void MDAttachments::insert(unsigned ID, MDNode &MD) {
1581 Attachments.push_back(Elt: {.MDKind: ID, .Node: TrackingMDNodeRef(&MD)});
1582}
1583
1584bool MDAttachments::erase(unsigned ID) {
1585 if (empty())
1586 return false;
1587
1588 // Common case is one value.
1589 if (Attachments.size() == 1 && Attachments.back().MDKind == ID) {
1590 Attachments.pop_back();
1591 return true;
1592 }
1593
1594 auto OldSize = Attachments.size();
1595 llvm::erase_if(C&: Attachments,
1596 P: [ID](const Attachment &A) { return A.MDKind == ID; });
1597 return OldSize != Attachments.size();
1598}
1599
1600MDNode *Value::getMetadata(StringRef Kind) const {
1601 if (!hasMetadata())
1602 return nullptr;
1603 unsigned KindID = getContext().getMDKindID(Name: Kind);
1604 return getMetadataImpl(KindID);
1605}
1606
1607MDNode *Value::getMetadataImpl(unsigned KindID) const {
1608 const LLVMContext &Ctx = getContext();
1609 const MDAttachments &Attachements = Ctx.pImpl->ValueMetadata.at(Val: this);
1610 return Attachements.lookup(ID: KindID);
1611}
1612
1613void Value::getMetadata(unsigned KindID, SmallVectorImpl<MDNode *> &MDs) const {
1614 if (hasMetadata())
1615 getContext().pImpl->ValueMetadata.at(Val: this).get(ID: KindID, Result&: MDs);
1616}
1617
1618void Value::getMetadata(StringRef Kind, SmallVectorImpl<MDNode *> &MDs) const {
1619 if (hasMetadata())
1620 getMetadata(KindID: getContext().getMDKindID(Name: Kind), MDs);
1621}
1622
1623void Value::getAllMetadata(
1624 SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const {
1625 if (hasMetadata()) {
1626 assert(getContext().pImpl->ValueMetadata.count(this) &&
1627 "bit out of sync with hash table");
1628 const MDAttachments &Info = getContext().pImpl->ValueMetadata.at(Val: this);
1629 Info.getAll(Result&: MDs);
1630 }
1631}
1632
1633void Value::setMetadata(unsigned KindID, MDNode *Node) {
1634 assert(isa<Instruction>(this) || isa<GlobalObject>(this));
1635
1636 // Handle the case when we're adding/updating metadata on a value.
1637 if (Node) {
1638 MDAttachments &Info = getContext().pImpl->ValueMetadata[this];
1639 assert(!Info.empty() == HasMetadata && "bit out of sync with hash table");
1640 if (Info.empty())
1641 HasMetadata = true;
1642 Info.set(ID: KindID, MD: Node);
1643 return;
1644 }
1645
1646 // Otherwise, we're removing metadata from an instruction.
1647 assert((HasMetadata == (getContext().pImpl->ValueMetadata.count(this) > 0)) &&
1648 "bit out of sync with hash table");
1649 if (!HasMetadata)
1650 return; // Nothing to remove!
1651 MDAttachments &Info = getContext().pImpl->ValueMetadata.find(Val: this)->second;
1652
1653 // Handle removal of an existing value.
1654 Info.erase(ID: KindID);
1655 if (!Info.empty())
1656 return;
1657 getContext().pImpl->ValueMetadata.erase(Val: this);
1658 HasMetadata = false;
1659}
1660
1661void Value::setMetadata(StringRef Kind, MDNode *Node) {
1662 if (!Node && !HasMetadata)
1663 return;
1664 setMetadata(KindID: getContext().getMDKindID(Name: Kind), Node);
1665}
1666
1667void Value::addMetadata(unsigned KindID, MDNode &MD) {
1668 assert(isa<Instruction>(this) || isa<GlobalObject>(this));
1669 if (!HasMetadata)
1670 HasMetadata = true;
1671 getContext().pImpl->ValueMetadata[this].insert(ID: KindID, MD);
1672}
1673
1674void Value::addMetadata(StringRef Kind, MDNode &MD) {
1675 addMetadata(KindID: getContext().getMDKindID(Name: Kind), MD);
1676}
1677
1678bool Value::eraseMetadata(unsigned KindID) {
1679 // Nothing to unset.
1680 if (!HasMetadata)
1681 return false;
1682
1683 MDAttachments &Store = getContext().pImpl->ValueMetadata.find(Val: this)->second;
1684 bool Changed = Store.erase(ID: KindID);
1685 if (Store.empty())
1686 clearMetadata();
1687 return Changed;
1688}
1689
1690void Value::eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred) {
1691 if (!HasMetadata)
1692 return;
1693
1694 auto &MetadataStore = getContext().pImpl->ValueMetadata;
1695 MDAttachments &Info = MetadataStore.find(Val: this)->second;
1696 assert(!Info.empty() && "bit out of sync with hash table");
1697 Info.remove_if(shouldRemove: [Pred](const MDAttachments::Attachment &I) {
1698 return Pred(I.MDKind, I.Node);
1699 });
1700
1701 if (Info.empty())
1702 clearMetadata();
1703}
1704
1705void Value::clearMetadata() {
1706 if (!HasMetadata)
1707 return;
1708 assert(getContext().pImpl->ValueMetadata.count(this) &&
1709 "bit out of sync with hash table");
1710 getContext().pImpl->ValueMetadata.erase(Val: this);
1711 HasMetadata = false;
1712}
1713
1714void Instruction::setMetadata(StringRef Kind, MDNode *Node) {
1715 if (!Node && !hasMetadata())
1716 return;
1717 setMetadata(KindID: getContext().getMDKindID(Name: Kind), Node);
1718}
1719
1720MDNode *Instruction::getMetadataImpl(StringRef Kind) const {
1721 const LLVMContext &Ctx = getContext();
1722 unsigned KindID = Ctx.getMDKindID(Name: Kind);
1723 if (KindID == LLVMContext::MD_dbg)
1724 return DbgLoc.getAsMDNode();
1725 return Value::getMetadata(KindID);
1726}
1727
1728void Instruction::eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred) {
1729 if (DbgLoc && Pred(LLVMContext::MD_dbg, DbgLoc.getAsMDNode()))
1730 DbgLoc = {};
1731
1732 Value::eraseMetadataIf(Pred);
1733}
1734
1735void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) {
1736 if (!Value::hasMetadata())
1737 return; // Nothing to remove!
1738
1739 SmallSet<unsigned, 32> KnownSet(llvm::from_range, KnownIDs);
1740
1741 // A DIAssignID attachment is debug metadata, don't drop it.
1742 KnownSet.insert(V: LLVMContext::MD_DIAssignID);
1743
1744 Value::eraseMetadataIf(Pred: [&KnownSet](unsigned MDKind, MDNode *Node) {
1745 return !KnownSet.count(V: MDKind);
1746 });
1747}
1748
1749void Instruction::updateDIAssignIDMapping(DIAssignID *ID) {
1750 auto &IDToInstrs = getContext().pImpl->AssignmentIDToInstrs;
1751 if (const DIAssignID *CurrentID =
1752 cast_or_null<DIAssignID>(Val: getMetadata(KindID: LLVMContext::MD_DIAssignID))) {
1753 // Nothing to do if the ID isn't changing.
1754 if (ID == CurrentID)
1755 return;
1756
1757 // Unmap this instruction from its current ID.
1758 auto InstrsIt = IDToInstrs.find(Val: CurrentID);
1759 assert(InstrsIt != IDToInstrs.end() &&
1760 "Expect existing attachment to be mapped");
1761
1762 auto &InstVec = InstrsIt->second;
1763 auto *InstIt = llvm::find(Range&: InstVec, Val: this);
1764 assert(InstIt != InstVec.end() &&
1765 "Expect instruction to be mapped to attachment");
1766 // The vector contains a ptr to this. If this is the only element in the
1767 // vector, remove the ID:vector entry, otherwise just remove the
1768 // instruction from the vector.
1769 if (InstVec.size() == 1)
1770 IDToInstrs.erase(I: InstrsIt);
1771 else
1772 InstVec.erase(CI: InstIt);
1773 }
1774
1775 // Map this instruction to the new ID.
1776 if (ID)
1777 IDToInstrs[ID].push_back(Elt: this);
1778}
1779
1780void Instruction::setMetadata(unsigned KindID, MDNode *Node) {
1781 if (!Node && !hasMetadata())
1782 return;
1783
1784 // Handle 'dbg' as a special case since it is not stored in the hash table.
1785 if (KindID == LLVMContext::MD_dbg) {
1786 DbgLoc = DebugLoc(Node);
1787 return;
1788 }
1789
1790 // Update DIAssignID to Instruction(s) mapping.
1791 if (KindID == LLVMContext::MD_DIAssignID) {
1792 // The DIAssignID tracking infrastructure doesn't support RAUWing temporary
1793 // nodes with DIAssignIDs. The cast_or_null below would also catch this, but
1794 // having a dedicated assert helps make this obvious.
1795 assert((!Node || !Node->isTemporary()) &&
1796 "Temporary DIAssignIDs are invalid");
1797 updateDIAssignIDMapping(ID: cast_or_null<DIAssignID>(Val: Node));
1798 }
1799
1800 Value::setMetadata(KindID, Node);
1801}
1802
1803void Instruction::addAnnotationMetadata(SmallVector<StringRef> Annotations) {
1804 SmallVector<Metadata *, 4> Names;
1805 if (auto *Existing = getMetadata(KindID: LLVMContext::MD_annotation)) {
1806 SmallSetVector<StringRef, 2> AnnotationsSet(Annotations.begin(),
1807 Annotations.end());
1808 auto *Tuple = cast<MDTuple>(Val: Existing);
1809 for (auto &N : Tuple->operands()) {
1810 if (isa<MDString>(Val: N.get())) {
1811 Names.push_back(Elt: N);
1812 continue;
1813 }
1814 auto *MDAnnotationTuple = cast<MDTuple>(Val: N);
1815 if (any_of(Range: MDAnnotationTuple->operands(), P: [&AnnotationsSet](auto &Op) {
1816 return AnnotationsSet.contains(key: cast<MDString>(Op)->getString());
1817 }))
1818 return;
1819 Names.push_back(Elt: N);
1820 }
1821 }
1822
1823 MDBuilder MDB(getContext());
1824 SmallVector<Metadata *> MDAnnotationStrings;
1825 for (StringRef Annotation : Annotations)
1826 MDAnnotationStrings.push_back(Elt: MDB.createString(Str: Annotation));
1827 MDNode *InfoTuple = MDTuple::get(Context&: getContext(), MDs: MDAnnotationStrings);
1828 Names.push_back(Elt: InfoTuple);
1829 MDNode *MD = MDTuple::get(Context&: getContext(), MDs: Names);
1830 setMetadata(KindID: LLVMContext::MD_annotation, Node: MD);
1831}
1832
1833void Instruction::addAnnotationMetadata(StringRef Name) {
1834 SmallVector<Metadata *, 4> Names;
1835 if (auto *Existing = getMetadata(KindID: LLVMContext::MD_annotation)) {
1836 auto *Tuple = cast<MDTuple>(Val: Existing);
1837 for (auto &N : Tuple->operands()) {
1838 if (isa<MDString>(Val: N.get()) &&
1839 cast<MDString>(Val: N.get())->getString() == Name)
1840 return;
1841 Names.push_back(Elt: N.get());
1842 }
1843 }
1844
1845 MDBuilder MDB(getContext());
1846 Names.push_back(Elt: MDB.createString(Str: Name));
1847 MDNode *MD = MDTuple::get(Context&: getContext(), MDs: Names);
1848 setMetadata(KindID: LLVMContext::MD_annotation, Node: MD);
1849}
1850
1851AAMDNodes Instruction::getAAMetadata() const {
1852 AAMDNodes Result;
1853 // Not using Instruction::hasMetadata() because we're not interested in
1854 // DebugInfoMetadata.
1855 if (Value::hasMetadata()) {
1856 const MDAttachments &Info = getContext().pImpl->ValueMetadata.at(Val: this);
1857 Result.TBAA = Info.lookup(ID: LLVMContext::MD_tbaa);
1858 Result.TBAAStruct = Info.lookup(ID: LLVMContext::MD_tbaa_struct);
1859 Result.Scope = Info.lookup(ID: LLVMContext::MD_alias_scope);
1860 Result.NoAlias = Info.lookup(ID: LLVMContext::MD_noalias);
1861 Result.NoAliasAddrSpace = Info.lookup(ID: LLVMContext::MD_noalias_addrspace);
1862 }
1863 return Result;
1864}
1865
1866void Instruction::setAAMetadata(const AAMDNodes &N) {
1867 setMetadata(KindID: LLVMContext::MD_tbaa, Node: N.TBAA);
1868 setMetadata(KindID: LLVMContext::MD_tbaa_struct, Node: N.TBAAStruct);
1869 setMetadata(KindID: LLVMContext::MD_alias_scope, Node: N.Scope);
1870 setMetadata(KindID: LLVMContext::MD_noalias, Node: N.NoAlias);
1871 setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: N.NoAliasAddrSpace);
1872}
1873
1874void Instruction::setNoSanitizeMetadata() {
1875 setMetadata(KindID: llvm::LLVMContext::MD_nosanitize,
1876 Node: llvm::MDNode::get(Context&: getContext(), MDs: {}));
1877}
1878
1879void Instruction::getAllMetadataImpl(
1880 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
1881 Result.clear();
1882
1883 // Handle 'dbg' as a special case since it is not stored in the hash table.
1884 if (DbgLoc) {
1885 Result.push_back(
1886 Elt: std::make_pair(x: (unsigned)LLVMContext::MD_dbg, y: DbgLoc.getAsMDNode()));
1887 }
1888 Value::getAllMetadata(MDs&: Result);
1889}
1890
1891bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) const {
1892 assert(
1893 (getOpcode() == Instruction::Br || getOpcode() == Instruction::Select ||
1894 getOpcode() == Instruction::Call || getOpcode() == Instruction::Invoke ||
1895 getOpcode() == Instruction::IndirectBr ||
1896 getOpcode() == Instruction::Switch) &&
1897 "Looking for branch weights on something besides branch");
1898
1899 return ::extractProfTotalWeight(I: *this, TotalWeights&: TotalVal);
1900}
1901
1902void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) {
1903 SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
1904 Other->getAllMetadata(MDs);
1905 for (auto &MD : MDs) {
1906 // We need to adjust the type metadata offset.
1907 if (Offset != 0 && MD.first == LLVMContext::MD_type) {
1908 auto *OffsetConst = cast<ConstantInt>(
1909 Val: cast<ConstantAsMetadata>(Val: MD.second->getOperand(I: 0))->getValue());
1910 Metadata *TypeId = MD.second->getOperand(I: 1);
1911 auto *NewOffsetMD = ConstantAsMetadata::get(C: ConstantInt::get(
1912 Ty: OffsetConst->getType(), V: OffsetConst->getValue() + Offset));
1913 addMetadata(KindID: LLVMContext::MD_type,
1914 MD&: *MDNode::get(Context&: getContext(), MDs: {NewOffsetMD, TypeId}));
1915 continue;
1916 }
1917 // If an offset adjustment was specified we need to modify the DIExpression
1918 // to prepend the adjustment:
1919 // !DIExpression(DW_OP_plus, Offset, [original expr])
1920 auto *Attachment = MD.second;
1921 if (Offset != 0 && MD.first == LLVMContext::MD_dbg) {
1922 DIGlobalVariable *GV = dyn_cast<DIGlobalVariable>(Val: Attachment);
1923 DIExpression *E = nullptr;
1924 if (!GV) {
1925 auto *GVE = cast<DIGlobalVariableExpression>(Val: Attachment);
1926 GV = GVE->getVariable();
1927 E = GVE->getExpression();
1928 }
1929 ArrayRef<uint64_t> OrigElements;
1930 if (E)
1931 OrigElements = E->getElements();
1932 std::vector<uint64_t> Elements(OrigElements.size() + 2);
1933 Elements[0] = dwarf::DW_OP_plus_uconst;
1934 Elements[1] = Offset;
1935 llvm::copy(Range&: OrigElements, Out: Elements.begin() + 2);
1936 E = DIExpression::get(Context&: getContext(), Elements);
1937 Attachment = DIGlobalVariableExpression::get(Context&: getContext(), Variable: GV, Expression: E);
1938 }
1939 addMetadata(KindID: MD.first, MD&: *Attachment);
1940 }
1941}
1942
1943void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) {
1944 addMetadata(
1945 KindID: LLVMContext::MD_type,
1946 MD&: *MDTuple::get(Context&: getContext(),
1947 MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1948 Ty: Type::getInt64Ty(C&: getContext()), V: Offset)),
1949 TypeID}));
1950}
1951
1952void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) {
1953 // Remove any existing vcall visibility metadata first in case we are
1954 // updating.
1955 eraseMetadata(KindID: LLVMContext::MD_vcall_visibility);
1956 addMetadata(KindID: LLVMContext::MD_vcall_visibility,
1957 MD&: *MDNode::get(Context&: getContext(),
1958 MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
1959 Ty: Type::getInt64Ty(C&: getContext()), V: Visibility))}));
1960}
1961
1962GlobalObject::VCallVisibility GlobalObject::getVCallVisibility() const {
1963 if (MDNode *MD = getMetadata(KindID: LLVMContext::MD_vcall_visibility)) {
1964 uint64_t Val = cast<ConstantInt>(
1965 Val: cast<ConstantAsMetadata>(Val: MD->getOperand(I: 0))->getValue())
1966 ->getZExtValue();
1967 assert(Val <= 2 && "unknown vcall visibility!");
1968 return (VCallVisibility)Val;
1969 }
1970 return VCallVisibility::VCallVisibilityPublic;
1971}
1972
1973void Function::setSubprogram(DISubprogram *SP) {
1974 setMetadata(KindID: LLVMContext::MD_dbg, Node: SP);
1975}
1976
1977DISubprogram *Function::getSubprogram() const {
1978 return cast_or_null<DISubprogram>(Val: getMetadata(KindID: LLVMContext::MD_dbg));
1979}
1980
1981bool Function::shouldEmitDebugInfoForProfiling() const {
1982 if (DISubprogram *SP = getSubprogram()) {
1983 if (DICompileUnit *CU = SP->getUnit()) {
1984 return CU->getDebugInfoForProfiling();
1985 }
1986 }
1987 return false;
1988}
1989
1990void GlobalVariable::addDebugInfo(DIGlobalVariableExpression *GV) {
1991 addMetadata(KindID: LLVMContext::MD_dbg, MD&: *GV);
1992}
1993
1994void GlobalVariable::getDebugInfo(
1995 SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const {
1996 SmallVector<MDNode *, 1> MDs;
1997 getMetadata(KindID: LLVMContext::MD_dbg, MDs);
1998 for (MDNode *MD : MDs)
1999 GVs.push_back(Elt: cast<DIGlobalVariableExpression>(Val: MD));
2000}
2001