1//===- Metadata.cpp - Implement Metadata classes --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Metadata classes.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/IR/Metadata.h"
14#include "LLVMContextImpl.h"
15#include "MetadataImpl.h"
16#include "llvm/ADT/APFloat.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/DenseSet.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/ADT/Twine.h"
28#include "llvm/IR/Argument.h"
29#include "llvm/IR/BasicBlock.h"
30#include "llvm/IR/Constant.h"
31#include "llvm/IR/ConstantRange.h"
32#include "llvm/IR/ConstantRangeList.h"
33#include "llvm/IR/Constants.h"
34#include "llvm/IR/DebugInfoMetadata.h"
35#include "llvm/IR/DebugLoc.h"
36#include "llvm/IR/DebugProgramInstruction.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/GlobalObject.h"
39#include "llvm/IR/GlobalVariable.h"
40#include "llvm/IR/Instruction.h"
41#include "llvm/IR/LLVMContext.h"
42#include "llvm/IR/MDBuilder.h"
43#include "llvm/IR/Module.h"
44#include "llvm/IR/ProfDataUtils.h"
45#include "llvm/IR/TrackingMDRef.h"
46#include "llvm/IR/Type.h"
47#include "llvm/IR/Value.h"
48#include "llvm/Support/Casting.h"
49
50#include "llvm/Support/ErrorHandling.h"
51#include "llvm/Support/MathExtras.h"
52#include "llvm/Support/ModRef.h"
53#include <cassert>
54#include <cstddef>
55#include <cstdint>
56#include <type_traits>
57#include <utility>
58#include <vector>
59
60using namespace llvm;
61
62MetadataAsValue::MetadataAsValue(Type *Ty, Metadata *MD)
63 : Value(Ty, MetadataAsValueVal), MD(MD) {
64 track();
65}
66
67MetadataAsValue::~MetadataAsValue() {
68 getType()->getContext().pImpl->MetadataAsValues.erase(Val: MD);
69 untrack();
70}
71
72/// Canonicalize metadata arguments to intrinsics.
73///
74/// To support bitcode upgrades (and assembly semantic sugar) for \a
75/// MetadataAsValue, we need to canonicalize certain metadata.
76///
77/// - nullptr is replaced by an empty MDNode.
78/// - An MDNode with a single null operand is replaced by an empty MDNode.
79/// - An MDNode whose only operand is a \a ConstantAsMetadata gets skipped.
80///
81/// This maintains readability of bitcode from when metadata was a type of
82/// value, and these bridges were unnecessary.
83static Metadata *canonicalizeMetadataForValue(LLVMContext &Context,
84 Metadata *MD) {
85 if (!MD)
86 // !{}
87 return MDNode::get(Context, MDs: {});
88
89 // Return early if this isn't a single-operand MDNode.
90 auto *N = dyn_cast<MDNode>(Val: MD);
91 if (!N || N->getNumOperands() != 1)
92 return MD;
93
94 if (!N->getOperand(I: 0))
95 // !{}
96 return MDNode::get(Context, MDs: {});
97
98 if (auto *C = dyn_cast<ConstantAsMetadata>(Val: N->getOperand(I: 0)))
99 // Look through the MDNode.
100 return C;
101
102 return MD;
103}
104
105MetadataAsValue *MetadataAsValue::get(LLVMContext &Context, Metadata *MD) {
106 MD = canonicalizeMetadataForValue(Context, MD);
107 auto *&Entry = Context.pImpl->MetadataAsValues[MD];
108 if (!Entry)
109 Entry = new MetadataAsValue(Type::getMetadataTy(C&: Context), MD);
110 return Entry;
111}
112
113MetadataAsValue *MetadataAsValue::getIfExists(LLVMContext &Context,
114 Metadata *MD) {
115 MD = canonicalizeMetadataForValue(Context, MD);
116 auto &Store = Context.pImpl->MetadataAsValues;
117 return Store.lookup(Val: MD);
118}
119
120void MetadataAsValue::handleChangedMetadata(Metadata *MD) {
121 LLVMContext &Context = getContext();
122 MD = canonicalizeMetadataForValue(Context, MD);
123 auto &Store = Context.pImpl->MetadataAsValues;
124
125 // Stop tracking the old metadata.
126 Store.erase(Val: this->MD);
127 untrack();
128 this->MD = nullptr;
129
130 // Start tracking MD, or RAUW if necessary.
131 auto *&Entry = Store[MD];
132 if (Entry) {
133 replaceAllUsesWith(V: Entry);
134 delete this;
135 return;
136 }
137
138 this->MD = MD;
139 track();
140 Entry = this;
141}
142
143void MetadataAsValue::track() {
144 if (MD)
145 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
146}
147
148void MetadataAsValue::untrack() {
149 if (MD)
150 MetadataTracking::untrack(MD);
151}
152
153DbgVariableRecord *DebugValueUser::getUser() {
154 return static_cast<DbgVariableRecord *>(this);
155}
156const DbgVariableRecord *DebugValueUser::getUser() const {
157 return static_cast<const DbgVariableRecord *>(this);
158}
159
160void DebugValueUser::handleChangedValue(void *Old, Metadata *New) {
161 // NOTE: We could inform the "owner" that a value has changed through
162 // getOwner, if needed.
163 auto OldMD = static_cast<Metadata **>(Old);
164 ptrdiff_t Idx = std::distance(first: &*DebugValues.begin(), last: OldMD);
165 // If replacing a ValueAsMetadata with a nullptr, replace it with a
166 // PoisonValue instead.
167 if (OldMD && isa<ValueAsMetadata>(Val: *OldMD) && !New) {
168 auto *OldVAM = cast<ValueAsMetadata>(Val: *OldMD);
169 New = ValueAsMetadata::get(V: PoisonValue::get(T: OldVAM->getValue()->getType()));
170 }
171 resetDebugValue(Idx, DebugValue: New);
172}
173
174void DebugValueUser::trackDebugValue(size_t Idx) {
175 assert(Idx < 3 && "Invalid debug value index.");
176 Metadata *&MD = DebugValues[Idx];
177 if (!MD)
178 return;
179 MetadataTracking::track(Ref: &MD, MD&: *MD, Owner&: *this);
180 if (auto *ID = Idx == AssignIDIdx ? dyn_cast<DIAssignID>(Val: MD) : nullptr)
181 ID->Records.push_back(NewVal: getUser());
182}
183
184void DebugValueUser::trackDebugValues() {
185 for (size_t I = 0, E = DebugValues.size(); I != E; ++I)
186 trackDebugValue(Idx: I);
187}
188
189void DebugValueUser::untrackDebugValue(size_t Idx) {
190 assert(Idx < 3 && "Invalid debug value index.");
191 Metadata *&MD = DebugValues[Idx];
192 if (!MD)
193 return;
194 MetadataTracking::untrack(MD);
195 if (auto *ID = Idx == AssignIDIdx ? dyn_cast<DIAssignID>(Val: MD) : nullptr)
196 ID->Records.erase(I: llvm::find(Range&: ID->Records, Val: getUser()));
197}
198
199void DebugValueUser::untrackDebugValues() {
200 for (size_t I = 0, E = DebugValues.size(); I != E; ++I)
201 untrackDebugValue(Idx: I);
202}
203
204void DebugValueUser::retrackDebugValues(DebugValueUser &X) {
205 assert(DebugValueUser::operator==(X) && "Expected values to match");
206 for (const auto &[MD, XMD] : zip(t&: DebugValues, u&: X.DebugValues))
207 if (XMD)
208 MetadataTracking::retrack(MD&: XMD, New&: MD);
209 if (auto *ID = dyn_cast_or_null<DIAssignID>(Val: DebugValues[AssignIDIdx]))
210 *llvm::find(Range&: ID->Records, Val: X.getUser()) = getUser();
211 X.DebugValues.fill(u: nullptr);
212}
213
214bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) {
215 assert(Ref && "Expected live reference");
216 assert((Owner || *static_cast<Metadata **>(Ref) == &MD) &&
217 "Reference without owner must be direct");
218 if (auto *R = ReplaceableUses::getOrCreate(MD)) {
219 R->addRef(Ref, Owner);
220 return true;
221 }
222 if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(Val: &MD)) {
223 assert(!PH->Use && "Placeholders can only be used once");
224 assert(!Owner && "Unexpected callback to owner");
225 PH->Use = static_cast<Metadata **>(Ref);
226 return true;
227 }
228 return false;
229}
230
231void MetadataTracking::untrack(void *Ref, Metadata &MD) {
232 assert(Ref && "Expected live reference");
233 if (auto *R = ReplaceableUses::getIfExists(MD))
234 R->dropRef(Ref);
235 else if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(Val: &MD))
236 PH->Use = nullptr;
237}
238
239bool MetadataTracking::retrack(void *Ref, Metadata &MD, void *New) {
240 assert(Ref && "Expected live reference");
241 assert(New && "Expected live reference");
242 assert(Ref != New && "Expected change");
243 if (auto *R = ReplaceableUses::getIfExists(MD)) {
244 R->moveRef(Ref, New, MD);
245 return true;
246 }
247 assert(!isa<DistinctMDOperandPlaceholder>(MD) &&
248 "Unexpected move of an MDOperand");
249 assert(!isReplaceable(MD) &&
250 "Expected un-replaceable metadata, since we didn't move a reference");
251 return false;
252}
253
254bool MetadataTracking::isReplaceable(const Metadata &MD) {
255 return ReplaceableUses::isReplaceable(MD);
256}
257
258SmallVector<Metadata *> ReplaceableUses::getAllArgListUsers() {
259 SmallVector<std::pair<OwnerTy, uint64_t> *> MDUsersWithID;
260 for (auto Pair : UseMap) {
261 OwnerTy Owner = Pair.second.first;
262 if (Owner.isNull())
263 continue;
264 if (!isa<Metadata *>(Val: Owner))
265 continue;
266 Metadata *OwnerMD = cast<Metadata *>(Val&: Owner);
267 if (OwnerMD->getMetadataID() == Metadata::DIArgListKind)
268 MDUsersWithID.push_back(Elt: &UseMap[Pair.first]);
269 }
270 llvm::sort(C&: MDUsersWithID, Comp: [](auto UserA, auto UserB) {
271 return UserA->second < UserB->second;
272 });
273 SmallVector<Metadata *> MDUsers;
274 for (auto *UserWithID : MDUsersWithID)
275 MDUsers.push_back(Elt: cast<Metadata *>(Val&: UserWithID->first));
276 return MDUsers;
277}
278
279SmallVector<DbgVariableRecord *>
280ReplaceableUses::getAllDbgVariableRecordUsers() {
281 SmallVector<std::pair<OwnerTy, uint64_t> *> DVRUsersWithID;
282 for (auto Pair : UseMap) {
283 OwnerTy Owner = Pair.second.first;
284 if (Owner.isNull())
285 continue;
286 if (!isa<DebugValueUser *>(Val: Owner))
287 continue;
288 DVRUsersWithID.push_back(Elt: &UseMap[Pair.first]);
289 }
290 // Order DbgVariableRecord users in reverse-creation order. Normal dbg.value
291 // users of MetadataAsValues are ordered by their UseList, i.e. reverse order
292 // of when they were added: we need to replicate that here. The structure of
293 // debug-info output depends on the ordering of intrinsics, thus we need
294 // to keep them consistent for comparisons sake.
295 llvm::sort(C&: DVRUsersWithID, Comp: [](auto UserA, auto UserB) {
296 return UserA->second > UserB->second;
297 });
298 SmallVector<DbgVariableRecord *> DVRUsers;
299 for (auto UserWithID : DVRUsersWithID)
300 DVRUsers.push_back(Elt: cast<DebugValueUser *>(Val&: UserWithID->first)->getUser());
301 return DVRUsers;
302}
303
304void ReplaceableUses::addRef(void *Ref, OwnerTy Owner) {
305 bool WasInserted =
306 UseMap.insert(KV: std::make_pair(x&: Ref, y: std::make_pair(x&: Owner, y&: NextIndex)))
307 .second;
308 (void)WasInserted;
309 assert(WasInserted && "Expected to add a reference");
310
311 ++NextIndex;
312 assert(NextIndex != 0 && "Unexpected overflow");
313}
314
315void ReplaceableUses::dropRef(void *Ref) {
316 bool WasErased = UseMap.erase(Val: Ref);
317 (void)WasErased;
318 assert(WasErased && "Expected to drop a reference");
319}
320
321void ReplaceableUses::moveRef(void *Ref, void *New, const Metadata &MD) {
322 auto I = UseMap.find(Val: Ref);
323 assert(I != UseMap.end() && "Expected to move a reference");
324 auto OwnerAndIndex = I->second;
325 UseMap.erase(I);
326 bool WasInserted = UseMap.insert(KV: std::make_pair(x&: New, y&: OwnerAndIndex)).second;
327 (void)WasInserted;
328 assert(WasInserted && "Expected to add a reference");
329
330 // Check that the references are direct if there's no owner.
331 (void)MD;
332 assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) &&
333 "Reference without owner must be direct");
334 assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) &&
335 "Reference without owner must be direct");
336}
337
338void ReplaceableUses::SalvageDebugInfo(const Constant &C) {
339 if (!C.isUsedByMetadata()) {
340 return;
341 }
342
343 LLVMContext &Context = C.getType()->getContext();
344 auto &Store = Context.pImpl->ValuesAsMetadata;
345 auto I = Store.find(Val: &C);
346 ValueAsMetadata *MD = I->second;
347 using UseTy =
348 std::pair<void *, std::pair<MetadataTracking::OwnerTy, uint64_t>>;
349 // Copy out uses and update value of Constant used by debug info metadata with
350 // poison below
351 SmallVector<UseTy, 8> Uses(MD->UseMap.begin(), MD->UseMap.end());
352
353 for (const auto &Pair : Uses) {
354 MetadataTracking::OwnerTy Owner = Pair.second.first;
355 if (!Owner)
356 continue;
357 // Check for MetadataAsValue.
358 if (isa<MetadataAsValue *>(Val: Owner)) {
359 cast<MetadataAsValue *>(Val&: Owner)->handleChangedMetadata(
360 MD: ValueAsMetadata::get(V: PoisonValue::get(T: C.getType())));
361 continue;
362 }
363 if (!isa<Metadata *>(Val: Owner))
364 continue;
365 auto *OwnerMD = dyn_cast_if_present<MDNode>(Val: cast<Metadata *>(Val&: Owner));
366 if (!OwnerMD)
367 continue;
368 if (isa<DINode>(Val: OwnerMD)) {
369 OwnerMD->handleChangedOperand(
370 Ref: Pair.first, New: ValueAsMetadata::get(V: PoisonValue::get(T: C.getType())));
371 }
372 }
373}
374
375void ReplaceableUses::replaceAllUsesWith(Metadata *MD) {
376 if (UseMap.empty())
377 return;
378
379 // Copy out uses since UseMap will get touched below.
380 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
381 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
382 llvm::sort(C&: Uses, Comp: [](const UseTy &L, const UseTy &R) {
383 return L.second.second < R.second.second;
384 });
385 for (const auto &Pair : Uses) {
386 // Check that this Ref hasn't disappeared after RAUW (when updating a
387 // previous Ref).
388 if (!UseMap.count(Val: Pair.first))
389 continue;
390
391 OwnerTy Owner = Pair.second.first;
392 if (!Owner) {
393 // Update unowned tracking references directly.
394 Metadata *&Ref = *static_cast<Metadata **>(Pair.first);
395 Ref = MD;
396 if (MD)
397 MetadataTracking::track(MD&: Ref);
398 UseMap.erase(Val: Pair.first);
399 continue;
400 }
401
402 // Check for MetadataAsValue.
403 if (isa<MetadataAsValue *>(Val: Owner)) {
404 cast<MetadataAsValue *>(Val&: Owner)->handleChangedMetadata(MD);
405 continue;
406 }
407
408 if (auto *DVU = dyn_cast<DebugValueUser *>(Val&: Owner)) {
409 DVU->handleChangedValue(Old: Pair.first, New: MD);
410 continue;
411 }
412
413 // There's a Metadata owner -- dispatch.
414 Metadata *OwnerMD = cast<Metadata *>(Val&: Owner);
415 switch (OwnerMD->getMetadataID()) {
416#define HANDLE_METADATA_LEAF(CLASS) \
417 case Metadata::CLASS##Kind: \
418 cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \
419 continue;
420#include "llvm/IR/Metadata.def"
421 default:
422 llvm_unreachable("Invalid metadata subclass");
423 }
424 }
425 assert(UseMap.empty() && "Expected all uses to be replaced");
426}
427
428void ReplaceableUses::resolveAllUses(bool ResolveUsers) {
429 if (UseMap.empty())
430 return;
431
432 if (!ResolveUsers) {
433 UseMap.clear();
434 return;
435 }
436
437 // Copy out uses since UseMap could get touched below.
438 using UseTy = std::pair<void *, std::pair<OwnerTy, uint64_t>>;
439 SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
440 llvm::sort(C&: Uses, Comp: [](const UseTy &L, const UseTy &R) {
441 return L.second.second < R.second.second;
442 });
443 UseMap.clear();
444 for (const auto &Pair : Uses) {
445 auto Owner = Pair.second.first;
446 if (!Owner)
447 continue;
448 if (!isa<Metadata *>(Val: Owner))
449 continue;
450
451 // Resolve MDNodes that point at this.
452 auto *OwnerMD = dyn_cast_if_present<MDNode>(Val: cast<Metadata *>(Val&: Owner));
453 if (!OwnerMD)
454 continue;
455 if (OwnerMD->isResolved())
456 continue;
457 OwnerMD->decrementUnresolvedOperandCount();
458 }
459}
460
461// A value without a use list (e.g. ConstantData) is never RAUW'd, so don't
462// create a ReplaceableUses instance for it.
463static bool isTrackedValue(const Metadata &MD) {
464 auto *VAM = dyn_cast<ValueAsMetadata>(Val: &MD);
465 return VAM && VAM->getValue()->hasUseList();
466}
467
468// Special handing of DIArgList is required in the RemoveDIs project, see
469// commentry in DIArgList::handleChangedOperand for details. Hidden behind
470// conditional compilation to avoid a compile time regression.
471ReplaceableUses *ReplaceableUses::getOrCreate(Metadata &MD) {
472 if (auto *N = dyn_cast<MDNode>(Val: &MD)) {
473 return N->isResolved() ? nullptr : N->Context.getOrCreateReplaceableUses();
474 }
475 if (auto ArgList = dyn_cast<DIArgList>(Val: &MD))
476 return ArgList;
477 return isTrackedValue(MD) ? cast<ValueAsMetadata>(Val: &MD) : nullptr;
478}
479
480ReplaceableUses *ReplaceableUses::getIfExists(Metadata &MD) {
481 if (auto *N = dyn_cast<MDNode>(Val: &MD)) {
482 return N->isResolved() ? nullptr : N->Context.getReplaceableUses();
483 }
484 if (auto ArgList = dyn_cast<DIArgList>(Val: &MD))
485 return ArgList;
486 return isTrackedValue(MD) ? cast<ValueAsMetadata>(Val: &MD) : nullptr;
487}
488
489bool ReplaceableUses::isReplaceable(const Metadata &MD) {
490 if (auto *N = dyn_cast<MDNode>(Val: &MD))
491 return !N->isResolved();
492 return isTrackedValue(MD) || isa<DIArgList>(Val: &MD);
493}
494
495static DISubprogram *getLocalFunctionMetadata(Value *V) {
496 assert(V && "Expected value");
497 if (auto *A = dyn_cast<Argument>(Val: V)) {
498 if (auto *Fn = A->getParent())
499 return Fn->getSubprogram();
500 return nullptr;
501 }
502
503 if (BasicBlock *BB = cast<Instruction>(Val: V)->getParent()) {
504 if (auto *Fn = BB->getParent())
505 return Fn->getSubprogram();
506 return nullptr;
507 }
508
509 return nullptr;
510}
511
512ValueAsMetadata *ValueAsMetadata::get(Value *V) {
513 assert(V && "Unexpected null Value");
514
515 auto &Context = V->getContext();
516 auto *&Entry = Context.pImpl->ValuesAsMetadata[V];
517 if (!Entry) {
518 assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) &&
519 "Expected constant or function-local value");
520 assert(!V->IsUsedByMD && "Expected this to be the only metadata use");
521 V->IsUsedByMD = true;
522 if (auto *C = dyn_cast<Constant>(Val: V))
523 Entry = new ConstantAsMetadata(C);
524 else
525 Entry = new LocalAsMetadata(V);
526 }
527
528 return Entry;
529}
530
531ValueAsMetadata *ValueAsMetadata::getIfExists(Value *V) {
532 assert(V && "Unexpected null Value");
533 return V->getContext().pImpl->ValuesAsMetadata.lookup(Val: V);
534}
535
536void ValueAsMetadata::handleDeletion(Value *V) {
537 assert(V && "Expected valid value");
538
539 auto &Store = V->getType()->getContext().pImpl->ValuesAsMetadata;
540 auto I = Store.find(Val: V);
541 if (I == Store.end())
542 return;
543
544 // Remove old entry from the map.
545 ValueAsMetadata *MD = I->second;
546 assert(MD && "Expected valid metadata");
547 assert(MD->getValue() == V && "Expected valid mapping");
548 Store.erase(I);
549
550 // Delete the metadata.
551 MD->replaceAllUsesWith(MD: nullptr);
552 delete MD;
553}
554
555void ValueAsMetadata::handleRAUW(Value *From, Value *To) {
556 assert(From && "Expected valid value");
557 assert(To && "Expected valid value");
558 assert(From != To && "Expected changed value");
559 assert(&From->getContext() == &To->getContext() && "Expected same context");
560 assert(From->hasUseList() && "Must have use list");
561
562 auto &Store = From->getContext().pImpl->ValuesAsMetadata;
563 auto I = Store.find(Val: From);
564 if (I == Store.end()) {
565 assert(!From->IsUsedByMD && "Expected From not to be used by metadata");
566 return;
567 }
568
569 assert(From->IsUsedByMD && "Expected From to be used by metadata");
570 From->IsUsedByMD = false;
571 ValueAsMetadata *MD = I->second;
572 assert(MD && "Expected valid metadata");
573 assert(MD->getValue() == From && "Expected valid mapping");
574 Store.erase(I);
575
576 // Move the uses to To's node. Uses of a function-local value are dropped if
577 // it becomes a local of another function or replaces a constant.
578 Metadata *New = nullptr;
579 if (isa<Constant>(Val: To)) {
580 New = ValueAsMetadata::get(V: To);
581 } else if (isa<LocalAsMetadata>(Val: MD)) {
582 DISubprogram *FromSP = getLocalFunctionMetadata(V: From);
583 DISubprogram *ToSP = FromSP ? getLocalFunctionMetadata(V: To) : nullptr;
584 if (!FromSP || !ToSP || FromSP == ToSP)
585 New = ValueAsMetadata::get(V: To);
586 }
587 MD->replaceAllUsesWith(MD: New);
588 delete MD;
589}
590
591//===----------------------------------------------------------------------===//
592// MDString implementation.
593//
594
595MDString *MDString::get(LLVMContext &Context, StringRef Str) {
596 auto &Store = Context.pImpl->MDStringCache;
597 auto I = Store.try_emplace(Key: Str);
598 auto &MapEntry = I.first->getValue();
599 if (!I.second)
600 return &MapEntry;
601 MapEntry.Entry = &*I.first;
602 return &MapEntry;
603}
604
605MDString *MDString::getIfExists(LLVMContext &Context, StringRef Str) {
606 auto &Store = Context.pImpl->MDStringCache;
607 auto I = Store.find(Key: Str);
608 if (I == Store.end())
609 return nullptr;
610 return &I->getValue();
611}
612
613StringRef MDString::getString() const {
614 assert(Entry && "Expected to find string map entry");
615 return Entry->first();
616}
617
618//===----------------------------------------------------------------------===//
619// MDNode implementation.
620//
621
622// Assert that the MDNode types will not be unaligned by the objects
623// prepended to them.
624#define HANDLE_MDNODE_LEAF(CLASS) \
625 static_assert( \
626 alignof(uint64_t) >= alignof(CLASS), \
627 "Alignment is insufficient after objects prepended to " #CLASS);
628#include "llvm/IR/Metadata.def"
629
630void *MDNode::operator new(size_t Size, size_t NumOps, StorageType Storage) {
631 // uint64_t is the most aligned type we need support (ensured by static_assert
632 // above)
633 static_assert(sizeof(Header) == sizeof(size_t) + 2 * sizeof(uint32_t),
634 "MDNode header fields poorly packed");
635 size_t AllocSize =
636 alignTo(Value: Header::getAllocSize(Storage, NumOps), Align: alignof(uint64_t));
637 char *Mem = reinterpret_cast<char *>(::operator new(AllocSize + Size));
638 Header *H = new (Mem + AllocSize - sizeof(Header)) Header(NumOps, Storage);
639 return reinterpret_cast<void *>(H + 1);
640}
641
642void MDNode::operator delete(void *N) {
643 Header *H = reinterpret_cast<Header *>(N) - 1;
644 void *Mem = H->getAllocation();
645 H->~Header();
646 ::operator delete(Mem);
647}
648
649MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage,
650 ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2)
651 : Metadata(ID, Storage), Context(Context) {
652 getHeader().MetadataPrintID = Context.pImpl->allocateMetadataPrintID();
653
654 unsigned Op = 0;
655 for (Metadata *MD : Ops1)
656 setOperand(I: Op++, New: MD);
657 for (Metadata *MD : Ops2)
658 setOperand(I: Op++, New: MD);
659
660 if (!isUniqued())
661 return;
662
663 // Count the unresolved operands. If there are any, RAUW support will be
664 // added lazily on first reference.
665 countUnresolvedOperands();
666}
667
668TempMDNode MDNode::clone() const {
669 switch (getMetadataID()) {
670 default:
671 llvm_unreachable("Invalid MDNode subclass");
672#define HANDLE_MDNODE_LEAF(CLASS) \
673 case CLASS##Kind: \
674 return cast<CLASS>(this)->cloneImpl();
675#include "llvm/IR/Metadata.def"
676 }
677}
678
679MDNode::Header::Header(size_t NumOps, StorageType Storage) {
680 IsLarge = isLarge(NumOps);
681 IsResizable = isResizable(Storage);
682 SmallSize = getSmallSize(NumOps, IsResizable, IsLarge);
683 if (IsLarge) {
684 SmallNumOps = 0;
685 new (getLargePtr()) LargeStorageVector();
686 getLarge().resize(N: NumOps);
687 return;
688 }
689 SmallNumOps = NumOps;
690 MDOperand *O = reinterpret_cast<MDOperand *>(this) - SmallSize;
691 for (MDOperand *E = O + SmallSize; O != E;)
692 (void)new (O++) MDOperand();
693}
694
695MDNode::Header::~Header() {
696 if (IsLarge) {
697 getLarge().~LargeStorageVector();
698 return;
699 }
700 MDOperand *O = reinterpret_cast<MDOperand *>(this);
701 for (MDOperand *E = O - SmallSize; O != E; --O)
702 (O - 1)->~MDOperand();
703}
704
705void *MDNode::Header::getSmallPtr() {
706 static_assert(alignof(MDOperand) <= alignof(Header),
707 "MDOperand too strongly aligned");
708 return reinterpret_cast<char *>(const_cast<Header *>(this)) -
709 sizeof(MDOperand) * SmallSize;
710}
711
712void MDNode::Header::resize(size_t NumOps) {
713 assert(IsResizable && "Node is not resizable");
714 if (operands().size() == NumOps)
715 return;
716
717 if (IsLarge)
718 getLarge().resize(N: NumOps);
719 else if (NumOps <= SmallSize)
720 resizeSmall(NumOps);
721 else
722 resizeSmallToLarge(NumOps);
723}
724
725void MDNode::Header::resizeSmall(size_t NumOps) {
726 assert(!IsLarge && "Expected a small MDNode");
727 assert(NumOps <= SmallSize && "NumOps too large for small resize");
728
729 MutableArrayRef<MDOperand> ExistingOps = operands();
730 assert(NumOps != ExistingOps.size() && "Expected a different size");
731
732 int NumNew = (int)NumOps - (int)ExistingOps.size();
733 MDOperand *O = ExistingOps.end();
734 for (int I = 0, E = NumNew; I < E; ++I)
735 (O++)->reset();
736 for (int I = 0, E = NumNew; I > E; --I)
737 (--O)->reset();
738 SmallNumOps = NumOps;
739 assert(O == operands().end() && "Operands not (un)initialized until the end");
740}
741
742void MDNode::Header::resizeSmallToLarge(size_t NumOps) {
743 assert(!IsLarge && "Expected a small MDNode");
744 assert(NumOps > SmallSize && "Expected NumOps to be larger than allocation");
745 LargeStorageVector NewOps;
746 NewOps.resize(N: NumOps);
747 llvm::move(Range: operands(), Out: NewOps.begin());
748 resizeSmall(NumOps: 0);
749 new (getLargePtr()) LargeStorageVector(std::move(NewOps));
750 IsLarge = true;
751}
752
753static bool isOperandUnresolved(Metadata *Op) {
754 if (auto *N = dyn_cast_or_null<MDNode>(Val: Op))
755 return !N->isResolved();
756 return false;
757}
758
759void MDNode::countUnresolvedOperands() {
760 assert(getNumUnresolved() == 0 && "Expected unresolved ops to be uncounted");
761 assert(isUniqued() && "Expected this to be uniqued");
762 setNumUnresolved(count_if(Range: operands(), P: isOperandUnresolved));
763}
764
765void MDNode::makeUniqued() {
766 assert(isTemporary() && "Expected this to be temporary");
767 assert(!isResolved() && "Expected this to be unresolved");
768 bool WasTracked = getContext().pImpl->TemporaryMDNodes.erase(V: this);
769 assert(WasTracked && "Temporary node not tracked");
770 (void)WasTracked;
771
772 // Enable uniquing callbacks.
773 for (auto &Op : mutable_operands())
774 Op.reset(MD: Op.get(), Owner: this);
775
776 // Make this 'uniqued'.
777 Storage = Uniqued;
778 countUnresolvedOperands();
779 if (!getNumUnresolved()) {
780 dropReplaceableUses();
781 assert(isResolved() && "Expected this to be resolved");
782 }
783
784 assert(isUniqued() && "Expected this to be uniqued");
785}
786
787void MDNode::makeDistinct() {
788 assert(isTemporary() && "Expected this to be temporary");
789 assert(!isResolved() && "Expected this to be unresolved");
790
791 // Drop RAUW support and store as a distinct node.
792 dropReplaceableUses();
793 storeDistinctInContext();
794
795 assert(isDistinct() && "Expected this to be distinct");
796 assert(isResolved() && "Expected this to be resolved");
797}
798
799void MDNode::resolve() {
800 assert(isUniqued() && "Expected this to be uniqued");
801 assert(!isResolved() && "Expected this to be unresolved");
802
803 setNumUnresolved(0);
804 dropReplaceableUses();
805
806 assert(isResolved() && "Expected this to be resolved");
807}
808
809void MDNode::dropReplaceableUses() {
810 assert(!getNumUnresolved() && "Unexpected unresolved operand");
811
812 // Drop any RAUW support.
813 if (Context.hasReplaceableUses())
814 Context.takeReplaceableUses()->resolveAllUses();
815}
816
817void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) {
818 assert(isUniqued() && "Expected this to be uniqued");
819 assert(getNumUnresolved() != 0 && "Expected unresolved operands");
820
821 // Check if an operand was resolved.
822 if (!isOperandUnresolved(Op: Old)) {
823 if (isOperandUnresolved(Op: New))
824 // An operand was un-resolved!
825 setNumUnresolved(getNumUnresolved() + 1);
826 } else if (!isOperandUnresolved(Op: New))
827 decrementUnresolvedOperandCount();
828}
829
830void MDNode::decrementUnresolvedOperandCount() {
831 assert(!isResolved() && "Expected this to be unresolved");
832 if (isTemporary())
833 return;
834
835 assert(isUniqued() && "Expected this to be uniqued");
836 setNumUnresolved(getNumUnresolved() - 1);
837 if (getNumUnresolved())
838 return;
839
840 // Last unresolved operand has just been resolved.
841 dropReplaceableUses();
842 assert(isResolved() && "Expected this to become resolved");
843}
844
845void MDNode::resolveCycles() {
846 if (isResolved())
847 return;
848
849 // Resolve this node immediately.
850 resolve();
851
852 // Resolve all operands.
853 for (const auto &Op : operands()) {
854 auto *N = dyn_cast_or_null<MDNode>(Val: Op);
855 if (!N)
856 continue;
857
858 assert(!N->isTemporary() &&
859 "Expected all forward declarations to be resolved");
860 if (!N->isResolved())
861 N->resolveCycles();
862 }
863}
864
865static bool hasSelfReference(MDNode *N) {
866 return llvm::is_contained(Range: N->operands(), Element: N);
867}
868
869MDNode *MDNode::replaceWithPermanentImpl() {
870 switch (getMetadataID()) {
871 default:
872 // If this type isn't uniquable, replace with a distinct node.
873 return replaceWithDistinctImpl();
874
875#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
876 case CLASS##Kind: \
877 break;
878#include "llvm/IR/Metadata.def"
879 }
880
881 // Even if this type is uniquable, self-references have to be distinct.
882 if (hasSelfReference(N: this))
883 return replaceWithDistinctImpl();
884 return replaceWithUniquedImpl();
885}
886
887MDNode *MDNode::replaceWithUniquedImpl() {
888 // Try to uniquify in place.
889 MDNode *UniquedNode = uniquify();
890
891 if (UniquedNode == this) {
892 makeUniqued();
893 return this;
894 }
895
896 // Collision, so RAUW instead.
897 replaceAllUsesWith(MD: UniquedNode);
898 deleteAsSubclass();
899 return UniquedNode;
900}
901
902MDNode *MDNode::replaceWithDistinctImpl() {
903 makeDistinct();
904 return this;
905}
906
907void MDTuple::recalculateHash() {
908 setHash(MDTupleInfo::KeyTy::calculateHash(N: this));
909}
910
911void MDNode::dropAllReferences() {
912 for (unsigned I = 0, E = getNumOperands(); I != E; ++I)
913 setOperand(I, New: nullptr);
914 if (Context.hasReplaceableUses()) {
915 Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false);
916 (void)Context.takeReplaceableUses();
917 }
918}
919
920void MDNode::handleChangedOperand(void *Ref, Metadata *New) {
921 unsigned Op = static_cast<MDOperand *>(Ref) - op_begin();
922 assert(Op < getNumOperands() && "Expected valid operand");
923
924 if (!isUniqued()) {
925 // This node is not uniqued. Just set the operand and be done with it.
926 setOperand(I: Op, New);
927 return;
928 }
929
930 // This node is uniqued.
931 eraseFromStore();
932
933 Metadata *Old = getOperand(I: Op);
934 setOperand(I: Op, New);
935
936 // Drop uniquing for self-reference cycles and deleted constants.
937 if (New == this || (!New && Old && isa<ConstantAsMetadata>(Val: Old))) {
938 if (!isResolved())
939 resolve();
940 storeDistinctInContext();
941 return;
942 }
943
944 // Re-unique the node.
945 auto *Uniqued = uniquify();
946 if (Uniqued == this) {
947 if (!isResolved())
948 resolveAfterOperandChange(Old, New);
949 return;
950 }
951
952 // Collision.
953 if (!isResolved()) {
954 // Still unresolved, so RAUW.
955 //
956 // First, clear out all operands to prevent any recursion (similar to
957 // dropAllReferences(), but we still need the use-list).
958 for (unsigned O = 0, E = getNumOperands(); O != E; ++O)
959 setOperand(I: O, New: nullptr);
960 if (Context.hasReplaceableUses())
961 Context.getReplaceableUses()->replaceAllUsesWith(MD: Uniqued);
962 deleteAsSubclass();
963 return;
964 }
965
966 // Store in non-uniqued form if RAUW isn't possible.
967 storeDistinctInContext();
968}
969
970void MDNode::deleteAsSubclass() {
971 if (isTemporary()) {
972 bool WasTracked = getContext().pImpl->TemporaryMDNodes.erase(V: this);
973 assert(WasTracked && "Temporary node not tracked");
974 (void)WasTracked;
975 }
976 switch (getMetadataID()) {
977 default:
978 llvm_unreachable("Invalid subclass of MDNode");
979#define HANDLE_MDNODE_LEAF(CLASS) \
980 case CLASS##Kind: \
981 delete cast<CLASS>(this); \
982 break;
983#include "llvm/IR/Metadata.def"
984 }
985}
986
987template <class T, class InfoT>
988static T *uniquifyImpl(T *N, DenseSet<T *, InfoT> &Store) {
989 if (T *U = getUniqued(Store, N))
990 return U;
991
992 Store.insert(N);
993 return N;
994}
995
996template <class NodeTy> struct MDNode::HasCachedHash {
997 template <class U>
998 static std::true_type check(SameType<void (U::*)(unsigned), &U::setHash> *);
999 template <class U> static std::false_type check(...);
1000
1001 static constexpr bool value = decltype(check<NodeTy>(nullptr))::value;
1002};
1003
1004MDNode *MDNode::uniquify() {
1005 assert(!hasSelfReference(this) && "Cannot uniquify a self-referencing node");
1006
1007 // Try to insert into uniquing store.
1008 switch (getMetadataID()) {
1009 default:
1010 llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
1011#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
1012 case CLASS##Kind: { \
1013 CLASS *SubclassThis = cast<CLASS>(this); \
1014 dispatchRecalculateHash(SubclassThis); \
1015 return uniquifyImpl(SubclassThis, getContext().pImpl->CLASS##s); \
1016 }
1017#include "llvm/IR/Metadata.def"
1018 }
1019}
1020
1021void MDNode::eraseFromStore() {
1022 switch (getMetadataID()) {
1023 default:
1024 llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
1025#define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS) \
1026 case CLASS##Kind: \
1027 getContext().pImpl->CLASS##s.erase(cast<CLASS>(this)); \
1028 break;
1029#include "llvm/IR/Metadata.def"
1030 }
1031}
1032
1033MDTuple *MDTuple::getImpl(LLVMContext &Context, ArrayRef<Metadata *> MDs,
1034 StorageType Storage, bool ShouldCreate) {
1035 unsigned Hash = 0;
1036 if (Storage == Uniqued) {
1037 MDTupleInfo::KeyTy Key(MDs);
1038 if (auto *N = getUniqued(Store&: Context.pImpl->MDTuples, Key))
1039 return N;
1040 if (!ShouldCreate)
1041 return nullptr;
1042 Hash = Key.getHash();
1043 } else {
1044 assert(ShouldCreate && "Expected non-uniqued nodes to always be created");
1045 }
1046
1047 return storeImpl(N: new (MDs.size(), Storage)
1048 MDTuple(Context, Storage, Hash, MDs),
1049 Storage, Store&: Context.pImpl->MDTuples);
1050}
1051
1052void MDNode::deleteTemporary(MDNode *N) {
1053 assert(N->isTemporary() && "Expected temporary node");
1054 N->replaceAllUsesWith(MD: nullptr);
1055 N->deleteAsSubclass();
1056}
1057
1058void MDNode::storeDistinctInContext() {
1059 assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses");
1060 assert(!getNumUnresolved() && "Unexpected unresolved nodes");
1061 if (isTemporary()) {
1062 bool WasTracked = getContext().pImpl->TemporaryMDNodes.erase(V: this);
1063 assert(WasTracked && "Temporary node not tracked");
1064 (void)WasTracked;
1065 }
1066 Storage = Distinct;
1067 assert(isResolved() && "Expected this to be resolved");
1068
1069 // Reset the hash.
1070 switch (getMetadataID()) {
1071 default:
1072 llvm_unreachable("Invalid subclass of MDNode");
1073#define HANDLE_MDNODE_LEAF(CLASS) \
1074 case CLASS##Kind: { \
1075 dispatchResetHash(cast<CLASS>(this)); \
1076 break; \
1077 }
1078#include "llvm/IR/Metadata.def"
1079 }
1080
1081 getContext().pImpl->DistinctMDNodes.push_back(x: this);
1082}
1083
1084void MDNode::replaceOperandWith(unsigned I, Metadata *New) {
1085 if (getOperand(I) == New)
1086 return;
1087
1088 if (!isUniqued()) {
1089 setOperand(I, New);
1090 return;
1091 }
1092
1093 handleChangedOperand(Ref: mutable_begin() + I, New);
1094}
1095
1096void MDNode::setOperand(unsigned I, Metadata *New) {
1097 assert(I < getNumOperands());
1098 mutable_begin()[I].reset(MD: New, Owner: isUniqued() ? this : nullptr);
1099}
1100
1101/// Get a node or a self-reference that looks like it.
1102///
1103/// Special handling for finding self-references, for use by \a
1104/// MDNode::concatenate() and \a MDNode::intersect() to maintain behaviour from
1105/// when self-referencing nodes were still uniqued. If the first operand has
1106/// the same operands as \c Ops, return the first operand instead.
1107static MDNode *getOrSelfReference(LLVMContext &Context,
1108 ArrayRef<Metadata *> Ops) {
1109 if (!Ops.empty())
1110 if (MDNode *N = dyn_cast_or_null<MDNode>(Val: Ops[0]))
1111 if (N->getNumOperands() == Ops.size() && N == N->getOperand(I: 0)) {
1112 for (unsigned I = 1, E = Ops.size(); I != E; ++I)
1113 if (Ops[I] != N->getOperand(I))
1114 return MDNode::get(Context, MDs: Ops);
1115 return N;
1116 }
1117
1118 return MDNode::get(Context, MDs: Ops);
1119}
1120
1121MDNode *MDNode::concatenate(MDNode *A, MDNode *B) {
1122 if (!A)
1123 return B;
1124 if (!B)
1125 return A;
1126
1127 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end());
1128 MDs.insert(Start: B->op_begin(), End: B->op_end());
1129
1130 // FIXME: This preserves long-standing behaviour, but is it really the right
1131 // behaviour? Or was that an unintended side-effect of node uniquing?
1132 return getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1133}
1134
1135MDNode *MDNode::intersect(MDNode *A, MDNode *B) {
1136 if (!A || !B)
1137 return nullptr;
1138
1139 SmallSetVector<Metadata *, 4> MDs(A->op_begin(), A->op_end());
1140 SmallPtrSet<Metadata *, 4> BSet(B->op_begin(), B->op_end());
1141 MDs.remove_if(P: [&](Metadata *MD) { return !BSet.count(Ptr: MD); });
1142
1143 // FIXME: This preserves long-standing behaviour, but is it really the right
1144 // behaviour? Or was that an unintended side-effect of node uniquing?
1145 return getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1146}
1147
1148MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) {
1149 if (!A || !B)
1150 return nullptr;
1151
1152 // Take the intersection of domains then union the scopes
1153 // within those domains
1154 SmallPtrSet<const MDNode *, 16> ADomains;
1155 SmallPtrSet<const MDNode *, 16> IntersectDomains;
1156 SmallSetVector<Metadata *, 4> MDs;
1157 for (const MDOperand &MDOp : A->operands())
1158 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1159 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1160 ADomains.insert(Ptr: Domain);
1161
1162 for (const MDOperand &MDOp : B->operands())
1163 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1164 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1165 if (ADomains.contains(Ptr: Domain)) {
1166 IntersectDomains.insert(Ptr: Domain);
1167 MDs.insert(X: MDOp);
1168 }
1169
1170 for (const MDOperand &MDOp : A->operands())
1171 if (const MDNode *NAMD = dyn_cast<MDNode>(Val: MDOp))
1172 if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
1173 if (IntersectDomains.contains(Ptr: Domain))
1174 MDs.insert(X: MDOp);
1175
1176 return MDs.empty() ? nullptr
1177 : getOrSelfReference(Context&: A->getContext(), Ops: MDs.getArrayRef());
1178}
1179
1180MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) {
1181 if (!A || !B)
1182 return nullptr;
1183
1184 APFloat AVal = mdconst::extract<ConstantFP>(MD: A->getOperand(I: 0))->getValueAPF();
1185 APFloat BVal = mdconst::extract<ConstantFP>(MD: B->getOperand(I: 0))->getValueAPF();
1186 if (AVal < BVal)
1187 return A;
1188 return B;
1189}
1190
1191// Call instructions with branch weights are only used in SamplePGO as
1192// documented in
1193/// https://llvm.org/docs/BranchWeightMetadata.html#callinst).
1194MDNode *MDNode::mergeDirectCallProfMetadata(MDNode *A, MDNode *B,
1195 const Instruction *AInstr,
1196 const Instruction *BInstr) {
1197 assert(A && B && AInstr && BInstr && "Caller should guarantee");
1198 auto &Ctx = AInstr->getContext();
1199 MDBuilder MDHelper(Ctx);
1200
1201 // LLVM IR verifier verifies !prof metadata has at least 2 operands.
1202 assert(A->getNumOperands() >= 2 && B->getNumOperands() >= 2 &&
1203 "!prof annotations should have no less than 2 operands");
1204 MDString *AMDS = dyn_cast<MDString>(Val: A->getOperand(I: 0));
1205 MDString *BMDS = dyn_cast<MDString>(Val: B->getOperand(I: 0));
1206 // LLVM IR verfier verifies first operand is MDString.
1207 assert(AMDS != nullptr && BMDS != nullptr &&
1208 "first operand should be a non-null MDString");
1209 StringRef AProfName = AMDS->getString();
1210 StringRef BProfName = BMDS->getString();
1211 if (AProfName == MDProfLabels::BranchWeights &&
1212 BProfName == MDProfLabels::BranchWeights) {
1213 ConstantInt *AInstrWeight = mdconst::dyn_extract<ConstantInt>(
1214 MD: A->getOperand(I: getBranchWeightOffset(ProfileData: A)));
1215 ConstantInt *BInstrWeight = mdconst::dyn_extract<ConstantInt>(
1216 MD: B->getOperand(I: getBranchWeightOffset(ProfileData: B)));
1217 assert(AInstrWeight && BInstrWeight && "verified by LLVM verifier");
1218 return MDNode::get(Context&: Ctx,
1219 MDs: {MDHelper.createString(Str: MDProfLabels::BranchWeights),
1220 MDHelper.createConstant(C: ConstantInt::get(
1221 Ty: Type::getInt64Ty(C&: Ctx),
1222 V: SaturatingAdd(X: AInstrWeight->getZExtValue(),
1223 Y: BInstrWeight->getZExtValue())))});
1224 }
1225 return nullptr;
1226}
1227
1228// Pass in both instructions and nodes. Instruction information (e.g.,
1229// instruction type) helps interpret profiles and make implementation clearer.
1230MDNode *MDNode::getMergedProfMetadata(MDNode *A, MDNode *B,
1231 const Instruction *AInstr,
1232 const Instruction *BInstr) {
1233 // Check that it is legal to merge prof metadata based on the opcode.
1234 auto IsLegal = [](const Instruction &I) -> bool {
1235 switch (I.getOpcode()) {
1236 case Instruction::Invoke:
1237 case Instruction::CondBr:
1238 case Instruction::Switch:
1239 case Instruction::Call:
1240 case Instruction::IndirectBr:
1241 case Instruction::Select:
1242 case Instruction::CallBr:
1243 return true;
1244 default:
1245 return false;
1246 }
1247 };
1248 if (AInstr && !IsLegal(*AInstr))
1249 return nullptr;
1250 if (BInstr && !IsLegal(*BInstr))
1251 return nullptr;
1252
1253 if (!(A && B)) {
1254 return A ? A : B;
1255 }
1256
1257 assert(AInstr->getMetadata(LLVMContext::MD_prof) == A &&
1258 "Caller should guarantee");
1259 assert(BInstr->getMetadata(LLVMContext::MD_prof) == B &&
1260 "Caller should guarantee");
1261
1262 const CallInst *ACall = dyn_cast<CallInst>(Val: AInstr);
1263 const CallInst *BCall = dyn_cast<CallInst>(Val: BInstr);
1264
1265 // Both ACall and BCall are direct callsites.
1266 if (ACall && BCall && ACall->getCalledFunction() &&
1267 BCall->getCalledFunction())
1268 return mergeDirectCallProfMetadata(A, B, AInstr, BInstr);
1269
1270 if (A == B)
1271 return A;
1272
1273 // The rest of the cases are not implemented but could be added
1274 // when there are use cases.
1275 return nullptr;
1276}
1277
1278static bool isContiguous(const ConstantRange &A, const ConstantRange &B) {
1279 return A.getUpper() == B.getLower() || A.getLower() == B.getUpper();
1280}
1281
1282static bool canBeMerged(const ConstantRange &A, const ConstantRange &B) {
1283 return !A.intersectWith(CR: B).isEmptySet() || isContiguous(A, B);
1284}
1285
1286static bool tryMergeRange(SmallVectorImpl<ConstantInt *> &EndPoints,
1287 ConstantInt *Low, ConstantInt *High) {
1288 ConstantRange NewRange(Low->getValue(), High->getValue());
1289 unsigned Size = EndPoints.size();
1290 const APInt &LB = EndPoints[Size - 2]->getValue();
1291 const APInt &LE = EndPoints[Size - 1]->getValue();
1292 ConstantRange LastRange(LB, LE);
1293 if (canBeMerged(A: NewRange, B: LastRange)) {
1294 ConstantRange Union = LastRange.unionWith(CR: NewRange);
1295 Type *Ty = High->getType();
1296 EndPoints[Size - 2] =
1297 cast<ConstantInt>(Val: ConstantInt::get(Ty, V: Union.getLower()));
1298 EndPoints[Size - 1] =
1299 cast<ConstantInt>(Val: ConstantInt::get(Ty, V: Union.getUpper()));
1300 return true;
1301 }
1302 return false;
1303}
1304
1305static void addRange(SmallVectorImpl<ConstantInt *> &EndPoints,
1306 ConstantInt *Low, ConstantInt *High) {
1307 if (!EndPoints.empty())
1308 if (tryMergeRange(EndPoints, Low, High))
1309 return;
1310
1311 EndPoints.push_back(Elt: Low);
1312 EndPoints.push_back(Elt: High);
1313}
1314
1315MDNode *MDNode::getMergedCalleeTypeMetadata(const MDNode *A, const MDNode *B) {
1316 // Drop the callee_type metadata if either of the call instructions do not
1317 // have it.
1318 if (!A || !B)
1319 return nullptr;
1320 SmallVector<Metadata *, 8> AB;
1321 SmallPtrSet<Metadata *, 8> MergedCallees;
1322 auto AddUniqueCallees = [&AB, &MergedCallees](const MDNode *N) {
1323 for (Metadata *MD : N->operands()) {
1324 if (MergedCallees.insert(Ptr: MD).second)
1325 AB.push_back(Elt: MD);
1326 }
1327 };
1328 AddUniqueCallees(A);
1329 AddUniqueCallees(B);
1330 return MDNode::get(Context&: A->getContext(), MDs: AB);
1331}
1332
1333MDNode *MDNode::getMergedCalleesMetadata(MDNode *A, MDNode *B) {
1334 // The callees of the merged call are unknown unless both calls list theirs.
1335 if (!A || !B)
1336 return nullptr;
1337 if (A == B)
1338 return A;
1339 // The merged call may target any callee of either call.
1340 SmallSetVector<Metadata *, 8> Callees(llvm::from_range, A->operands());
1341 Callees.insert_range(R: B->operands());
1342 return MDNode::get(Context&: A->getContext(), MDs: Callees.getArrayRef());
1343}
1344
1345MDNode *MDNode::getMergedPCSectionsMetadata(MDNode *A, MDNode *B) {
1346 // The merged instruction stands for both, so its PC belongs in every section
1347 // either requested, with each request's auxiliary data.
1348 if (!A || A == B)
1349 return B;
1350 if (!B)
1351 return A;
1352 // Requests are a section name followed by auxiliary data tuples. Append B's
1353 // requests that A does not already contain.
1354 const auto Requests = [](MDNode *N) {
1355 SmallVector<ArrayRef<MDOperand>, 4> Rs;
1356 const ArrayRef<MDOperand> Ops = N->operands();
1357 for (size_t I = 0; I < Ops.size();) {
1358 const size_t Begin = I++;
1359 while (I < Ops.size() && !isa<MDString>(Val: Ops[I]))
1360 ++I;
1361 Rs.push_back(Elt: Ops.slice(N: Begin, M: I - Begin));
1362 }
1363 return Rs;
1364 };
1365 const auto RequestsA = Requests(A);
1366 const auto Same = [](ArrayRef<MDOperand> X, ArrayRef<MDOperand> Y) {
1367 return equal(LRange&: X, RRange&: Y, P: [](const MDOperand &L, const MDOperand &R) {
1368 return L.get() == R.get();
1369 });
1370 };
1371 SmallVector<Metadata *, 8> Ops(A->op_begin(), A->op_end());
1372 for (const ArrayRef<MDOperand> R : Requests(B))
1373 if (none_of(Range: RequestsA, P: [&](ArrayRef<MDOperand> RA) { return Same(RA, R); }))
1374 Ops.append(in_start: R.begin(), in_end: R.end());
1375 return MDNode::get(Context&: A->getContext(), MDs: Ops);
1376}
1377
1378MDNode *MDNode::getMergedAllocTokenMetadata(const MDNode *A, const MDNode *B) {
1379 // Drop !alloc_token metadata if either instruction lacks it to avoid mis-
1380 // classifying unclassified allocations, where the fallback token must be
1381 // used instead.
1382 if (!A || !B)
1383 return nullptr;
1384 if (A == B)
1385 return const_cast<MDNode *>(A);
1386 const unsigned NumOps = A->getNumOperands();
1387 if ((NumOps != 2 && NumOps != 3) || B->getNumOperands() != NumOps)
1388 return nullptr;
1389 auto *CIA = mdconst::dyn_extract_or_null<ConstantInt>(MD: A->getOperand(I: 1));
1390 auto *CIB = mdconst::dyn_extract_or_null<ConstantInt>(MD: B->getOperand(I: 1));
1391 if (!CIA || !CIB)
1392 return nullptr;
1393
1394 // Join different names with '|'.
1395 LLVMContext &Ctx = A->getContext();
1396 auto MergeNames = [&](unsigned Idx) -> Metadata * {
1397 auto *NameA = dyn_cast_or_null<MDString>(Val: A->getOperand(I: Idx));
1398 auto *NameB = dyn_cast_or_null<MDString>(Val: B->getOperand(I: Idx));
1399 if (!NameA || !NameB)
1400 return nullptr;
1401 if (NameA == NameB)
1402 return NameA;
1403 // An empty string denotes an unknown type, which must be preserved.
1404 if (Idx == 0) {
1405 if (NameA->getString().empty())
1406 return NameA;
1407 if (NameB->getString().empty())
1408 return NameB;
1409 }
1410 return MDString::get(Context&: Ctx,
1411 Str: (NameA->getString() + "|" + NameB->getString()).str());
1412 };
1413
1414 bool MergedContainsPointer = CIA->isOne() || CIB->isOne();
1415 SmallVector<Metadata *, 3> Ops = {
1416 MergeNames(0), ConstantAsMetadata::get(C: ConstantInt::get(
1417 Ty: Type::getInt1Ty(C&: Ctx), V: MergedContainsPointer))};
1418 if (NumOps == 3)
1419 Ops.push_back(Elt: MergeNames(2));
1420 if (is_contained(Range&: Ops, Element: nullptr))
1421 return nullptr;
1422 return MDNode::get(Context&: Ctx, MDs: Ops);
1423}
1424
1425MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) {
1426 // Given two ranges, we want to compute the union of the ranges. This
1427 // is slightly complicated by having to combine the intervals and merge
1428 // the ones that overlap.
1429
1430 if (!A || !B)
1431 return nullptr;
1432
1433 if (A == B)
1434 return A;
1435
1436 // First, walk both lists in order of the lower boundary of each interval.
1437 // At each step, try to merge the new interval to the last one we added.
1438 SmallVector<ConstantInt *, 4> EndPoints;
1439 unsigned AI = 0;
1440 unsigned BI = 0;
1441 unsigned AN = A->getNumOperands() / 2;
1442 unsigned BN = B->getNumOperands() / 2;
1443 while (AI < AN && BI < BN) {
1444 ConstantInt *ALow = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI));
1445 ConstantInt *BLow = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI));
1446
1447 if (ALow->getValue().slt(RHS: BLow->getValue())) {
1448 addRange(EndPoints, Low: ALow,
1449 High: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI + 1)));
1450 ++AI;
1451 } else {
1452 addRange(EndPoints, Low: BLow,
1453 High: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI + 1)));
1454 ++BI;
1455 }
1456 }
1457 while (AI < AN) {
1458 addRange(EndPoints, Low: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI)),
1459 High: mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * AI + 1)));
1460 ++AI;
1461 }
1462 while (BI < BN) {
1463 addRange(EndPoints, Low: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI)),
1464 High: mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * BI + 1)));
1465 ++BI;
1466 }
1467
1468 // We haven't handled wrap in the previous merge,
1469 // if we have at least 2 ranges (4 endpoints) we have to try to merge
1470 // the last and first ones.
1471 unsigned Size = EndPoints.size();
1472 if (Size > 2) {
1473 ConstantInt *FB = EndPoints[0];
1474 ConstantInt *FE = EndPoints[1];
1475 if (tryMergeRange(EndPoints, Low: FB, High: FE)) {
1476 for (unsigned i = 0; i < Size - 2; ++i) {
1477 EndPoints[i] = EndPoints[i + 2];
1478 }
1479 EndPoints.resize(N: Size - 2);
1480 }
1481 }
1482
1483 // If in the end we have a single range, it is possible that it is now the
1484 // full range. Just drop the metadata in that case.
1485 if (EndPoints.size() == 2) {
1486 ConstantRange Range(EndPoints[0]->getValue(), EndPoints[1]->getValue());
1487 if (Range.isFullSet())
1488 return nullptr;
1489 }
1490
1491 SmallVector<Metadata *, 4> MDs;
1492 MDs.reserve(N: EndPoints.size());
1493 for (auto *I : EndPoints)
1494 MDs.push_back(Elt: ConstantAsMetadata::get(C: I));
1495 return MDNode::get(Context&: A->getContext(), MDs);
1496}
1497
1498MDNode *MDNode::getMostGenericNoFPClass(MDNode *A, MDNode *B) {
1499 if (!A || !B)
1500 return nullptr;
1501
1502 if (A == B)
1503 return A;
1504
1505 ConstantInt *AVal = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 0));
1506 ConstantInt *BVal = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 0));
1507 unsigned Intersect = AVal->getZExtValue() & BVal->getZExtValue();
1508 if (Intersect == 0)
1509 return nullptr;
1510
1511 return MDNode::get(Context&: A->getContext(), MDs: ConstantAsMetadata::get(C: ConstantInt::get(
1512 Ty: AVal->getType(), V: Intersect)));
1513}
1514
1515MDNode *MDNode::getMostGenericNoaliasAddrspace(MDNode *A, MDNode *B) {
1516 if (!A || !B)
1517 return nullptr;
1518
1519 if (A == B)
1520 return A;
1521
1522 SmallVector<ConstantRange> RangeListA, RangeListB;
1523 for (unsigned I = 0, E = A->getNumOperands() / 2; I != E; ++I) {
1524 auto *LowA = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * I + 0));
1525 auto *HighA = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 2 * I + 1));
1526 RangeListA.push_back(Elt: ConstantRange(LowA->getValue(), HighA->getValue()));
1527 }
1528
1529 for (unsigned I = 0, E = B->getNumOperands() / 2; I != E; ++I) {
1530 auto *LowB = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * I + 0));
1531 auto *HighB = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 2 * I + 1));
1532 RangeListB.push_back(Elt: ConstantRange(LowB->getValue(), HighB->getValue()));
1533 }
1534
1535 ConstantRangeList CRLA(RangeListA);
1536 ConstantRangeList CRLB(RangeListB);
1537 ConstantRangeList Result = CRLA.intersectWith(CRL: CRLB);
1538 if (Result.empty())
1539 return nullptr;
1540
1541 SmallVector<Metadata *> MDs;
1542 for (const ConstantRange &CR : Result) {
1543 MDs.push_back(Elt: ConstantAsMetadata::get(
1544 C: ConstantInt::get(Context&: A->getContext(), V: CR.getLower())));
1545 MDs.push_back(Elt: ConstantAsMetadata::get(
1546 C: ConstantInt::get(Context&: A->getContext(), V: CR.getUpper())));
1547 }
1548
1549 return MDNode::get(Context&: A->getContext(), MDs);
1550}
1551
1552MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) {
1553 if (!A || !B)
1554 return nullptr;
1555
1556 ConstantInt *AVal = mdconst::extract<ConstantInt>(MD: A->getOperand(I: 0));
1557 ConstantInt *BVal = mdconst::extract<ConstantInt>(MD: B->getOperand(I: 0));
1558 if (AVal->getZExtValue() < BVal->getZExtValue())
1559 return A;
1560 return B;
1561}
1562
1563CaptureComponents MDNode::toCaptureComponents(const MDNode *MD) {
1564 if (!MD)
1565 return CaptureComponents::All;
1566
1567 CaptureComponents CC = CaptureComponents::None;
1568 for (Metadata *Op : MD->operands()) {
1569 CaptureComponents Component =
1570 StringSwitch<CaptureComponents>(cast<MDString>(Val: Op)->getString())
1571 .Case(S: "address", Value: CaptureComponents::Address)
1572 .Case(S: "address_is_null", Value: CaptureComponents::AddressIsNull)
1573 .Case(S: "provenance", Value: CaptureComponents::Provenance)
1574 .Case(S: "read_provenance", Value: CaptureComponents::ReadProvenance);
1575 CC |= Component;
1576 }
1577 return CC;
1578}
1579
1580MDNode *MDNode::fromCaptureComponents(LLVMContext &Ctx, CaptureComponents CC) {
1581 assert(!capturesNothing(CC) && "Can't encode captures(none)");
1582 if (capturesAll(CC))
1583 return nullptr;
1584
1585 SmallVector<Metadata *> Components;
1586 if (capturesAddressIsNullOnly(CC))
1587 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "address_is_null"));
1588 else if (capturesAddress(CC))
1589 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "address"));
1590 if (capturesReadProvenanceOnly(CC))
1591 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "read_provenance"));
1592 else if (capturesFullProvenance(CC))
1593 Components.push_back(Elt: MDString::get(Context&: Ctx, Str: "provenance"));
1594 return MDNode::get(Context&: Ctx, MDs: Components);
1595}
1596
1597//===----------------------------------------------------------------------===//
1598// NamedMDNode implementation.
1599//
1600
1601static SmallVector<TrackingMDRef, 4> &getNMDOps(void *Operands) {
1602 return *(SmallVector<TrackingMDRef, 4> *)Operands;
1603}
1604
1605NamedMDNode::NamedMDNode(const Twine &N)
1606 : Name(N.str()), Operands(new SmallVector<TrackingMDRef, 4>()) {}
1607
1608NamedMDNode::~NamedMDNode() {
1609 dropAllReferences();
1610 delete &getNMDOps(Operands);
1611}
1612
1613unsigned NamedMDNode::getNumOperands() const {
1614 return (unsigned)getNMDOps(Operands).size();
1615}
1616
1617MDNode *NamedMDNode::getOperand(unsigned i) const {
1618 assert(i < getNumOperands() && "Invalid Operand number!");
1619 auto *N = getNMDOps(Operands)[i].get();
1620 return cast_or_null<MDNode>(Val: N);
1621}
1622
1623void NamedMDNode::addOperand(MDNode *M) { getNMDOps(Operands).emplace_back(Args&: M); }
1624
1625void NamedMDNode::setOperand(unsigned I, MDNode *New) {
1626 assert(I < getNumOperands() && "Invalid operand number");
1627 getNMDOps(Operands)[I].reset(MD: New);
1628}
1629
1630void NamedMDNode::eraseFromParent() { getParent()->eraseNamedMetadata(NMD: this); }
1631
1632void NamedMDNode::clearOperands() { getNMDOps(Operands).clear(); }
1633
1634StringRef NamedMDNode::getName() const { return StringRef(Name); }
1635
1636//===----------------------------------------------------------------------===//
1637// Instruction Metadata method implementations.
1638//
1639
1640unsigned &Value::getMetadataIndex() {
1641 if (auto *I = dyn_cast<Instruction>(Val: this))
1642 return I->MetadataIndex;
1643 return cast<GlobalObject>(Val: this)->MetadataIndex;
1644}
1645
1646unsigned Value::getMetadataIndex() const {
1647 return const_cast<Value *>(this)->getMetadataIndex();
1648}
1649
1650MDNode *Value::getMetadata(StringRef Kind) const {
1651 unsigned KindID = getContext().getMDKindID(Name: Kind);
1652 return getMetadataImpl(KindID);
1653}
1654
1655MDNode *Value::getMetadataImpl(unsigned KindID) const {
1656 const LLVMContext &Ctx = getContext();
1657 unsigned Idx = getMetadataIndex();
1658 while (Idx) {
1659 const MDAttachment &A = Ctx.pImpl->Metadatas[Idx];
1660 if (A.MDKind == KindID)
1661 return A.Node;
1662 Idx = A.Next;
1663 }
1664 return nullptr;
1665}
1666
1667void GlobalObject::getMetadata(unsigned KindID,
1668 SmallVectorImpl<MDNode *> &MDs) const {
1669 const LLVMContext &Ctx = getContext();
1670 unsigned Idx = MetadataIndex;
1671 while (Idx) {
1672 const MDAttachment &A = Ctx.pImpl->Metadatas[Idx];
1673 if (A.MDKind == KindID)
1674 MDs.push_back(Elt: A.Node);
1675 Idx = A.Next;
1676 }
1677 // We store metadata in reverse order, so reverse for output.
1678 std::reverse(first: MDs.begin(), last: MDs.end());
1679}
1680
1681void GlobalObject::getMetadata(StringRef Kind,
1682 SmallVectorImpl<MDNode *> &MDs) const {
1683 getMetadata(KindID: getContext().getMDKindID(Name: Kind), MDs);
1684}
1685
1686void Value::getAllMetadata(
1687 SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const {
1688 const LLVMContext &Ctx = getContext();
1689 unsigned Idx = getMetadataIndex();
1690 while (Idx) {
1691 const MDAttachment &A = Ctx.pImpl->Metadatas[Idx];
1692 MDs.emplace_back(Args: A.MDKind, Args: A.Node);
1693 Idx = A.Next;
1694 }
1695 // We store metadata in reverse order, so reverse for output in insertion
1696 // order. Sort by metadata ID for stable output.
1697 if (MDs.size() > 1) {
1698 std::reverse(first: MDs.begin(), last: MDs.end());
1699 llvm::stable_sort(Range&: MDs, C: less_first());
1700 }
1701}
1702
1703void Value::setMetadata(unsigned KindID, MDNode *Node) {
1704 assert(isa<Instruction>(this) || isa<GlobalObject>(this));
1705
1706 if (getMetadataIndex() != 0)
1707 eraseMetadata(KindID);
1708 if (Node)
1709 addMetadata(KindID, MD&: *Node);
1710}
1711
1712void Value::setMetadata(StringRef Kind, MDNode *Node) {
1713 if (!Node && getMetadataIndex() == 0)
1714 return;
1715 setMetadata(KindID: getContext().getMDKindID(Name: Kind), Node);
1716}
1717
1718void Value::addMetadata(unsigned KindID, MDNode &MD) {
1719 const LLVMContext &Ctx = getContext();
1720 unsigned &Idx = getMetadataIndex();
1721 unsigned NewIdx = Ctx.pImpl->MetadataRecycleHead;
1722 if (NewIdx == 0) {
1723 NewIdx = Ctx.pImpl->Metadatas.size();
1724 if (NewIdx == 0)
1725 NewIdx = 1;
1726 Ctx.pImpl->Metadatas.resize(N: NewIdx + 1);
1727 } else {
1728 Ctx.pImpl->MetadataRecycleHead = Ctx.pImpl->Metadatas[NewIdx].Next;
1729#ifndef NDEBUG
1730 Ctx.pImpl->MetadataRecycleSize -= 1;
1731#endif
1732 }
1733 Ctx.pImpl->Metadatas[NewIdx] =
1734 MDAttachment{.Next: Idx, .MDKind: KindID, .Node: TrackingMDNodeRef(&MD)};
1735 Idx = NewIdx;
1736}
1737
1738void Value::addMetadata(StringRef Kind, MDNode &MD) {
1739 addMetadata(KindID: getContext().getMDKindID(Name: Kind), MD);
1740}
1741
1742bool Value::eraseMetadata(unsigned KindID) {
1743 bool Changed = false;
1744 eraseMetadataIf(Pred: [&Changed, KindID](unsigned MDKind, MDNode *) {
1745 Changed |= MDKind == KindID;
1746 return MDKind == KindID;
1747 });
1748 return Changed;
1749}
1750
1751void Value::eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred) {
1752 unsigned *Idx = &getMetadataIndex();
1753 const LLVMContext &Ctx = getContext();
1754 while (*Idx) {
1755 MDAttachment &A = Ctx.pImpl->Metadatas[*Idx];
1756 if (Pred(A.MDKind, A.Node)) {
1757 A.Node.reset();
1758 unsigned FreeIdx = *Idx;
1759 *Idx = A.Next;
1760 A.Next = Ctx.pImpl->MetadataRecycleHead;
1761 Ctx.pImpl->MetadataRecycleHead = FreeIdx;
1762#ifndef NDEBUG
1763 Ctx.pImpl->MetadataRecycleSize += 1;
1764#endif
1765 } else {
1766 Idx = &A.Next;
1767 }
1768 }
1769}
1770
1771void Value::clearMetadata() {
1772 eraseMetadataIf(Pred: [](unsigned, MDNode *) { return true; });
1773}
1774
1775void Instruction::setMetadata(StringRef Kind, MDNode *Node) {
1776 if (!Node && MetadataIndex == 0)
1777 return;
1778 setMetadata(KindID: getContext().getMDKindID(Name: Kind), Node);
1779}
1780
1781MDNode *Instruction::getMetadataImpl(StringRef Kind) const {
1782 const LLVMContext &Ctx = getContext();
1783 unsigned KindID = Ctx.getMDKindID(Name: Kind);
1784 if (KindID == LLVMContext::MD_dbg)
1785 return DbgLoc.getAsMDNode();
1786 return Value::getMetadataImpl(KindID);
1787}
1788
1789void Instruction::eraseMetadataIf(function_ref<bool(unsigned, MDNode *)> Pred) {
1790 if (DbgLoc && Pred(LLVMContext::MD_dbg, DbgLoc.getAsMDNode()))
1791 DbgLoc = {};
1792
1793 Value::eraseMetadataIf(Pred);
1794}
1795
1796void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) {
1797 if (!hasMetadataOtherThanDebugLoc())
1798 return; // Nothing to remove!
1799
1800 SmallSet<unsigned, 32> KnownSet(llvm::from_range, KnownIDs);
1801
1802 // A DIAssignID attachment is debug metadata, don't drop it.
1803 KnownSet.insert(V: LLVMContext::MD_DIAssignID);
1804
1805 Value::eraseMetadataIf(Pred: [&KnownSet](unsigned MDKind, MDNode *Node) {
1806 return !KnownSet.count(V: MDKind);
1807 });
1808}
1809
1810void Instruction::updateDIAssignIDMapping(DIAssignID *ID) {
1811 if (auto *CurrentID =
1812 cast_or_null<DIAssignID>(Val: getMetadata(KindID: LLVMContext::MD_DIAssignID))) {
1813 if (ID == CurrentID)
1814 return;
1815 CurrentID->Instrs.erase(I: llvm::find(Range&: CurrentID->Instrs, Val: this));
1816 }
1817 if (ID)
1818 ID->Instrs.push_back(NewVal: this);
1819}
1820
1821void Instruction::setMetadata(unsigned KindID, MDNode *Node) {
1822 if (!Node && !hasMetadata())
1823 return;
1824
1825 // Handle 'dbg' as a special case since it is not stored in the hash table.
1826 if (KindID == LLVMContext::MD_dbg) {
1827 DbgLoc = DebugLoc(cast_or_null<DILocation>(Val: Node));
1828 return;
1829 }
1830
1831 // Update DIAssignID to Instruction(s) mapping.
1832 if (KindID == LLVMContext::MD_DIAssignID) {
1833 // The DIAssignID tracking infrastructure doesn't support RAUWing temporary
1834 // nodes with DIAssignIDs. The cast_or_null below would also catch this, but
1835 // having a dedicated assert helps make this obvious.
1836 assert((!Node || !Node->isTemporary()) &&
1837 "Temporary DIAssignIDs are invalid");
1838 updateDIAssignIDMapping(ID: cast_or_null<DIAssignID>(Val: Node));
1839 }
1840
1841 Value::setMetadata(KindID, Node);
1842}
1843
1844void Instruction::addAnnotationMetadata(SmallVector<StringRef> Annotations) {
1845 SmallVector<Metadata *, 4> Names;
1846 if (auto *Existing = getMetadata(KindID: LLVMContext::MD_annotation)) {
1847 SmallSetVector<StringRef, 2> AnnotationsSet(Annotations.begin(),
1848 Annotations.end());
1849 auto *Tuple = cast<MDTuple>(Val: Existing);
1850 for (auto &N : Tuple->operands()) {
1851 if (isa<MDString>(Val: N.get())) {
1852 Names.push_back(Elt: N);
1853 continue;
1854 }
1855 auto *MDAnnotationTuple = cast<MDTuple>(Val: N);
1856 if (any_of(Range: MDAnnotationTuple->operands(), P: [&AnnotationsSet](auto &Op) {
1857 return AnnotationsSet.contains(key: cast<MDString>(Op)->getString());
1858 }))
1859 return;
1860 Names.push_back(Elt: N);
1861 }
1862 }
1863
1864 MDBuilder MDB(getContext());
1865 SmallVector<Metadata *> MDAnnotationStrings;
1866 for (StringRef Annotation : Annotations)
1867 MDAnnotationStrings.push_back(Elt: MDB.createString(Str: Annotation));
1868 MDNode *InfoTuple = MDTuple::get(Context&: getContext(), MDs: MDAnnotationStrings);
1869 Names.push_back(Elt: InfoTuple);
1870 MDNode *MD = MDTuple::get(Context&: getContext(), MDs: Names);
1871 setMetadata(KindID: LLVMContext::MD_annotation, Node: MD);
1872}
1873
1874void Instruction::addAnnotationMetadata(StringRef Name) {
1875 SmallVector<Metadata *, 4> Names;
1876 if (auto *Existing = getMetadata(KindID: LLVMContext::MD_annotation)) {
1877 auto *Tuple = cast<MDTuple>(Val: Existing);
1878 for (auto &N : Tuple->operands()) {
1879 if (isa<MDString>(Val: N.get()) &&
1880 cast<MDString>(Val: N.get())->getString() == Name)
1881 return;
1882 Names.push_back(Elt: N.get());
1883 }
1884 }
1885
1886 MDBuilder MDB(getContext());
1887 Names.push_back(Elt: MDB.createString(Str: Name));
1888 MDNode *MD = MDTuple::get(Context&: getContext(), MDs: Names);
1889 setMetadata(KindID: LLVMContext::MD_annotation, Node: MD);
1890}
1891
1892AAMDNodes Instruction::getAAMetadata() const {
1893 AAMDNodes Result;
1894 if (hasMetadataOtherThanDebugLoc()) {
1895 unsigned Idx = MetadataIndex;
1896 const auto &Metadatas = getContext().pImpl->Metadatas;
1897 while (Idx) {
1898 const MDAttachment &A = Metadatas[Idx];
1899 switch (A.MDKind) {
1900 case LLVMContext::MD_tbaa:
1901 Result.TBAA = A.Node;
1902 break;
1903 case LLVMContext::MD_tbaa_struct:
1904 Result.TBAAStruct = A.Node;
1905 break;
1906 case LLVMContext::MD_alias_scope:
1907 Result.Scope = A.Node;
1908 break;
1909 case LLVMContext::MD_noalias:
1910 Result.NoAlias = A.Node;
1911 break;
1912 case LLVMContext::MD_noalias_addrspace:
1913 Result.NoAliasAddrSpace = A.Node;
1914 break;
1915 }
1916 Idx = A.Next;
1917 }
1918 }
1919 return Result;
1920}
1921
1922void Instruction::setAAMetadata(const AAMDNodes &N) {
1923 setMetadata(KindID: LLVMContext::MD_tbaa, Node: N.TBAA);
1924 setMetadata(KindID: LLVMContext::MD_tbaa_struct, Node: N.TBAAStruct);
1925 setMetadata(KindID: LLVMContext::MD_alias_scope, Node: N.Scope);
1926 setMetadata(KindID: LLVMContext::MD_noalias, Node: N.NoAlias);
1927 setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: N.NoAliasAddrSpace);
1928}
1929
1930void Instruction::setNoSanitizeMetadata() {
1931 setMetadata(KindID: llvm::LLVMContext::MD_nosanitize,
1932 Node: llvm::MDNode::get(Context&: getContext(), MDs: {}));
1933}
1934
1935void Instruction::getAllMetadataImpl(
1936 SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
1937 Result.clear();
1938
1939 // Handle 'dbg' as a special case since it is not stored in the hash table.
1940 if (DbgLoc) {
1941 Result.push_back(
1942 Elt: std::make_pair(x: (unsigned)LLVMContext::MD_dbg, y: DbgLoc.getAsMDNode()));
1943 }
1944 Value::getAllMetadata(MDs&: Result);
1945}
1946
1947bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) const {
1948 assert((getOpcode() == Instruction::CondBr ||
1949 getOpcode() == Instruction::Select ||
1950 getOpcode() == Instruction::Call ||
1951 getOpcode() == Instruction::Invoke ||
1952 getOpcode() == Instruction::IndirectBr ||
1953 getOpcode() == Instruction::Switch) &&
1954 "Looking for branch weights on something besides branch");
1955
1956 return ::extractProfTotalWeight(I: *this, TotalWeights&: TotalVal);
1957}
1958
1959void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) {
1960 SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
1961 Other->getAllMetadata(MDs);
1962 for (auto &MD : MDs) {
1963 // We need to adjust the type metadata offset.
1964 if (Offset != 0 && MD.first == LLVMContext::MD_type) {
1965 auto *OffsetConst = cast<ConstantInt>(
1966 Val: cast<ConstantAsMetadata>(Val: MD.second->getOperand(I: 0))->getValue());
1967 Metadata *TypeId = MD.second->getOperand(I: 1);
1968 auto *NewOffsetMD = ConstantAsMetadata::get(C: ConstantInt::get(
1969 Ty: OffsetConst->getType(), V: OffsetConst->getValue() + Offset));
1970 addMetadata(KindID: LLVMContext::MD_type,
1971 MD&: *MDNode::get(Context&: getContext(), MDs: {NewOffsetMD, TypeId}));
1972 continue;
1973 }
1974 // If an offset adjustment was specified we need to modify the DIExpression
1975 // to prepend the adjustment:
1976 // !DIExpression(DW_OP_plus, Offset, [original expr])
1977 auto *Attachment = MD.second;
1978 if (Offset != 0 && MD.first == LLVMContext::MD_dbg) {
1979 DIGlobalVariable *GV = dyn_cast<DIGlobalVariable>(Val: Attachment);
1980 DIExpression *E = nullptr;
1981 if (!GV) {
1982 auto *GVE = cast<DIGlobalVariableExpression>(Val: Attachment);
1983 GV = GVE->getVariable();
1984 E = GVE->getExpression();
1985 }
1986 ArrayRef<uint64_t> OrigElements;
1987 if (E)
1988 OrigElements = E->getElements();
1989 std::vector<uint64_t> Elements(OrigElements.size() + 2);
1990 Elements[0] = dwarf::DW_OP_plus_uconst;
1991 Elements[1] = Offset;
1992 llvm::copy(Range&: OrigElements, Out: Elements.begin() + 2);
1993 E = DIExpression::get(Context&: getContext(), Elements);
1994 Attachment = DIGlobalVariableExpression::get(Context&: getContext(), Variable: GV, Expression: E);
1995 }
1996 addMetadata(KindID: MD.first, MD&: *Attachment);
1997 }
1998}
1999
2000void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) {
2001 addMetadata(
2002 KindID: LLVMContext::MD_type,
2003 MD&: *MDTuple::get(Context&: getContext(),
2004 MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
2005 Ty: Type::getInt64Ty(C&: getContext()), V: Offset)),
2006 TypeID}));
2007}
2008
2009void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) {
2010 // Remove any existing vcall visibility metadata first in case we are
2011 // updating.
2012 eraseMetadata(KindID: LLVMContext::MD_vcall_visibility);
2013 addMetadata(KindID: LLVMContext::MD_vcall_visibility,
2014 MD&: *MDNode::get(Context&: getContext(),
2015 MDs: {ConstantAsMetadata::get(C: ConstantInt::get(
2016 Ty: Type::getInt64Ty(C&: getContext()), V: Visibility))}));
2017}
2018
2019GlobalObject::VCallVisibility GlobalObject::getVCallVisibility() const {
2020 if (MDNode *MD = getMetadata(KindID: LLVMContext::MD_vcall_visibility)) {
2021 uint64_t Val = cast<ConstantInt>(
2022 Val: cast<ConstantAsMetadata>(Val: MD->getOperand(I: 0))->getValue())
2023 ->getZExtValue();
2024 assert(Val <= 2 && "unknown vcall visibility!");
2025 return (VCallVisibility)Val;
2026 }
2027 return VCallVisibility::VCallVisibilityPublic;
2028}
2029
2030void Function::setSubprogram(DISubprogram *SP) {
2031 setMetadata(KindID: LLVMContext::MD_dbg, Node: SP);
2032}
2033
2034DISubprogram *Function::getSubprogram() const {
2035 return cast_or_null<DISubprogram>(Val: getMetadata(KindID: LLVMContext::MD_dbg));
2036}
2037
2038bool Function::shouldEmitDebugInfoForProfiling() const {
2039 if (DISubprogram *SP = getSubprogram()) {
2040 if (DICompileUnit *CU = SP->getUnit()) {
2041 return CU->getDebugInfoForProfiling();
2042 }
2043 }
2044 return false;
2045}
2046
2047void GlobalVariable::addDebugInfo(DIGlobalVariableExpression *GV) {
2048 addMetadata(KindID: LLVMContext::MD_dbg, MD&: *GV);
2049}
2050
2051void GlobalVariable::getDebugInfo(
2052 SmallVectorImpl<DIGlobalVariableExpression *> &GVs) const {
2053 SmallVector<MDNode *, 1> MDs;
2054 getMetadata(KindID: LLVMContext::MD_dbg, MDs);
2055 for (MDNode *MD : MDs)
2056 GVs.push_back(Elt: cast<DIGlobalVariableExpression>(Val: MD));
2057}
2058