| 1 | //===- MemCacheHint.cpp - Cache hint metadata helpers --------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "llvm/IR/MemCacheHint.h" |
| 10 | #include "llvm/ADT/STLExtras.h" |
| 11 | #include "llvm/ADT/SmallVector.h" |
| 12 | #include "llvm/IR/Constants.h" |
| 13 | #include "llvm/IR/Instructions.h" |
| 14 | #include "llvm/IR/IntrinsicInst.h" |
| 15 | #include "llvm/IR/LLVMContext.h" |
| 16 | #include "llvm/IR/Metadata.h" |
| 17 | #include <optional> |
| 18 | #include <utility> |
| 19 | |
| 20 | using namespace llvm; |
| 21 | |
| 22 | MDNode *llvm::getMemCacheHint(const Instruction &I, unsigned OperandNo) { |
| 23 | MDNode *MD = I.getMetadata(KindID: LLVMContext::MD_mem_cache_hint); |
| 24 | if (!MD) |
| 25 | return nullptr; |
| 26 | |
| 27 | for (unsigned Idx = 0; Idx + 1 < MD->getNumOperands(); Idx += 2) |
| 28 | if (mdconst::extract<ConstantInt>(MD: MD->getOperand(I: Idx))->getZExtValue() == |
| 29 | OperandNo) |
| 30 | return cast<MDNode>(Val: MD->getOperand(I: Idx + 1)); |
| 31 | return nullptr; |
| 32 | } |
| 33 | |
| 34 | void llvm::setMemCacheHint(Instruction &I, unsigned OperandNo, MDNode *Hint) { |
| 35 | SmallVector<std::pair<unsigned, MDNode *>, 2> Entries; |
| 36 | if (MDNode *MD = I.getMetadata(KindID: LLVMContext::MD_mem_cache_hint)) |
| 37 | for (unsigned Idx = 0; Idx + 1 < MD->getNumOperands(); Idx += 2) |
| 38 | Entries.emplace_back( |
| 39 | Args: mdconst::extract<ConstantInt>(MD: MD->getOperand(I: Idx))->getZExtValue(), |
| 40 | Args: cast<MDNode>(Val: MD->getOperand(I: Idx + 1))); |
| 41 | |
| 42 | auto It = llvm::lower_bound(Range&: Entries, Value&: OperandNo, |
| 43 | C: [](const auto &Entry, unsigned OperandNo) { |
| 44 | return Entry.first < OperandNo; |
| 45 | }); |
| 46 | if (It != Entries.end() && It->first == OperandNo) { |
| 47 | if (Hint) |
| 48 | It->second = Hint; |
| 49 | else |
| 50 | Entries.erase(CI: It); |
| 51 | } else if (Hint) { |
| 52 | Entries.insert(I: It, Elt: {OperandNo, Hint}); |
| 53 | } else { |
| 54 | return; |
| 55 | } |
| 56 | |
| 57 | SmallVector<Metadata *, 4> Operands; |
| 58 | for (const auto &[Index, Payload] : Entries) { |
| 59 | Operands.push_back(Elt: ConstantAsMetadata::get( |
| 60 | C: ConstantInt::get(Ty: Type::getInt32Ty(C&: I.getContext()), V: Index))); |
| 61 | Operands.push_back(Elt: Payload); |
| 62 | } |
| 63 | I.setMetadata(KindID: LLVMContext::MD_mem_cache_hint, |
| 64 | Node: Operands.empty() ? nullptr |
| 65 | : MDNode::get(Context&: I.getContext(), MDs: Operands)); |
| 66 | } |
| 67 | |
| 68 | // The supported accesses have one memory operand. Its position also |
| 69 | // distinguishes reads (0) from writes (1). |
| 70 | static std::optional<unsigned> getPointerOperandIndex(const Instruction &I) { |
| 71 | if (isa<LoadInst>(Val: I)) |
| 72 | return LoadInst::getPointerOperandIndex(); |
| 73 | if (isa<StoreInst>(Val: I)) |
| 74 | return StoreInst::getPointerOperandIndex(); |
| 75 | const auto *II = dyn_cast<IntrinsicInst>(Val: &I); |
| 76 | if (!II) |
| 77 | return std::nullopt; |
| 78 | switch (II->getIntrinsicID()) { |
| 79 | case Intrinsic::masked_load: |
| 80 | case Intrinsic::masked_gather: |
| 81 | return 0; |
| 82 | case Intrinsic::masked_store: |
| 83 | case Intrinsic::masked_scatter: |
| 84 | return 1; |
| 85 | default: |
| 86 | return std::nullopt; |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | static MDNode *getCommonCacheHint(const Instruction &Dest, |
| 91 | ArrayRef<Value *> Sources) { |
| 92 | std::optional<unsigned> PointerIndex = getPointerOperandIndex(I: Dest); |
| 93 | if (!PointerIndex || Sources.empty()) |
| 94 | return nullptr; |
| 95 | |
| 96 | const auto *First = dyn_cast<Instruction>(Val: Sources.front()); |
| 97 | if (!First) |
| 98 | return nullptr; |
| 99 | MDNode *MD = First->getMetadata(KindID: LLVMContext::MD_mem_cache_hint); |
| 100 | if (!MD) |
| 101 | return nullptr; |
| 102 | |
| 103 | for (Value *V : Sources) { |
| 104 | const auto *Source = dyn_cast<Instruction>(Val: V); |
| 105 | if (!Source || getPointerOperandIndex(I: *Source) != PointerIndex || |
| 106 | Source->getMetadata(KindID: LLVMContext::MD_mem_cache_hint) != MD) |
| 107 | return nullptr; |
| 108 | } |
| 109 | return MD; |
| 110 | } |
| 111 | |
| 112 | void llvm::propagateMemCacheHint(Instruction &Dest, ArrayRef<Value *> Sources) { |
| 113 | Dest.setMetadata(KindID: LLVMContext::MD_mem_cache_hint, |
| 114 | Node: getCommonCacheHint(Dest, Sources)); |
| 115 | } |
| 116 | |