1//===- MemCacheHint.cpp - Cache hint metadata helpers --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/IR/MemCacheHint.h"
10#include "llvm/ADT/STLExtras.h"
11#include "llvm/ADT/SmallVector.h"
12#include "llvm/IR/Constants.h"
13#include "llvm/IR/Instructions.h"
14#include "llvm/IR/IntrinsicInst.h"
15#include "llvm/IR/LLVMContext.h"
16#include "llvm/IR/Metadata.h"
17#include <optional>
18#include <utility>
19
20using namespace llvm;
21
22MDNode *llvm::getMemCacheHint(const Instruction &I, unsigned OperandNo) {
23 MDNode *MD = I.getMetadata(KindID: LLVMContext::MD_mem_cache_hint);
24 if (!MD)
25 return nullptr;
26
27 for (unsigned Idx = 0; Idx + 1 < MD->getNumOperands(); Idx += 2)
28 if (mdconst::extract<ConstantInt>(MD: MD->getOperand(I: Idx))->getZExtValue() ==
29 OperandNo)
30 return cast<MDNode>(Val: MD->getOperand(I: Idx + 1));
31 return nullptr;
32}
33
34void llvm::setMemCacheHint(Instruction &I, unsigned OperandNo, MDNode *Hint) {
35 SmallVector<std::pair<unsigned, MDNode *>, 2> Entries;
36 if (MDNode *MD = I.getMetadata(KindID: LLVMContext::MD_mem_cache_hint))
37 for (unsigned Idx = 0; Idx + 1 < MD->getNumOperands(); Idx += 2)
38 Entries.emplace_back(
39 Args: mdconst::extract<ConstantInt>(MD: MD->getOperand(I: Idx))->getZExtValue(),
40 Args: cast<MDNode>(Val: MD->getOperand(I: Idx + 1)));
41
42 auto It = llvm::lower_bound(Range&: Entries, Value&: OperandNo,
43 C: [](const auto &Entry, unsigned OperandNo) {
44 return Entry.first < OperandNo;
45 });
46 if (It != Entries.end() && It->first == OperandNo) {
47 if (Hint)
48 It->second = Hint;
49 else
50 Entries.erase(CI: It);
51 } else if (Hint) {
52 Entries.insert(I: It, Elt: {OperandNo, Hint});
53 } else {
54 return;
55 }
56
57 SmallVector<Metadata *, 4> Operands;
58 for (const auto &[Index, Payload] : Entries) {
59 Operands.push_back(Elt: ConstantAsMetadata::get(
60 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: I.getContext()), V: Index)));
61 Operands.push_back(Elt: Payload);
62 }
63 I.setMetadata(KindID: LLVMContext::MD_mem_cache_hint,
64 Node: Operands.empty() ? nullptr
65 : MDNode::get(Context&: I.getContext(), MDs: Operands));
66}
67
68// The supported accesses have one memory operand. Its position also
69// distinguishes reads (0) from writes (1).
70static std::optional<unsigned> getPointerOperandIndex(const Instruction &I) {
71 if (isa<LoadInst>(Val: I))
72 return LoadInst::getPointerOperandIndex();
73 if (isa<StoreInst>(Val: I))
74 return StoreInst::getPointerOperandIndex();
75 const auto *II = dyn_cast<IntrinsicInst>(Val: &I);
76 if (!II)
77 return std::nullopt;
78 switch (II->getIntrinsicID()) {
79 case Intrinsic::masked_load:
80 case Intrinsic::masked_gather:
81 return 0;
82 case Intrinsic::masked_store:
83 case Intrinsic::masked_scatter:
84 return 1;
85 default:
86 return std::nullopt;
87 }
88}
89
90static MDNode *getCommonCacheHint(const Instruction &Dest,
91 ArrayRef<Value *> Sources) {
92 std::optional<unsigned> PointerIndex = getPointerOperandIndex(I: Dest);
93 if (!PointerIndex || Sources.empty())
94 return nullptr;
95
96 const auto *First = dyn_cast<Instruction>(Val: Sources.front());
97 if (!First)
98 return nullptr;
99 MDNode *MD = First->getMetadata(KindID: LLVMContext::MD_mem_cache_hint);
100 if (!MD)
101 return nullptr;
102
103 for (Value *V : Sources) {
104 const auto *Source = dyn_cast<Instruction>(Val: V);
105 if (!Source || getPointerOperandIndex(I: *Source) != PointerIndex ||
106 Source->getMetadata(KindID: LLVMContext::MD_mem_cache_hint) != MD)
107 return nullptr;
108 }
109 return MD;
110}
111
112void llvm::propagateMemCacheHint(Instruction &Dest, ArrayRef<Value *> Sources) {
113 Dest.setMetadata(KindID: LLVMContext::MD_mem_cache_hint,
114 Node: getCommonCacheHint(Dest, Sources));
115}
116