1//===-- memtag.h ------------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef SCUDO_MEMTAG_H_
10#define SCUDO_MEMTAG_H_
11
12#include "internal_defs.h"
13
14#if SCUDO_CAN_USE_MTE
15#include <sys/auxv.h>
16#include <sys/prctl.h>
17#endif
18
19namespace scudo {
20
21#if (__clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)) || \
22 defined(SCUDO_FUZZ)
23
24// We assume that Top-Byte Ignore is enabled if the architecture supports memory
25// tagging. Not all operating systems enable TBI, so we only claim architectural
26// support for memory tagging if the operating system enables TBI.
27// HWASan uses the top byte for its own purpose and Scudo should not touch it.
28// ASan also does not support TBI when computing shadow addresses.
29#if SCUDO_CAN_USE_MTE && !defined(SCUDO_DISABLE_TBI) && \
30 !__has_feature(hwaddress_sanitizer) && !__has_feature(address_sanitizer)
31inline constexpr bool archSupportsMemoryTagging() { return true; }
32#else
33inline constexpr bool archSupportsMemoryTagging() { return false; }
34#endif
35
36inline constexpr uptr archMemoryTagGranuleSize() { return 16; }
37
38inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); }
39
40inline uint8_t extractTag(uptr Ptr) { return (Ptr >> 56) & 0xf; }
41
42#else
43
44inline constexpr bool archSupportsMemoryTagging() { return false; }
45
46inline NORETURN uptr archMemoryTagGranuleSize() {
47 UNREACHABLE("memory tagging not supported");
48}
49
50inline NORETURN uptr untagPointer(uptr Ptr) {
51 (void)Ptr;
52 UNREACHABLE("memory tagging not supported");
53}
54
55inline NORETURN uint8_t extractTag(uptr Ptr) {
56 (void)Ptr;
57 UNREACHABLE("memory tagging not supported");
58}
59
60#endif
61
62#if __clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)
63
64#if SCUDO_CAN_USE_MTE
65
66inline bool systemSupportsMemoryTagging() {
67#ifndef HWCAP2_MTE
68#define HWCAP2_MTE (1 << 18)
69#endif
70 return getauxval(AT_HWCAP2) & HWCAP2_MTE;
71}
72
73inline bool systemDetectsMemoryTagFaultsTestOnly() {
74#ifndef PR_SET_TAGGED_ADDR_CTRL
75#define PR_SET_TAGGED_ADDR_CTRL 54
76#endif
77#ifndef PR_GET_TAGGED_ADDR_CTRL
78#define PR_GET_TAGGED_ADDR_CTRL 56
79#endif
80#ifndef PR_TAGGED_ADDR_ENABLE
81#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
82#endif
83#ifndef PR_MTE_TCF_SHIFT
84#define PR_MTE_TCF_SHIFT 1
85#endif
86#ifndef PR_MTE_TAG_SHIFT
87#define PR_MTE_TAG_SHIFT 3
88#endif
89#ifndef PR_MTE_TCF_NONE
90#define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
91#endif
92#ifndef PR_MTE_TCF_SYNC
93#define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
94#endif
95#ifndef PR_MTE_TCF_MASK
96#define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
97#endif
98 int res = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
99 if (res == -1)
100 return false;
101 return (static_cast<unsigned long>(res) & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE;
102}
103
104inline void enableSystemMemoryTaggingTestOnly() {
105 prctl(PR_SET_TAGGED_ADDR_CTRL,
106 PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT),
107 0, 0, 0);
108}
109
110#else // !SCUDO_CAN_USE_MTE
111
112inline constexpr bool systemSupportsMemoryTagging() { return false; }
113
114inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
115 UNREACHABLE("memory tagging not supported");
116}
117
118inline NORETURN void enableSystemMemoryTaggingTestOnly() {
119 UNREACHABLE("memory tagging not supported");
120}
121
122#endif // SCUDO_CAN_USE_MTE
123
124class ScopedDisableMemoryTagChecks {
125 uptr PrevTCO;
126 bool active;
127
128public:
129 ScopedDisableMemoryTagChecks(bool cond = true) : active(cond) {
130 if (!active)
131 return;
132 __asm__ __volatile__(
133 R"(
134 .arch_extension memtag
135 mrs %0, tco
136 msr tco, #1
137 )"
138 : "=r"(PrevTCO));
139 }
140
141 ~ScopedDisableMemoryTagChecks() {
142 if (!active)
143 return;
144 __asm__ __volatile__(
145 R"(
146 .arch_extension memtag
147 msr tco, %0
148 )"
149 :
150 : "r"(PrevTCO));
151 }
152};
153
154inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
155 ExcludeMask |= 1; // Always exclude Tag 0.
156 uptr TaggedPtr;
157 __asm__ __volatile__(
158 R"(
159 .arch_extension memtag
160 irg %[TaggedPtr], %[Ptr], %[ExcludeMask]
161 )"
162 : [TaggedPtr] "=r"(TaggedPtr)
163 : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask));
164 return TaggedPtr;
165}
166
167inline uptr addFixedTag(uptr Ptr, uptr Tag) {
168 DCHECK_LT(Tag, 16);
169 DCHECK_EQ(untagPointer(Ptr), Ptr);
170 return Ptr | (Tag << 56);
171}
172
173inline uptr storeTags(uptr Begin, uptr End) {
174 DCHECK_EQ(0, Begin % 16);
175 uptr LineSize, Next, Tmp;
176 __asm__ __volatile__(
177 R"(
178 .arch_extension memtag
179
180 // Compute the cache line size in bytes (DCZID_EL0 stores it as the log2
181 // of the number of 4-byte words) and bail out to the slow path if DCZID_EL0
182 // indicates that the DC instructions are unavailable.
183 DCZID .req %[Tmp]
184 mrs DCZID, dczid_el0
185 tbnz DCZID, #4, 3f
186 and DCZID, DCZID, #15
187 mov %[LineSize], #4
188 lsl %[LineSize], %[LineSize], DCZID
189 .unreq DCZID
190
191 // Our main loop doesn't handle the case where we don't need to perform any
192 // DC GZVA operations. If the size of our tagged region is less than
193 // twice the cache line size, bail out to the slow path since it's not
194 // guaranteed that we'll be able to do a DC GZVA.
195 Size .req %[Tmp]
196 sub Size, %[End], %[Cur]
197 cmp Size, %[LineSize], lsl #1
198 b.lt 3f
199 .unreq Size
200
201 LineMask .req %[Tmp]
202 sub LineMask, %[LineSize], #1
203
204 // STZG until the start of the next cache line.
205 orr %[Next], %[Cur], LineMask
206 1:
207 stzg %[Cur], [%[Cur]], #16
208 cmp %[Cur], %[Next]
209 b.lt 1b
210
211 // DC GZVA cache lines until we have no more full cache lines.
212 bic %[Next], %[End], LineMask
213 .unreq LineMask
214 2:
215 dc gzva, %[Cur]
216 add %[Cur], %[Cur], %[LineSize]
217 cmp %[Cur], %[Next]
218 b.lt 2b
219
220 // STZG until the end of the tagged region. This loop is also used to handle
221 // slow path cases.
222 3:
223 cmp %[Cur], %[End]
224 b.ge 4f
225 stzg %[Cur], [%[Cur]], #16
226 b 3b
227
228 4:
229 )"
230 : [Cur] "+&r"(Begin), [LineSize] "=&r"(LineSize), [Next] "=&r"(Next),
231 [Tmp] "=&r"(Tmp)
232 : [End] "r"(End)
233 : "memory");
234 DCHECK_EQ(0, Begin % 16);
235 return Begin;
236}
237
238inline void storeTag(uptr Ptr) {
239 DCHECK_EQ(0, Ptr % 16);
240 __asm__ __volatile__(R"(
241 .arch_extension memtag
242 stg %0, [%0]
243 )"
244 :
245 : "r"(Ptr)
246 : "memory");
247}
248
249inline uptr loadTag(uptr Ptr) {
250 DCHECK_EQ(0, Ptr % 16);
251 uptr TaggedPtr = Ptr;
252 __asm__ __volatile__(
253 R"(
254 .arch_extension memtag
255 ldg %0, [%0]
256 )"
257 : "+r"(TaggedPtr)
258 :
259 : "memory");
260 return TaggedPtr;
261}
262
263inline uptr loadTagUnaligned(uptr Ptr) {
264 uptr AlignedPtr = Ptr & ~static_cast<uptr>(0xF);
265 return loadTag(AlignedPtr) | (Ptr & 0xF);
266}
267
268#else
269
270inline constexpr bool systemSupportsMemoryTagging() { return false; }
271
272inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
273 UNREACHABLE("memory tagging not supported");
274}
275
276inline NORETURN void enableSystemMemoryTaggingTestOnly() {
277 UNREACHABLE("memory tagging not supported");
278}
279
280struct ScopedDisableMemoryTagChecks {
281 ScopedDisableMemoryTagChecks(UNUSED bool cond = true) {}
282};
283
284inline NORETURN uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
285 (void)Ptr;
286 (void)ExcludeMask;
287 UNREACHABLE("memory tagging not supported");
288}
289
290inline NORETURN uptr addFixedTag(uptr Ptr, uptr Tag) {
291 (void)Ptr;
292 (void)Tag;
293 UNREACHABLE("memory tagging not supported");
294}
295
296inline NORETURN uptr storeTags(uptr Begin, uptr End) {
297 (void)Begin;
298 (void)End;
299 UNREACHABLE("memory tagging not supported");
300}
301
302inline NORETURN void storeTag(uptr Ptr) {
303 (void)Ptr;
304 UNREACHABLE("memory tagging not supported");
305}
306
307inline NORETURN uptr loadTag(uptr Ptr) {
308 (void)Ptr;
309 UNREACHABLE("memory tagging not supported");
310}
311
312inline uptr loadTagUnaligned(uptr Ptr) {
313 (void)Ptr;
314 UNREACHABLE("memory tagging not supported");
315}
316
317#endif
318
319#pragma GCC diagnostic push
320#pragma GCC diagnostic ignored "-Wmissing-noreturn"
321inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
322 uptr *TaggedBegin, uptr *TaggedEnd) {
323 *TaggedBegin = selectRandomTag(Ptr: reinterpret_cast<uptr>(Ptr), ExcludeMask);
324 *TaggedEnd = storeTags(Begin: *TaggedBegin, End: *TaggedBegin + Size);
325}
326#pragma GCC diagnostic pop
327
328inline void *untagPointer(void *Ptr) {
329 return reinterpret_cast<void *>(untagPointer(Ptr: reinterpret_cast<uptr>(Ptr)));
330}
331
332inline void *loadTag(void *Ptr) {
333 return reinterpret_cast<void *>(loadTag(Ptr: reinterpret_cast<uptr>(Ptr)));
334}
335
336inline void *addFixedTag(void *Ptr, uptr Tag) {
337 return reinterpret_cast<void *>(
338 addFixedTag(Ptr: reinterpret_cast<uptr>(Ptr), Tag));
339}
340
341template <typename Config>
342inline constexpr bool allocatorSupportsMemoryTagging() {
343 return archSupportsMemoryTagging() && Config::getMaySupportMemoryTagging() &&
344 (1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize();
345}
346
347} // namespace scudo
348
349#endif
350