1//===-------- cfi.cpp -----------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the runtime support for the cross-DSO CFI.
10//
11//===----------------------------------------------------------------------===//
12
13#include <assert.h>
14#include <elf.h>
15
16#include "sanitizer_common/sanitizer_common.h"
17#if SANITIZER_FREEBSD
18#include <sys/link_elf.h>
19#endif
20#include <link.h>
21#include <string.h>
22#include <stdlib.h>
23#include <sys/mman.h>
24
25#if SANITIZER_LINUX
26typedef ElfW(Phdr) Elf_Phdr;
27typedef ElfW(Ehdr) Elf_Ehdr;
28typedef ElfW(Addr) Elf_Addr;
29typedef ElfW(Sym) Elf_Sym;
30typedef ElfW(Dyn) Elf_Dyn;
31#elif SANITIZER_FREEBSD
32#if SANITIZER_WORDSIZE == 64
33#define ElfW64_Dyn Elf_Dyn
34#define ElfW64_Sym Elf_Sym
35#else
36#define ElfW32_Dyn Elf_Dyn
37#define ElfW32_Sym Elf_Sym
38#endif
39#endif
40
41#include "interception/interception.h"
42#include "sanitizer_common/sanitizer_flag_parser.h"
43#include "ubsan/ubsan_init.h"
44#include "ubsan/ubsan_flags.h"
45
46#ifdef CFI_ENABLE_DIAG
47#include "ubsan/ubsan_handlers.h"
48#endif
49
50using namespace __sanitizer;
51
52namespace __cfi {
53
54#if SANITIZER_LOONGARCH64
55#define kCfiShadowLimitsStorageSize 16384 // 16KiB on loongarch64 per page
56#else
57#define kCfiShadowLimitsStorageSize 4096 // 1 page
58#endif
59// Lets hope that the data segment is mapped with 4K pages.
60// The pointer to the cfi shadow region is stored at the start of this page.
61// The rest of the page is unused and re-mapped read-only.
62static union {
63 char space[kCfiShadowLimitsStorageSize];
64 struct {
65 uptr start;
66 uptr size;
67 } limits;
68} cfi_shadow_limits_storage
69 __attribute__((aligned(kCfiShadowLimitsStorageSize)));
70static constexpr uptr kShadowGranularity = 12;
71static constexpr uptr kShadowAlign = 1UL << kShadowGranularity; // 4096
72
73static constexpr uint16_t kInvalidShadow = 0;
74static constexpr uint16_t kUncheckedShadow = 0xFFFFU;
75
76// Get the start address of the CFI shadow region.
77uptr GetShadow() {
78 return cfi_shadow_limits_storage.limits.start;
79}
80
81uptr GetShadowSize() {
82 return cfi_shadow_limits_storage.limits.size;
83}
84
85// This will only work while the shadow is not allocated.
86void SetShadowSize(uptr size) {
87 cfi_shadow_limits_storage.limits.size = size;
88}
89
90uptr MemToShadowOffset(uptr x) {
91 return (x >> kShadowGranularity) << 1;
92}
93
94uint16_t *MemToShadow(uptr x, uptr shadow_base) {
95 return (uint16_t *)(shadow_base + MemToShadowOffset(x));
96}
97
98typedef int (*CFICheckFn)(u64, void *, void *);
99
100// This class reads and decodes the shadow contents.
101class ShadowValue {
102 uptr addr;
103 uint16_t v;
104 explicit ShadowValue(uptr addr, uint16_t v) : addr(addr), v(v) {}
105
106public:
107 bool is_invalid() const { return v == kInvalidShadow; }
108
109 bool is_unchecked() const { return v == kUncheckedShadow; }
110
111 CFICheckFn get_cfi_check() const {
112 assert(!is_invalid() && !is_unchecked());
113 uptr aligned_addr = addr & ~(kShadowAlign - 1);
114 uptr p = aligned_addr - (((uptr)v - 1) << kShadowGranularity);
115 return reinterpret_cast<CFICheckFn>(p);
116 }
117
118 // Load a shadow value for the given application memory address.
119 static const ShadowValue load(uptr addr) {
120 uptr shadow_base = GetShadow();
121 uptr shadow_offset = MemToShadowOffset(x: addr);
122 if (shadow_offset > GetShadowSize())
123 return ShadowValue(addr, kInvalidShadow);
124 else
125 return ShadowValue(
126 addr, *reinterpret_cast<uint16_t *>(shadow_base + shadow_offset));
127 }
128};
129
130class ShadowBuilder {
131 uptr shadow_;
132
133public:
134 // Allocate a new empty shadow (for the entire address space) on the side.
135 void Start();
136 // Mark the given address range as unchecked.
137 // This is used for uninstrumented libraries like libc.
138 // Any CFI check with a target in that range will pass.
139 void AddUnchecked(uptr begin, uptr end);
140 // Mark the given address range as belonging to a library with the given
141 // cfi_check function.
142 void Add(uptr begin, uptr end, uptr cfi_check);
143 // Finish shadow construction. Atomically switch the current active shadow
144 // region with the newly constructed one and deallocate the former.
145 void Install();
146};
147
148void ShadowBuilder::Start() {
149 shadow_ = (uptr)MmapNoReserveOrDie(size: GetShadowSize(), mem_type: "CFI shadow");
150 VReport(1, "CFI: shadow at %zx .. %zx\n", shadow_, shadow_ + GetShadowSize());
151}
152
153void ShadowBuilder::AddUnchecked(uptr begin, uptr end) {
154 uint16_t *shadow_begin = MemToShadow(x: begin, shadow_base: shadow_);
155 uint16_t *shadow_end = MemToShadow(x: end - 1, shadow_base: shadow_) + 1;
156 // memset takes a byte, so our unchecked shadow value requires both bytes to
157 // be the same. Make sure we're ok during compilation.
158 static_assert((kUncheckedShadow & 0xff) == ((kUncheckedShadow >> 8) & 0xff),
159 "Both bytes of the 16-bit value must be the same!");
160 memset(s: shadow_begin, c: kUncheckedShadow & 0xff,
161 n: (shadow_end - shadow_begin) * sizeof(*shadow_begin));
162}
163
164void ShadowBuilder::Add(uptr begin, uptr end, uptr cfi_check) {
165 assert((cfi_check & (kShadowAlign - 1)) == 0);
166
167 // Don't fill anything below cfi_check. We can not represent those addresses
168 // in the shadow, and must make sure at codegen to place all valid call
169 // targets above cfi_check.
170 begin = Max(a: begin, b: cfi_check);
171 uint16_t *s = MemToShadow(x: begin, shadow_base: shadow_);
172 uint16_t *s_end = MemToShadow(x: end - 1, shadow_base: shadow_) + 1;
173 uint16_t sv = ((begin - cfi_check) >> kShadowGranularity) + 1;
174 for (; s < s_end; s++, sv++)
175 *s = sv;
176}
177
178#if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
179void ShadowBuilder::Install() {
180 MprotectReadOnly(addr: shadow_, size: GetShadowSize());
181 uptr main_shadow = GetShadow();
182 if (main_shadow) {
183 // Update.
184#if SANITIZER_LINUX
185 void *res = mremap(addr: (void *)shadow_, old_len: GetShadowSize(), new_len: GetShadowSize(),
186 MREMAP_MAYMOVE | MREMAP_FIXED, (void *)main_shadow);
187 CHECK(res != MAP_FAILED);
188#elif SANITIZER_NETBSD
189 void *res = mremap((void *)shadow_, GetShadowSize(), (void *)main_shadow,
190 GetShadowSize(), MAP_FIXED);
191 CHECK(res != MAP_FAILED);
192#else
193 void *res = MmapFixedOrDie(shadow_, GetShadowSize(), "cfi shadow");
194 CHECK(res != MAP_FAILED);
195 ::memcpy(&shadow_, &main_shadow, GetShadowSize());
196#endif
197 } else {
198 // Initial setup.
199 CHECK_EQ(kCfiShadowLimitsStorageSize, GetPageSizeCached());
200 CHECK_EQ(0, GetShadow());
201 cfi_shadow_limits_storage.limits.start = shadow_;
202 MprotectReadOnly(addr: (uptr)&cfi_shadow_limits_storage,
203 size: sizeof(cfi_shadow_limits_storage));
204 CHECK_EQ(shadow_, GetShadow());
205 }
206}
207#else
208#error not implemented
209#endif
210
211// This is a workaround for a glibc bug:
212// https://sourceware.org/bugzilla/show_bug.cgi?id=15199
213// Other platforms can, hopefully, just do
214// dlopen(RTLD_NOLOAD | RTLD_LAZY)
215// dlsym("__cfi_check").
216uptr find_cfi_check_in_dso(dl_phdr_info *info) {
217 const Elf_Dyn *dynamic = nullptr;
218 for (int i = 0; i < info->dlpi_phnum; ++i) {
219 if (info->dlpi_phdr[i].p_type == PT_DYNAMIC) {
220 dynamic =
221 (const Elf_Dyn *)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr);
222 break;
223 }
224 }
225 if (!dynamic) return 0;
226 uptr strtab = 0, symtab = 0, strsz = 0;
227 for (const Elf_Dyn *p = dynamic; p->d_tag != PT_NULL; ++p) {
228 if (p->d_tag == DT_SYMTAB)
229 symtab = p->d_un.d_ptr;
230 else if (p->d_tag == DT_STRTAB)
231 strtab = p->d_un.d_ptr;
232 else if (p->d_tag == DT_STRSZ)
233 strsz = p->d_un.d_ptr;
234 }
235
236 if (symtab > strtab) {
237 VReport(1, "Can not handle: symtab > strtab (%zx > %zx)\n", symtab, strtab);
238 return 0;
239 }
240
241 // Verify that strtab and symtab are inside of the same LOAD segment.
242 // This excludes VDSO, which has (very high) bogus strtab and symtab pointers.
243 int phdr_idx;
244 for (phdr_idx = 0; phdr_idx < info->dlpi_phnum; phdr_idx++) {
245 const Elf_Phdr *phdr = &info->dlpi_phdr[phdr_idx];
246 if (phdr->p_type == PT_LOAD) {
247 uptr beg = info->dlpi_addr + phdr->p_vaddr;
248 uptr end = beg + phdr->p_memsz;
249 if (strtab >= beg && strtab + strsz < end && symtab >= beg &&
250 symtab < end)
251 break;
252 }
253 }
254 if (phdr_idx == info->dlpi_phnum) {
255 // Nope, either different segments or just bogus pointers.
256 // Can not handle this.
257 VReport(1, "Can not handle: symtab %zx, strtab %zx\n", symtab, strtab);
258 return 0;
259 }
260
261 for (const Elf_Sym *p = (const Elf_Sym *)symtab; (Elf_Addr)p < strtab;
262 ++p) {
263 // There is no reliable way to find the end of the symbol table. In
264 // lld-produces files, there are other sections between symtab and strtab.
265 // Stop looking when the symbol name is not inside strtab.
266 if (p->st_name >= strsz) break;
267 char *name = (char*)(strtab + p->st_name);
268 if (strcmp(s1: name, s2: "__cfi_check") == 0) {
269 assert(p->st_info == ELF32_ST_INFO(STB_GLOBAL, STT_FUNC) ||
270 p->st_info == ELF32_ST_INFO(STB_WEAK, STT_FUNC));
271 uptr addr = info->dlpi_addr + p->st_value;
272 return addr;
273 }
274 }
275 return 0;
276}
277
278int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *data) {
279 uptr cfi_check = find_cfi_check_in_dso(info);
280 if (cfi_check)
281 VReport(1, "Module '%s' __cfi_check %zx\n", info->dlpi_name, cfi_check);
282
283 ShadowBuilder *b = reinterpret_cast<ShadowBuilder *>(data);
284
285 for (int i = 0; i < info->dlpi_phnum; i++) {
286 const Elf_Phdr *phdr = &info->dlpi_phdr[i];
287 if (phdr->p_type == PT_LOAD) {
288 // Jump tables are in the executable segment.
289 // VTables are in the non-executable one.
290 // Need to fill shadow for both.
291 // FIXME: reject writable if vtables are in the r/o segment. Depend on
292 // PT_RELRO?
293 uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;
294 uptr cur_end = cur_beg + phdr->p_memsz;
295 if (cfi_check) {
296 VReport(1, " %zx .. %zx\n", cur_beg, cur_end);
297 b->Add(begin: cur_beg, end: cur_end, cfi_check);
298 } else {
299 b->AddUnchecked(begin: cur_beg, end: cur_end);
300 }
301 }
302 }
303 return 0;
304}
305
306// Init or update shadow for the current set of loaded libraries.
307void UpdateShadow() {
308 ShadowBuilder b;
309 b.Start();
310 dl_iterate_phdr(callback: dl_iterate_phdr_cb, data: &b);
311 b.Install();
312}
313
314void InitShadow() {
315 CHECK_EQ(0, GetShadow());
316 CHECK_EQ(0, GetShadowSize());
317
318 uptr vma = GetMaxUserVirtualAddress();
319 // Shadow is 2 -> 2**kShadowGranularity.
320 SetShadowSize((vma >> (kShadowGranularity - 1)) + 1);
321 VReport(1, "CFI: VMA size %zx, shadow size %zx\n", vma, GetShadowSize());
322
323 UpdateShadow();
324}
325
326THREADLOCAL int in_loader;
327Mutex shadow_update_lock;
328
329void EnterLoader() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
330 if (in_loader == 0) {
331 shadow_update_lock.Lock();
332 }
333 ++in_loader;
334}
335
336void ExitLoader() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {
337 CHECK(in_loader > 0);
338 --in_loader;
339 UpdateShadow();
340 if (in_loader == 0) {
341 shadow_update_lock.Unlock();
342 }
343}
344
345ALWAYS_INLINE void CfiSlowPathCommon(u64 CallSiteTypeId, void *Ptr,
346 void *DiagData) {
347 uptr Addr = (uptr)Ptr;
348 VReport(3, "__cfi_slowpath: %llx, %p\n", CallSiteTypeId, Ptr);
349 ShadowValue sv = ShadowValue::load(addr: Addr);
350 if (sv.is_invalid()) {
351 VReport(1, "CFI: invalid memory region for a check target: %p\n", Ptr);
352#ifdef CFI_ENABLE_DIAG
353 if (DiagData) {
354 __ubsan_handle_cfi_check_fail(
355 reinterpret_cast<__ubsan::CFICheckFailData *>(DiagData), Addr, false);
356 return;
357 }
358#endif
359 Trap();
360 }
361 if (sv.is_unchecked()) {
362 VReport(2, "CFI: unchecked call (shadow=FFFF): %p\n", Ptr);
363 return;
364 }
365 CFICheckFn cfi_check = sv.get_cfi_check();
366 VReport(2, "__cfi_check at %p\n", (void *)cfi_check);
367 cfi_check(CallSiteTypeId, Ptr, DiagData);
368}
369
370void InitializeFlags() {
371 SetCommonFlagsDefaults();
372#ifdef CFI_ENABLE_DIAG
373 __ubsan::Flags *uf = __ubsan::flags();
374 uf->SetDefaults();
375#endif
376
377 FlagParser cfi_parser;
378 RegisterCommonFlags(parser: &cfi_parser);
379 cfi_parser.ParseStringFromEnv(env_name: "CFI_OPTIONS");
380
381#ifdef CFI_ENABLE_DIAG
382 FlagParser ubsan_parser;
383 __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
384 RegisterCommonFlags(&ubsan_parser);
385
386 const char *ubsan_default_options = __ubsan_default_options();
387 ubsan_parser.ParseString(ubsan_default_options);
388 ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
389#endif
390
391 InitializeCommonFlags();
392
393 if (Verbosity())
394 ReportUnrecognizedFlags();
395
396 if (common_flags()->help) {
397 cfi_parser.PrintFlagDescriptions();
398 }
399}
400
401} // namespace __cfi
402
403using namespace __cfi;
404
405extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
406__cfi_slowpath(u64 CallSiteTypeId, void *Ptr) {
407 CfiSlowPathCommon(CallSiteTypeId, Ptr, DiagData: nullptr);
408}
409
410#ifdef CFI_ENABLE_DIAG
411extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
412__cfi_slowpath_diag(u64 CallSiteTypeId, void *Ptr, void *DiagData) {
413 CfiSlowPathCommon(CallSiteTypeId, Ptr, DiagData);
414}
415#endif
416
417static void EnsureInterceptorsInitialized();
418
419// Setup shadow for dlopen()ed libraries.
420// The actual shadow setup happens after dlopen() returns, which means that
421// a library can not be a target of any CFI checks while its constructors are
422// running. It's unclear how to fix this without some extra help from libc.
423// In glibc, mmap inside dlopen is not interceptable.
424// Maybe a seccomp-bpf filter?
425// We could insert a high-priority constructor into the library, but that would
426// not help with the uninstrumented libraries.
427INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
428 EnsureInterceptorsInitialized();
429 EnterLoader();
430 void *handle = REAL(dlopen)(filename, flag);
431 ExitLoader();
432 return handle;
433}
434
435INTERCEPTOR(int, dlclose, void *handle) {
436 EnsureInterceptorsInitialized();
437 EnterLoader();
438 int res = REAL(dlclose)(handle);
439 ExitLoader();
440 return res;
441}
442
443static Mutex interceptor_init_lock;
444static bool interceptors_inited = false;
445
446static void EnsureInterceptorsInitialized() {
447 Lock lock(&interceptor_init_lock);
448 if (interceptors_inited)
449 return;
450
451 INTERCEPT_FUNCTION(dlopen);
452 INTERCEPT_FUNCTION(dlclose);
453
454 interceptors_inited = true;
455}
456
457extern "C" SANITIZER_INTERFACE_ATTRIBUTE
458#if !SANITIZER_CAN_USE_PREINIT_ARRAY
459// On ELF platforms, the constructor is invoked using .preinit_array (see below)
460__attribute__((constructor(0)))
461#endif
462void __cfi_init() {
463 SanitizerToolName = "CFI";
464 InitializeFlags();
465 InitShadow();
466
467#ifdef CFI_ENABLE_DIAG
468 __ubsan::InitAsPlugin();
469#endif
470}
471
472#if SANITIZER_CAN_USE_PREINIT_ARRAY
473// On ELF platforms, run cfi initialization before any other constructors.
474// On other platforms we use the constructor attribute to arrange to run our
475// initialization early.
476extern "C" {
477__attribute__((section(".preinit_array"),
478 used)) void (*__cfi_preinit)(void) = __cfi_init;
479}
480#endif
481