1//===- ARM64.cpp ----------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Arch/ARM64Common.h"
10#include "InputFiles.h"
11#include "Symbols.h"
12#include "SyntheticSections.h"
13#include "Target.h"
14
15#include "lld/Common/ErrorHandler.h"
16#include "mach-o/compact_unwind_encoding.h"
17#include "llvm/BinaryFormat/MachO.h"
18
19using namespace llvm;
20using namespace llvm::MachO;
21using namespace lld;
22using namespace lld::macho;
23
24namespace {
25
26struct ARM64 : ARM64Common {
27 ARM64();
28 void writeStub(uint8_t *buf, const Symbol &, uint64_t) const override;
29 void writeStubHelperHeader(uint8_t *buf) const override;
30 void writeStubHelperEntry(uint8_t *buf, const Symbol &,
31 uint64_t entryAddr) const override;
32
33 void writeObjCMsgSendStub(uint8_t *buf, Symbol *sym, uint64_t stubsAddr,
34 uint64_t &stubOffset, uint64_t selrefVA,
35 Symbol *objcMsgSend) const override;
36 void writeObjCMsgSendClassStub(uint8_t *buf, Symbol *sym, uint64_t stubsAddr,
37 uint64_t &stubOffset, uint64_t selrefVA,
38 Symbol *classSym,
39 Symbol *objcMsgSend) const override;
40 bool supportsObjCClassStubs() const override { return true; }
41 void populateThunk(InputSection *thunk, Symbol *funcSym,
42 int64_t addend) override;
43
44 void initICFSafeThunkBody(InputSection *thunk,
45 Symbol *targetSym) const override;
46 Symbol *getThunkBranchTarget(InputSection *thunk) const override;
47 uint32_t getICFSafeThunkSize() const override;
48};
49
50struct ARM64e final : ARM64 {
51 ARM64e();
52 void writeStub(uint8_t *buf, const Symbol &, uint64_t) const override;
53 void writeObjCMsgSendStub(uint8_t *buf, Symbol *sym, uint64_t stubsAddr,
54 uint64_t &stubOffset, uint64_t selrefVA,
55 Symbol *objcMsgSend) const override;
56 void writeObjCMsgSendClassStub(uint8_t *buf, Symbol *sym, uint64_t stubsAddr,
57 uint64_t &stubOffset, uint64_t selrefVA,
58 Symbol *classSym,
59 Symbol *objcMsgSend) const override;
60};
61} // namespace
62
63// Random notes on reloc types:
64// ADDEND always pairs with BRANCH26, PAGE21, or PAGEOFF12
65// POINTER_TO_GOT: ld64 supports a 4-byte pc-relative form as well as an 8-byte
66// absolute version of this relocation. The semantics of the absolute relocation
67// are weird -- it results in the value of the GOT slot being written, instead
68// of the address. Let's not support it unless we find a real-world use case.
69static constexpr std::array<RelocAttrs, 12> relocAttrsArray{._M_elems: {
70#define B(x) RelocAttrBits::x
71 {.name: "UNSIGNED",
72 B(UNSIGNED) | B(ABSOLUTE) | B(EXTERN) | B(LOCAL) | B(BYTE4) | B(BYTE8)},
73 {.name: "SUBTRACTOR", B(SUBTRAHEND) | B(EXTERN) | B(BYTE4) | B(BYTE8)},
74 {.name: "BRANCH26", B(PCREL) | B(EXTERN) | B(BRANCH) | B(BYTE4)},
75 {.name: "PAGE21", B(PCREL) | B(EXTERN) | B(BYTE4)},
76 {.name: "PAGEOFF12", B(ABSOLUTE) | B(EXTERN) | B(BYTE4)},
77 {.name: "GOT_LOAD_PAGE21", B(PCREL) | B(EXTERN) | B(GOT) | B(BYTE4)},
78 {.name: "GOT_LOAD_PAGEOFF12",
79 B(ABSOLUTE) | B(EXTERN) | B(GOT) | B(LOAD) | B(BYTE4)},
80 {.name: "POINTER_TO_GOT", B(PCREL) | B(EXTERN) | B(GOT) | B(POINTER) | B(BYTE4)},
81 {.name: "TLVP_LOAD_PAGE21", B(PCREL) | B(EXTERN) | B(TLV) | B(BYTE4)},
82 {.name: "TLVP_LOAD_PAGEOFF12",
83 B(ABSOLUTE) | B(EXTERN) | B(TLV) | B(LOAD) | B(BYTE4)},
84 {.name: "ADDEND", B(ADDEND)},
85 // arm64e only.
86 {.name: "AUTHENTICATED_POINTER",
87 B(ABSOLUTE) | B(UNSIGNED) | B(EXTERN) | B(BYTE8) | B(AUTH)},
88#undef B
89}};
90
91static constexpr uint32_t stubCode[] = {
92 0x90000010, // 00: adrp x16, __la_symbol_ptr@page
93 0xf9400210, // 04: ldr x16, [x16, __la_symbol_ptr@pageoff]
94 0xd61f0200, // 08: br x16
95};
96
97void ARM64::writeStub(uint8_t *buf8, const Symbol &sym,
98 uint64_t pointerVA) const {
99 ::writeStub(buf8, stubCode, sym, pointerVA);
100}
101
102static constexpr uint32_t stubHelperHeaderCode[] = {
103 0x90000011, // 00: adrp x17, _dyld_private@page
104 0x91000231, // 04: add x17, x17, _dyld_private@pageoff
105 0xa9bf47f0, // 08: stp x16/x17, [sp, #-16]!
106 0x90000010, // 0c: adrp x16, dyld_stub_binder@page
107 0xf9400210, // 10: ldr x16, [x16, dyld_stub_binder@pageoff]
108 0xd61f0200, // 14: br x16
109};
110
111void ARM64::writeStubHelperHeader(uint8_t *buf8) const {
112 ::writeStubHelperHeader<LP64>(buf8, stubHelperHeaderCode);
113}
114
115static constexpr uint32_t stubHelperEntryCode[] = {
116 0x18000050, // 00: ldr w16, l0
117 0x14000000, // 04: b stubHelperHeader
118 0x00000000, // 08: l0: .long 0
119};
120
121void ARM64::writeStubHelperEntry(uint8_t *buf8, const Symbol &sym,
122 uint64_t entryVA) const {
123 ::writeStubHelperEntry(buf8, stubHelperEntryCode, sym, entryVA);
124}
125
126static constexpr uint32_t objcStubsFastCode[] = {
127 0x90000001, // adrp x1, __objc_selrefs@page
128 0xf9400021, // ldr x1, [x1, @selector("foo")@pageoff]
129 0x90000010, // adrp x16, _got@page
130 0xf9400210, // ldr x16, [x16, _objc_msgSend@pageoff]
131 0xd61f0200, // br x16
132 0xd4200020, // brk #0x1
133 0xd4200020, // brk #0x1
134 0xd4200020, // brk #0x1
135};
136
137static constexpr uint32_t objcStubsSmallCode[] = {
138 0x90000001, // adrp x1, __objc_selrefs@page
139 0xf9400021, // ldr x1, [x1, @selector("foo")@pageoff]
140 0x14000000, // b _objc_msgSend
141};
142
143static constexpr uint32_t objcClassStubsFastCode[] = {
144 // A locally defined class uses adrp @page and add @pageoff instead.
145 0x90000000, // adrp x0, _OBJC_CLASS_$_Foo@gotpage
146 0xf9400000, // ldr x0, [x0, _OBJC_CLASS_$_Foo@gotpageoff]
147 0x90000001, // adrp x1, @selector("foo")@page
148 0xf9400021, // ldr x1, [x1, @selector("foo")@pageoff]
149 0x90000010, // adrp x16, _objc_msgSend@gotpage
150 0xf9400210, // ldr x16, [x16, _objc_msgSend@gotpageoff]
151 0xd61f0200, // br x16
152 0xd4200020, // brk #0x1
153};
154
155static constexpr uint32_t objcClassStubsSmallCode[] = {
156 // A locally defined class uses adrp @page and add @pageoff instead.
157 0x90000000, // adrp x0, _OBJC_CLASS_$_Foo@gotpage
158 0xf9400000, // ldr x0, [x0, _OBJC_CLASS_$_Foo@gotpageoff]
159 0x90000001, // adrp x1, @selector("foo")@page
160 0xf9400021, // ldr x1, [x1, @selector("foo")@pageoff]
161 0x14000000, // b _objc_msgSend
162};
163
164void ARM64::writeObjCMsgSendStub(uint8_t *buf, Symbol *sym, uint64_t stubsAddr,
165 uint64_t &stubOffset, uint64_t selrefVA,
166 Symbol *objcMsgSend) const {
167 uint64_t objcMsgSendAddr;
168 uint64_t objcStubSize;
169 uint64_t objcMsgSendIndex;
170
171 if (config->objcStubsMode == ObjCStubsMode::fast) {
172 objcStubSize = target->objcStubsFastSize;
173 objcMsgSendAddr = in.got->addr;
174 objcMsgSendIndex = objcMsgSend->gotIndex;
175 ::writeObjCMsgSendFastStub<LP64>(buf, objcStubsFastCode, sym, stubsAddr,
176 stubOffset, selrefVA, gotAddr: objcMsgSendAddr,
177 msgSendIndex: objcMsgSendIndex);
178 } else {
179 assert(config->objcStubsMode == ObjCStubsMode::small);
180 objcStubSize = target->objcStubsSmallSize;
181 if (auto *d = dyn_cast<Defined>(Val: objcMsgSend)) {
182 objcMsgSendAddr = d->getVA();
183 objcMsgSendIndex = 0;
184 } else {
185 objcMsgSendAddr = in.stubs->addr;
186 objcMsgSendIndex = objcMsgSend->stubsIndex;
187 }
188 ::writeObjCMsgSendSmallStub<LP64>(buf, objcStubsSmallCode, sym, stubsAddr,
189 stubOffset, selrefVA, msgSendAddr: objcMsgSendAddr,
190 msgSendIndex: objcMsgSendIndex);
191 }
192 stubOffset += objcStubSize;
193}
194
195// Loads the class into x0, through its GOT slot if it needs binding.
196static void writeObjCClassLoad(uint32_t *buf32, const SymbolDiagnostic &d,
197 Symbol *classSym, const uint32_t code[2],
198 uint64_t pcVA) {
199 uint64_t classVA;
200 uint32_t pageOffCode = code[1];
201 if (needsBinding(sym: classSym)) {
202 classVA = in.got->getVA(gotIndex: classSym->gotIndex);
203 } else {
204 classVA = classSym->getVA();
205 pageOffCode = 0x91000000; // add x0, x0, _OBJC_CLASS_$_Foo@pageoff
206 }
207 encodePage21(loc: &buf32[0], d, base: code[0], va: pageBits(address: classVA) - pageBits(address: pcVA));
208 encodePageOff12(loc: &buf32[1], t: d, base: pageOffCode, va: classVA);
209}
210
211// Shared with ARM64e, which passes its own code and _objc_msgSend slot.
212static void writeObjCMsgSendClassFastStub(uint8_t *buf, Symbol *stubSym,
213 uint64_t stubsAddr,
214 uint64_t &stubOffset,
215 uint64_t selrefVA, Symbol *classSym,
216 const uint32_t fastCode[8],
217 uint64_t msgSendSlotVA) {
218 SymbolDiagnostic d = {.symbol: stubSym, .reason: stubSym->getName()};
219 auto *buf32 = reinterpret_cast<uint32_t *>(buf);
220 uint64_t pcVA = stubsAddr + stubOffset;
221 writeObjCClassLoad(buf32, d, classSym, code: fastCode, pcVA);
222 encodePage21(loc: &buf32[2], d, base: fastCode[2],
223 va: pageBits(address: selrefVA) - pageBits(address: pcVA + 2 * sizeof(uint32_t)));
224 encodePageOff12(loc: &buf32[3], t: d, base: fastCode[3], va: selrefVA);
225 encodePage21(loc: &buf32[4], d, base: fastCode[4],
226 va: pageBits(address: msgSendSlotVA) - pageBits(address: pcVA + 4 * sizeof(uint32_t)));
227 encodePageOff12(loc: &buf32[5], t: d, base: fastCode[5], va: msgSendSlotVA);
228 buf32[6] = fastCode[6];
229 buf32[7] = fastCode[7];
230 assert(target->objcClassStubsFastSize == 8 * sizeof(uint32_t));
231 stubOffset += target->objcClassStubsFastSize;
232}
233
234void ARM64::writeObjCMsgSendClassStub(uint8_t *buf, Symbol *stubSym,
235 uint64_t stubsAddr, uint64_t &stubOffset,
236 uint64_t selrefVA, Symbol *classSym,
237 Symbol *objcMsgSend) const {
238 if (config->objcStubsMode == ObjCStubsMode::fast) {
239 writeObjCMsgSendClassFastStub(buf, stubSym, stubsAddr, stubOffset, selrefVA,
240 classSym, fastCode: objcClassStubsFastCode,
241 msgSendSlotVA: in.got->getVA(gotIndex: objcMsgSend->gotIndex));
242 return;
243 }
244
245 assert(config->objcStubsMode == ObjCStubsMode::small);
246 SymbolDiagnostic d = {.symbol: stubSym, .reason: stubSym->getName()};
247 auto *buf32 = reinterpret_cast<uint32_t *>(buf);
248 uint64_t pcVA = stubsAddr + stubOffset;
249 writeObjCClassLoad(buf32, d, classSym, code: objcClassStubsSmallCode, pcVA);
250 encodePage21(loc: &buf32[2], d, base: objcClassStubsSmallCode[2],
251 va: pageBits(address: selrefVA) - pageBits(address: pcVA + 2 * sizeof(uint32_t)));
252 encodePageOff12(loc: &buf32[3], t: d, base: objcClassStubsSmallCode[3], va: selrefVA);
253 uint64_t msgSendStubVA;
254 if (auto *defined = dyn_cast<Defined>(Val: objcMsgSend))
255 msgSendStubVA = defined->getVA();
256 else
257 msgSendStubVA = in.stubs->addr + objcMsgSend->stubsIndex * target->stubSize;
258 encodeBranch26(loc: &buf32[4], d: {.symbol: nullptr, .reason: "objc_msgSend class stub"},
259 base: objcClassStubsSmallCode[4],
260 va: msgSendStubVA - (pcVA + 4 * sizeof(uint32_t)));
261 assert(target->objcClassStubsSmallSize == 5 * sizeof(uint32_t));
262 stubOffset += target->objcClassStubsSmallSize;
263}
264
265// A thunk is the relaxed variation of stubCode. We don't need the
266// extra indirection through a lazy pointer because the target address
267// is known at link time.
268static constexpr uint32_t thunkCode[] = {
269 0x90000010, // 00: adrp x16, <thunk.ptr>@page
270 0x91000210, // 04: add x16, [x16,<thunk.ptr>@pageoff]
271 0xd61f0200, // 08: br x16
272};
273
274void ARM64::populateThunk(InputSection *thunk, Symbol *funcSym,
275 int64_t addend) {
276 thunk->align = 4;
277 thunk->data = {reinterpret_cast<const uint8_t *>(thunkCode),
278 sizeof(thunkCode)};
279 thunk->relocs.emplace_back(/*type=*/args: ARM64_RELOC_PAGEOFF12,
280 /*pcrel=*/args: false, /*length=*/args: 2,
281 /*offset=*/args: 4, /*addend=*/args&: addend,
282 /*referent=*/args&: funcSym);
283 thunk->relocs.emplace_back(/*type=*/args: ARM64_RELOC_PAGE21,
284 /*pcrel=*/args: true, /*length=*/args: 2,
285 /*offset=*/args: 0, /*addend=*/args&: addend,
286 /*referent=*/args&: funcSym);
287}
288// Just a single direct branch to the target function.
289static constexpr uint32_t icfSafeThunkCode[] = {
290 0x14000000, // 08: b target
291};
292
293void ARM64::initICFSafeThunkBody(InputSection *thunk, Symbol *targetSym) const {
294 // The base data here will not be itself modified, we'll just be adding a
295 // reloc below. So we can directly use the constexpr above as the data.
296 thunk->data = {reinterpret_cast<const uint8_t *>(icfSafeThunkCode),
297 sizeof(icfSafeThunkCode)};
298
299 thunk->relocs.emplace_back(/*type=*/args: ARM64_RELOC_BRANCH26,
300 /*pcrel=*/args: true, /*length=*/args: 2,
301 /*offset=*/args: 0, /*addend=*/args: 0,
302 /*referent=*/args&: targetSym);
303}
304
305Symbol *ARM64::getThunkBranchTarget(InputSection *thunk) const {
306 assert(thunk->relocs.size() == 1 &&
307 "expected a single reloc on ARM64 ICF thunk");
308 auto &reloc = thunk->relocs[0];
309 assert(isa<Symbol *>(reloc.referent) &&
310 "ARM64 thunk reloc is expected to point to a Symbol");
311
312 return cast<Symbol *>(Val&: reloc.referent);
313}
314
315uint32_t ARM64::getICFSafeThunkSize() const { return sizeof(icfSafeThunkCode); }
316
317ARM64::ARM64() : ARM64Common(LP64()) {
318 cpuType = CPU_TYPE_ARM64;
319 cpuSubtype = CPU_SUBTYPE_ARM64_ALL;
320
321 stubSize = sizeof(stubCode);
322 thunkSize = sizeof(thunkCode);
323
324 objcStubsFastSize = sizeof(objcStubsFastCode);
325 objcStubsFastAlignment = 32;
326 objcStubsSmallSize = sizeof(objcStubsSmallCode);
327 objcStubsSmallAlignment = 4;
328 objcClassStubsFastSize = sizeof(objcClassStubsFastCode);
329 objcClassStubsSmallSize = sizeof(objcClassStubsSmallCode);
330
331 // Branch immediate is two's complement 26 bits, which is implicitly
332 // multiplied by 4 (since all functions are 4-aligned: The branch range
333 // is -4*(2**(26-1))..4*(2**(26-1) - 1).
334 backwardBranchRange = 128 * 1024 * 1024;
335 forwardBranchRange = backwardBranchRange - 4;
336
337 modeDwarfEncoding = UNWIND_ARM64_MODE_DWARF;
338 subtractorRelocType = ARM64_RELOC_SUBTRACTOR;
339 unsignedRelocType = ARM64_RELOC_UNSIGNED;
340
341 stubHelperHeaderSize = sizeof(stubHelperHeaderCode);
342 stubHelperEntrySize = sizeof(stubHelperEntryCode);
343
344 relocAttrs = ArrayRef(relocAttrsArray).drop_back();
345}
346
347TargetInfo *macho::createARM64TargetInfo() {
348 static ARM64 t;
349 return &t;
350}
351
352// arm64e stubs authenticate the target they load from __auth_got.
353static constexpr uint32_t authStubCode[] = {
354 0x90000011, // 00: adrp x17, __auth_got@page
355 0x91000231, // 04: add x17, x17, __auth_got@pageoff
356 0xf9400230, // 08: ldr x16, [x17]
357 0xd71f0a11, // 0c: braa x16, x17
358};
359
360void ARM64e::writeStub(uint8_t *buf8, const Symbol &sym,
361 uint64_t pointerVA) const {
362 auto *buf32 = reinterpret_cast<uint32_t *>(buf8);
363 SymbolDiagnostic d = {.symbol: &sym, .reason: "stub"};
364 uint64_t pcPageBits = pageBits(address: in.stubs->addr + sym.stubsIndex * stubSize);
365 encodePage21(loc: &buf32[0], d, base: authStubCode[0], va: pageBits(address: pointerVA) - pcPageBits);
366 encodePageOff12(loc: &buf32[1], t: d, base: authStubCode[1], va: pointerVA);
367 buf32[2] = authStubCode[2];
368 buf32[3] = authStubCode[3];
369}
370
371// The fast stubs reach _objc_msgSend through __auth_got in the same way.
372static constexpr uint32_t authObjcStubsFastCode[] = {
373 0x90000001, // adrp x1, __objc_selrefs@page
374 0xf9400021, // ldr x1, [x1, @selector("foo")@pageoff]
375 0x90000011, // adrp x17, __auth_got@page
376 0x91000231, // add x17, x17, _objc_msgSend@pageoff
377 0xf9400230, // ldr x16, [x17]
378 0xd71f0a11, // braa x16, x17
379 0xd4200020, // brk #0x1
380 0xd4200020, // brk #0x1
381};
382
383void ARM64e::writeObjCMsgSendStub(uint8_t *buf, Symbol *sym, uint64_t stubsAddr,
384 uint64_t &stubOffset, uint64_t selrefVA,
385 Symbol *objcMsgSend) const {
386 // Small stubs branch to _objc_msgSend or its stub, as on arm64.
387 if (config->objcStubsMode == ObjCStubsMode::small) {
388 ARM64::writeObjCMsgSendStub(buf, sym, stubsAddr, stubOffset, selrefVA,
389 objcMsgSend);
390 return;
391 }
392 ::writeObjCMsgSendFastStub<LP64>(buf, objcStubsFastCode: authObjcStubsFastCode, sym, stubsAddr,
393 stubOffset, selrefVA,
394 gotAddr: in.authGot->getVA(sym: objcMsgSend),
395 /*msgSendIndex=*/0);
396 stubOffset += objcStubsFastSize;
397}
398
399// Fast class stubs reach _objc_msgSend in the same way. The class pointer they
400// load is the receiver and stays unsigned, as on arm64.
401static constexpr uint32_t authObjcClassStubsFastCode[] = {
402 // A locally defined class uses adrp @page and add @pageoff instead.
403 0x90000000, // adrp x0, _OBJC_CLASS_$_Foo@gotpage
404 0xf9400000, // ldr x0, [x0, _OBJC_CLASS_$_Foo@gotpageoff]
405 0x90000001, // adrp x1, @selector("foo")@page
406 0xf9400021, // ldr x1, [x1, @selector("foo")@pageoff]
407 0x90000011, // adrp x17, __auth_got@page
408 0x91000231, // add x17, x17, _objc_msgSend@pageoff
409 0xf9400230, // ldr x16, [x17]
410 0xd71f0a11, // braa x16, x17
411};
412static_assert(sizeof(authObjcClassStubsFastCode) ==
413 sizeof(objcClassStubsFastCode),
414 "objcClassStubsFastSize is shared with arm64");
415
416void ARM64e::writeObjCMsgSendClassStub(uint8_t *buf, Symbol *stubSym,
417 uint64_t stubsAddr, uint64_t &stubOffset,
418 uint64_t selrefVA, Symbol *classSym,
419 Symbol *objcMsgSend) const {
420 // Small stubs branch to _objc_msgSend or its stub, as on arm64.
421 if (config->objcStubsMode == ObjCStubsMode::small) {
422 ARM64::writeObjCMsgSendClassStub(buf, stubSym, stubsAddr, stubOffset,
423 selrefVA, classSym, objcMsgSend);
424 return;
425 }
426 writeObjCMsgSendClassFastStub(buf, stubSym, stubsAddr, stubOffset, selrefVA,
427 classSym, fastCode: authObjcClassStubsFastCode,
428 msgSendSlotVA: in.authGot->getVA(sym: objcMsgSend));
429}
430
431ARM64e::ARM64e() {
432 cpuSubtype = CPU_SUBTYPE_ARM64E;
433 stubSize = sizeof(authStubCode);
434 relocAttrs = relocAttrsArray;
435}
436
437TargetInfo *macho::createARM64eTargetInfo() {
438 static ARM64e t;
439 return &t;
440}
441