1//===-- RuntimeDyldCOFFAArch64.h --- COFF/AArch64 specific code ---*- C++*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// COFF AArch64 support for MC-JIT runtime dynamic linker.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFFAARCH64_H
14#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFFAARCH64_H
15
16#include "../RuntimeDyldCOFF.h"
17#include "llvm/ADT/SmallString.h"
18#include "llvm/BinaryFormat/COFF.h"
19#include "llvm/Object/COFF.h"
20#include "llvm/Support/Endian.h"
21
22#define DEBUG_TYPE "dyld"
23
24namespace llvm {
25
26// This relocation type is used for handling long branch instruction
27// through the Stub.
28enum InternalRelocationType : unsigned {
29 INTERNAL_REL_ARM64_LONG_BRANCH26 = 0x111,
30};
31
32static void add16(uint8_t *p, int16_t v) {
33 using namespace llvm::support::endian;
34 write16le(P: p, V: read16le(P: p) + v);
35}
36
37static void or32le(void *P, int32_t V) {
38 using namespace llvm::support::endian;
39
40 write32le(P, V: read32le(P) | V);
41}
42
43static void write32AArch64Imm(uint8_t *T, uint64_t imm, uint32_t rangeLimit) {
44 using namespace llvm::support::endian;
45
46 uint32_t orig = read32le(P: T);
47 orig &= ~(0xFFF << 10);
48 write32le(P: T, V: orig | ((imm & (0xFFF >> rangeLimit)) << 10));
49}
50
51/// Byte addend held in an LDR/STR (unsigned immediate) instruction. Its imm12
52/// field counts access-size units, so reading it back has to undo that scaling
53/// the way write32AArch64Ldr applies it.
54static uint64_t read32AArch64LdrAddend(uint32_t Inst) {
55 uint32_t Size = Inst >> 30;
56 // 0x04000000 indicates SIMD/FP registers, 0x00800000 indicates 128 bit.
57 if ((Inst & 0x04800000) == 0x04800000)
58 Size += 4;
59 return static_cast<uint64_t>((Inst >> 10) & 0xFFF) << Size;
60}
61
62static void write32AArch64Ldr(uint8_t *T, uint64_t imm) {
63 using namespace llvm::support::endian;
64
65 uint32_t orig = read32le(P: T);
66 uint32_t size = orig >> 30;
67 // 0x04000000 indicates SIMD/FP registers
68 // 0x00800000 indicates 128 bit
69 if ((orig & 0x04800000) == 0x04800000)
70 size += 4;
71 if ((imm & ((1 << size) - 1)) != 0)
72 assert(0 && "misaligned ldr/str offset");
73 write32AArch64Imm(T, imm: imm >> size, rangeLimit: size);
74}
75
76static void write32AArch64Addr(void *T, uint64_t s, uint64_t p, int shift) {
77 using namespace llvm::support::endian;
78
79 uint64_t Imm = (s >> shift) - (p >> shift);
80 // ADRP's page offset is 21 bits signed (+/-4 GiB); fail loudly rather than
81 // truncate into a wild pointer.
82 if (!isInt<21>(x: static_cast<int64_t>(Imm))) {
83 report_fatal_error(reason: Twine("ADRP target out of range: target=") + Twine(s) +
84 " pc=" + Twine(p) +
85 " delta=" + Twine(s > p ? s - p : p - s) +
86 (s > p ? " (target above pc)" : " (target below pc)") +
87 " -- needs a 21-bit signed page offset (+/-4 GiB)");
88 }
89 uint32_t ImmLo = (Imm & 0x3) << 29;
90 uint32_t ImmHi = (Imm & 0x1FFFFC) << 3;
91 uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3);
92 write32le(P: T, V: (read32le(P: T) & ~Mask) | ImmLo | ImmHi);
93}
94
95class RuntimeDyldCOFFAArch64 : public RuntimeDyldCOFF {
96
97private:
98 uint64_t ImageBase;
99
100 // Fake an __ImageBase pointer by returning the section with the lowest adress
101 uint64_t getImageBase() {
102 if (!ImageBase) {
103 ImageBase = std::numeric_limits<uint64_t>::max();
104 for (const SectionEntry &Section : Sections)
105 // The Sections list may contain sections that weren't loaded for
106 // whatever reason: they may be debug sections, and ProcessAllSections
107 // is false, or they may be sections that contain 0 bytes. If the
108 // section isn't loaded, the load address will be 0, and it should not
109 // be included in the ImageBase calculation.
110 if (Section.getLoadAddress() != 0)
111 ImageBase = std::min(a: ImageBase, b: Section.getLoadAddress());
112 }
113 return ImageBase;
114 }
115
116public:
117 RuntimeDyldCOFFAArch64(RuntimeDyld::MemoryManager &MM,
118 JITSymbolResolver &Resolver)
119 : RuntimeDyldCOFF(MM, Resolver, 8, COFF::IMAGE_REL_ARM64_ADDR64),
120 ImageBase(0) {}
121
122 Align getStubAlignment() override { return Align(8); }
123
124 unsigned getMaxStubSize() const override { return 20; }
125
126 /// GOT slot in this section's stub area holding the address of \p Name, so
127 /// an ADRP can reach external symbols (e.g. DLL exports) beyond +/-4 GiB.
128 /// computeSectionStubBufSize reserves getMaxStubSize() per relocation, which
129 /// covers the 8 bytes.
130 uint64_t getGOTEntryOffset(unsigned SectionID, StubMap &Stubs, StringRef Name,
131 bool CreateIfMissing) {
132 // Key on the symbol alone; ~0 cannot collide with a branch stub's offset.
133 RelocationValueRef Key;
134 Key.SectionID = SectionID;
135 Key.Offset = ~uint64_t(0);
136 Key.SymbolName = Name.data();
137
138 // RelocationValueRef compares symbol-name pointers rather than their
139 // contents, but paired COFF relocations need not carry the same pointer.
140 for (const auto &[StubKey, StubOffset] : Stubs)
141 if (StubKey.SectionID == SectionID && StubKey.Offset == ~uint64_t(0) &&
142 StubKey.SymbolName && StringRef(StubKey.SymbolName) == Name)
143 return StubOffset;
144 if (!CreateIfMissing)
145 return ~uint64_t(0);
146
147 SectionEntry &Sec = Sections[SectionID];
148 uint64_t EntryOffset = alignTo(Value: Sec.getStubOffset(), Align: 8);
149 Sec.advanceStubOffset(StubSize: EntryOffset + 8 - Sec.getStubOffset());
150 Stubs[Key] = EntryOffset;
151
152 // Have the resolver write the symbol's absolute address into the slot.
153 RelocationEntry RE(SectionID, EntryOffset, COFF::IMAGE_REL_ARM64_ADDR64, 0,
154 /*IsPCRel=*/false, Log2_64(Value: 8));
155 addRelocationForSymbol(RE, SymbolName: Name);
156
157 LLVM_DEBUG(dbgs() << " Created GOT entry for " << Name << " at offset "
158 << EntryOffset << "\n");
159 return EntryOffset;
160 }
161
162 /// ADD (immediate, no shift): the second half of the ADRP+ADD idiom.
163 static bool isAddImmediate(uint32_t Inst) {
164 // sf 0 0 100010 sh imm12 Rn Rd with sh == 0; excludes ADDS (bit 29).
165 return (Inst & 0x7FC00000) == 0x11000000;
166 }
167
168 std::tuple<uint64_t, uint64_t, uint64_t>
169 generateRelocationStub(unsigned SectionID, StringRef TargetName,
170 uint64_t Offset, uint64_t RelType, uint64_t Addend,
171 StubMap &Stubs) {
172 uintptr_t StubOffset;
173 SectionEntry &Section = Sections[SectionID];
174
175 RelocationValueRef OriginalRelValueRef;
176 OriginalRelValueRef.SectionID = SectionID;
177 OriginalRelValueRef.Offset = Offset;
178 OriginalRelValueRef.Addend = Addend;
179 OriginalRelValueRef.SymbolName = TargetName.data();
180
181 auto [Stub, Inserted] = Stubs.try_emplace(k: OriginalRelValueRef);
182 if (Inserted) {
183 LLVM_DEBUG(dbgs() << " Create a new stub function for "
184 << TargetName.data() << "\n");
185
186 StubOffset = Section.getStubOffset();
187 Stub->second = StubOffset;
188 createStubFunction(Addr: Section.getAddressWithOffset(OffsetBytes: StubOffset));
189 Section.advanceStubOffset(StubSize: getMaxStubSize());
190 } else {
191 LLVM_DEBUG(dbgs() << " Stub function found for " << TargetName.data()
192 << "\n");
193 StubOffset = Stub->second;
194 }
195
196 // Resolve original relocation to stub function.
197 const RelocationEntry RE(SectionID, Offset, RelType, Addend);
198 resolveRelocation(RE, Value: Section.getLoadAddressWithOffset(OffsetBytes: StubOffset));
199
200 // adjust relocation info so resolution writes to the stub function
201 // Here an internal relocation type is used for resolving long branch via
202 // stub instruction.
203 Addend = 0;
204 Offset = StubOffset;
205 RelType = INTERNAL_REL_ARM64_LONG_BRANCH26;
206
207 return std::make_tuple(args&: Offset, args&: RelType, args&: Addend);
208 }
209
210 Expected<object::relocation_iterator>
211 processRelocationRef(unsigned SectionID, object::relocation_iterator RelI,
212 const object::ObjectFile &Obj,
213 ObjSectionToIDMap &ObjSectionToID,
214 StubMap &Stubs) override {
215 using namespace llvm::support::endian;
216
217 auto Symbol = RelI->getSymbol();
218 if (Symbol == Obj.symbol_end())
219 report_fatal_error(reason: "Unknown symbol in relocation");
220
221 Expected<StringRef> TargetNameOrErr = Symbol->getName();
222 if (!TargetNameOrErr)
223 return TargetNameOrErr.takeError();
224 StringRef TargetName = *TargetNameOrErr;
225
226 auto SectionOrErr = Symbol->getSection();
227 if (!SectionOrErr)
228 return SectionOrErr.takeError();
229 auto Section = *SectionOrErr;
230
231 uint64_t RelType = RelI->getType();
232 uint64_t Offset = RelI->getOffset();
233
234 // If there is no section, this must be an external reference.
235 bool IsExtern = Section == Obj.section_end();
236
237 // Determine the Addend used to adjust the relocation value.
238 uint64_t Addend = 0;
239 SectionEntry &AddendSection = Sections[SectionID];
240 uintptr_t ObjTarget = AddendSection.getObjAddress() + Offset;
241 uint8_t *Displacement = (uint8_t *)ObjTarget;
242
243 unsigned TargetSectionID = -1;
244 uint64_t TargetOffset = -1;
245
246 if (TargetName.starts_with(Prefix: getImportSymbolPrefix())) {
247 TargetSectionID = SectionID;
248 TargetOffset = getDLLImportOffset(SectionID, Stubs, Name: TargetName);
249 TargetName = StringRef();
250 IsExtern = false;
251 } else if (!IsExtern) {
252 if (auto TargetSectionIDOrErr = findOrEmitSection(
253 Obj, Section: *Section, IsCode: Section->isText(), LocalSections&: ObjSectionToID))
254 TargetSectionID = *TargetSectionIDOrErr;
255 else
256 return TargetSectionIDOrErr.takeError();
257
258 TargetOffset = getSymbolOffset(Sym: *Symbol);
259 }
260
261 switch (RelType) {
262 case COFF::IMAGE_REL_ARM64_ADDR32:
263 case COFF::IMAGE_REL_ARM64_ADDR32NB:
264 case COFF::IMAGE_REL_ARM64_REL32:
265 case COFF::IMAGE_REL_ARM64_SECREL:
266 Addend = read32le(P: Displacement);
267 break;
268 case COFF::IMAGE_REL_ARM64_BRANCH26: {
269 uint32_t orig = read32le(P: Displacement);
270 Addend = (orig & 0x03FFFFFF) << 2;
271
272 if (IsExtern)
273 std::tie(args&: Offset, args&: RelType, args&: Addend) = generateRelocationStub(
274 SectionID, TargetName, Offset, RelType, Addend, Stubs);
275 break;
276 }
277 case COFF::IMAGE_REL_ARM64_BRANCH19: {
278 uint32_t orig = read32le(P: Displacement);
279 Addend = (orig & 0x00FFFFE0) >> 3;
280 break;
281 }
282 case COFF::IMAGE_REL_ARM64_BRANCH14: {
283 uint32_t orig = read32le(P: Displacement);
284 Addend = (orig & 0x000FFFE0) >> 3;
285 break;
286 }
287 case COFF::IMAGE_REL_ARM64_REL21:
288 case COFF::IMAGE_REL_ARM64_PAGEBASE_REL21: {
289 uint32_t orig = read32le(P: Displacement);
290 Addend = ((orig >> 29) & 0x3) | ((orig >> 3) & 0x1FFFFC);
291
292 // External symbols are routinely beyond ADRP's +/-4 GiB on Windows and
293 // there is no veneer, so route ADRP+ADD through a GOT slot. Only that
294 // idiom qualifies: ADRP+LDR reads the symbol's contents, and a non-zero
295 // addend cannot be expressed by a slot holding plain `sym`. LLVM codegen
296 // emits direct addresses as a MOVaddr pseudo and expands it after
297 // scheduling into this adjacent ADRP+ADD pair.
298 if (IsExtern && Addend == 0 &&
299 RelType == COFF::IMAGE_REL_ARM64_PAGEBASE_REL21) {
300 uint32_t NextInst = read32le(P: Displacement + 4);
301 if (isAddImmediate(Inst: NextInst) && ((NextInst >> 10) & 0xFFF) == 0) {
302 TargetOffset = getGOTEntryOffset(SectionID, Stubs, Name: TargetName,
303 /*CreateIfMissing=*/CreateIfMissing: true);
304 TargetSectionID = SectionID;
305 TargetName = StringRef();
306 IsExtern = false;
307 Addend = 0;
308 }
309 }
310 break;
311 }
312 case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12L: {
313 Addend = read32AArch64LdrAddend(Inst: read32le(P: Displacement));
314 break;
315 }
316 case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12A: {
317 uint32_t orig = read32le(P: Displacement);
318 Addend = ((orig >> 10) & 0xFFF);
319
320 // Partner of the ADRP above; no slot means that ADRP was left alone.
321 if (IsExtern && Addend == 0 &&
322 RelType == COFF::IMAGE_REL_ARM64_PAGEOFFSET_12A) {
323 uint64_t GOTOffset = getGOTEntryOffset(SectionID, Stubs, Name: TargetName,
324 /*CreateIfMissing=*/CreateIfMissing: false);
325 if (GOTOffset != ~uint64_t(0) && isAddImmediate(Inst: orig)) {
326 // ADD Xd, Xn, #imm -> LDR Xd, [Xn, #imm]: load the slot instead.
327 uint32_t Rd = orig & 0x1F;
328 uint32_t Rn = (orig >> 5) & 0x1F;
329 write32le(P: AddendSection.getAddressWithOffset(OffsetBytes: Offset),
330 V: 0xF9400000u | (Rn << 5) | Rd);
331
332 // 12L so the resolver scales the immediate for a 64-bit load.
333 RelType = COFF::IMAGE_REL_ARM64_PAGEOFFSET_12L;
334 TargetOffset = GOTOffset;
335 TargetSectionID = SectionID;
336 TargetName = StringRef();
337 IsExtern = false;
338 Addend = 0;
339 }
340 }
341 break;
342 }
343
344 case COFF::IMAGE_REL_ARM64_ADDR64: {
345 Addend = read64le(P: Displacement);
346 break;
347 }
348 default:
349 break;
350 }
351
352#if !defined(NDEBUG)
353 SmallString<32> RelTypeName;
354 RelI->getTypeName(RelTypeName);
355
356 LLVM_DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset
357 << " RelType: " << RelTypeName << " TargetName: "
358 << TargetName << " Addend " << Addend << "\n");
359#endif
360
361 if (IsExtern) {
362 RelocationEntry RE(SectionID, Offset, RelType, Addend);
363 addRelocationForSymbol(RE, SymbolName: TargetName);
364 } else {
365 RelocationEntry RE(SectionID, Offset, RelType, TargetOffset + Addend);
366 addRelocationForSection(RE, SectionID: TargetSectionID);
367 }
368 return ++RelI;
369 }
370
371 void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override {
372 using namespace llvm::support::endian;
373
374 const auto Section = Sections[RE.SectionID];
375 uint8_t *Target = Section.getAddressWithOffset(OffsetBytes: RE.Offset);
376 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: RE.Offset);
377
378 switch (RE.RelType) {
379 default:
380 llvm_unreachable("unsupported relocation type");
381 case COFF::IMAGE_REL_ARM64_ABSOLUTE: {
382 // This relocation is ignored.
383 break;
384 }
385 case COFF::IMAGE_REL_ARM64_PAGEBASE_REL21: {
386 // The page base of the target, for ADRP instruction.
387 Value += RE.Addend;
388 write32AArch64Addr(T: Target, s: Value, p: FinalAddress, shift: 12);
389 break;
390 }
391 case COFF::IMAGE_REL_ARM64_REL21: {
392 // The 12-bit relative displacement to the target, for instruction ADR
393 Value += RE.Addend;
394 write32AArch64Addr(T: Target, s: Value, p: FinalAddress, shift: 0);
395 break;
396 }
397 case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12A: {
398 // The 12-bit page offset of the target,
399 // for instructions ADD/ADDS (immediate) with zero shift.
400 Value += RE.Addend;
401 write32AArch64Imm(T: Target, imm: Value & 0xFFF, rangeLimit: 0);
402 break;
403 }
404 case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12L: {
405 // The 12-bit page offset of the target,
406 // for instruction LDR (indexed, unsigned immediate).
407 Value += RE.Addend;
408 write32AArch64Ldr(T: Target, imm: Value & 0xFFF);
409 break;
410 }
411 case COFF::IMAGE_REL_ARM64_ADDR32: {
412 // The 32-bit VA of the target.
413 uint32_t VA = Value + RE.Addend;
414 write32le(P: Target, V: VA);
415 break;
416 }
417 case COFF::IMAGE_REL_ARM64_ADDR32NB: {
418 // The target's 32-bit RVA.
419 uint64_t RVA = Value + RE.Addend - getImageBase();
420 write32le(P: Target, V: RVA);
421 break;
422 }
423 case INTERNAL_REL_ARM64_LONG_BRANCH26: {
424 // Encode the immadiate value for generated Stub instruction (MOVZ)
425 or32le(P: Target + 12, V: ((Value + RE.Addend) & 0xFFFF) << 5);
426 or32le(P: Target + 8, V: ((Value + RE.Addend) & 0xFFFF0000) >> 11);
427 or32le(P: Target + 4, V: ((Value + RE.Addend) & 0xFFFF00000000) >> 27);
428 or32le(P: Target + 0, V: ((Value + RE.Addend) & 0xFFFF000000000000) >> 43);
429 break;
430 }
431 case COFF::IMAGE_REL_ARM64_BRANCH26: {
432 // The 26-bit relative displacement to the target, for B and BL
433 // instructions.
434 uint64_t PCRelVal = Value + RE.Addend - FinalAddress;
435 assert(isInt<28>(PCRelVal) && "Branch target is out of range.");
436 write32le(P: Target, V: (read32le(P: Target) & ~(0x03FFFFFF)) |
437 (PCRelVal & 0x0FFFFFFC) >> 2);
438 break;
439 }
440 case COFF::IMAGE_REL_ARM64_BRANCH19: {
441 // The 19-bit offset to the relocation target,
442 // for conditional B instruction.
443 uint64_t PCRelVal = Value + RE.Addend - FinalAddress;
444 assert(isInt<21>(PCRelVal) && "Branch target is out of range.");
445 write32le(P: Target, V: (read32le(P: Target) & ~(0x00FFFFE0)) |
446 (PCRelVal & 0x001FFFFC) << 3);
447 break;
448 }
449 case COFF::IMAGE_REL_ARM64_BRANCH14: {
450 // The 14-bit offset to the relocation target,
451 // for instructions TBZ and TBNZ.
452 uint64_t PCRelVal = Value + RE.Addend - FinalAddress;
453 assert(isInt<16>(PCRelVal) && "Branch target is out of range.");
454 write32le(P: Target, V: (read32le(P: Target) & ~(0x000FFFE0)) |
455 (PCRelVal & 0x0000FFFC) << 3);
456 break;
457 }
458 case COFF::IMAGE_REL_ARM64_ADDR64: {
459 // The 64-bit VA of the relocation target.
460 write64le(P: Target, V: Value + RE.Addend);
461 break;
462 }
463 case COFF::IMAGE_REL_ARM64_SECTION: {
464 // 16-bit section index of the section that contains the target.
465 assert(static_cast<uint32_t>(RE.SectionID) <= UINT16_MAX &&
466 "relocation overflow");
467 add16(p: Target, v: RE.SectionID);
468 break;
469 }
470 case COFF::IMAGE_REL_ARM64_SECREL: {
471 // 32-bit offset of the target from the beginning of its section.
472 assert(static_cast<int64_t>(RE.Addend) <= INT32_MAX &&
473 "Relocation overflow");
474 assert(static_cast<int64_t>(RE.Addend) >= INT32_MIN &&
475 "Relocation underflow");
476 write32le(P: Target, V: RE.Addend);
477 break;
478 }
479 case COFF::IMAGE_REL_ARM64_REL32: {
480 // The 32-bit relative address from the byte following the relocation.
481 uint64_t Result = Value - FinalAddress - 4;
482 write32le(P: Target, V: Result + RE.Addend);
483 break;
484 }
485 }
486 }
487
488 void registerEHFrames() override {}
489};
490
491} // End namespace llvm
492
493#undef DEBUG_TYPE
494
495#endif // LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFFAARCH64_H
496