1//===-- RuntimeDyldELF.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Implementation of ELF support for the MC-JIT runtime dynamic linker.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RuntimeDyldELF.h"
14#include "Targets/RuntimeDyldELFMips.h"
15#include "llvm/ADT/StringRef.h"
16#include "llvm/BinaryFormat/ELF.h"
17#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
18#include "llvm/Object/ELFObjectFile.h"
19#include "llvm/Object/ObjectFile.h"
20#include "llvm/Support/Endian.h"
21#include "llvm/Support/MemoryBuffer.h"
22#include "llvm/TargetParser/Triple.h"
23
24using namespace llvm;
25using namespace llvm::object;
26using namespace llvm::support::endian;
27
28#define DEBUG_TYPE "dyld"
29
30static void or32le(void *P, int32_t V) { write32le(P, V: read32le(P) | V); }
31
32static void or32AArch64Imm(void *L, uint64_t Imm) {
33 or32le(P: L, V: (Imm & 0xFFF) << 10);
34}
35
36template <class T> static void write(bool isBE, void *P, T V) {
37 isBE ? write<T, llvm::endianness::big>(P, V)
38 : write<T, llvm::endianness::little>(P, V);
39}
40
41static void write32AArch64Addr(void *L, uint64_t Imm) {
42 uint32_t ImmLo = (Imm & 0x3) << 29;
43 uint32_t ImmHi = (Imm & 0x1FFFFC) << 3;
44 uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3);
45 write32le(P: L, V: (read32le(P: L) & ~Mask) | ImmLo | ImmHi);
46}
47
48// Return the bits [Start, End] from Val shifted Start bits.
49// For instance, getBits(0xF0, 4, 8) returns 0xF.
50static uint64_t getBits(uint64_t Val, int Start, int End) {
51 uint64_t Mask = ((uint64_t)1 << (End + 1 - Start)) - 1;
52 return (Val >> Start) & Mask;
53}
54
55namespace {
56
57template <class ELFT> class DyldELFObject : public ELFObjectFile<ELFT> {
58 LLVM_ELF_IMPORT_TYPES_ELFT(ELFT)
59
60 typedef typename ELFT::uint addr_type;
61
62 DyldELFObject(ELFObjectFile<ELFT> &&Obj);
63
64public:
65 static Expected<std::unique_ptr<DyldELFObject>>
66 create(MemoryBufferRef Wrapper);
67
68 void updateSectionAddress(const SectionRef &Sec, uint64_t Addr);
69
70 void updateSymbolAddress(const SymbolRef &SymRef, uint64_t Addr);
71
72 // Methods for type inquiry through isa, cast and dyn_cast
73 static bool classof(const Binary *v) {
74 return (isa<ELFObjectFile<ELFT>>(v) &&
75 classof(cast<ELFObjectFile<ELFT>>(v)));
76 }
77 static bool classof(const ELFObjectFile<ELFT> *v) {
78 return v->isDyldType();
79 }
80};
81
82
83
84// The MemoryBuffer passed into this constructor is just a wrapper around the
85// actual memory. Ultimately, the Binary parent class will take ownership of
86// this MemoryBuffer object but not the underlying memory.
87template <class ELFT>
88DyldELFObject<ELFT>::DyldELFObject(ELFObjectFile<ELFT> &&Obj)
89 : ELFObjectFile<ELFT>(std::move(Obj)) {
90 this->isDyldELFObject = true;
91}
92
93template <class ELFT>
94Expected<std::unique_ptr<DyldELFObject<ELFT>>>
95DyldELFObject<ELFT>::create(MemoryBufferRef Wrapper) {
96 auto Obj = ELFObjectFile<ELFT>::create(Wrapper);
97 if (auto E = Obj.takeError())
98 return std::move(E);
99 std::unique_ptr<DyldELFObject<ELFT>> Ret(
100 new DyldELFObject<ELFT>(std::move(*Obj)));
101 return std::move(Ret);
102}
103
104template <class ELFT>
105void DyldELFObject<ELFT>::updateSectionAddress(const SectionRef &Sec,
106 uint64_t Addr) {
107 DataRefImpl ShdrRef = Sec.getRawDataRefImpl();
108 Elf_Shdr *shdr =
109 const_cast<Elf_Shdr *>(reinterpret_cast<const Elf_Shdr *>(ShdrRef.p));
110
111 // This assumes the address passed in matches the target address bitness
112 // The template-based type cast handles everything else.
113 shdr->sh_addr = static_cast<addr_type>(Addr);
114}
115
116template <class ELFT>
117void DyldELFObject<ELFT>::updateSymbolAddress(const SymbolRef &SymRef,
118 uint64_t Addr) {
119
120 Elf_Sym *sym = const_cast<Elf_Sym *>(
121 ELFObjectFile<ELFT>::getSymbol(SymRef.getRawDataRefImpl()));
122
123 // This assumes the address passed in matches the target address bitness
124 // The template-based type cast handles everything else.
125 sym->st_value = static_cast<addr_type>(Addr);
126}
127
128class LoadedELFObjectInfo final
129 : public LoadedObjectInfoHelper<LoadedELFObjectInfo,
130 RuntimeDyld::LoadedObjectInfo> {
131public:
132 LoadedELFObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap)
133 : LoadedObjectInfoHelper(RTDyld, std::move(ObjSecToIDMap)) {}
134
135 OwningBinary<ObjectFile>
136 getObjectForDebug(const ObjectFile &Obj) const override;
137};
138
139template <typename ELFT>
140static Expected<std::unique_ptr<DyldELFObject<ELFT>>>
141createRTDyldELFObject(MemoryBufferRef Buffer, const ObjectFile &SourceObject,
142 const LoadedELFObjectInfo &L) {
143 typedef typename ELFT::Shdr Elf_Shdr;
144 typedef typename ELFT::uint addr_type;
145
146 Expected<std::unique_ptr<DyldELFObject<ELFT>>> ObjOrErr =
147 DyldELFObject<ELFT>::create(Buffer);
148 if (Error E = ObjOrErr.takeError())
149 return std::move(E);
150
151 std::unique_ptr<DyldELFObject<ELFT>> Obj = std::move(*ObjOrErr);
152
153 // Iterate over all sections in the object.
154 auto SI = SourceObject.section_begin();
155 for (const auto &Sec : Obj->sections()) {
156 Expected<StringRef> NameOrErr = Sec.getName();
157 if (!NameOrErr) {
158 consumeError(Err: NameOrErr.takeError());
159 continue;
160 }
161
162 if (*NameOrErr != "") {
163 DataRefImpl ShdrRef = Sec.getRawDataRefImpl();
164 Elf_Shdr *shdr = const_cast<Elf_Shdr *>(
165 reinterpret_cast<const Elf_Shdr *>(ShdrRef.p));
166
167 if (uint64_t SecLoadAddr = L.getSectionLoadAddress(Sec: *SI)) {
168 // This assumes that the address passed in matches the target address
169 // bitness. The template-based type cast handles everything else.
170 shdr->sh_addr = static_cast<addr_type>(SecLoadAddr);
171 }
172 }
173 ++SI;
174 }
175
176 return std::move(Obj);
177}
178
179static OwningBinary<ObjectFile>
180createELFDebugObject(const ObjectFile &Obj, const LoadedELFObjectInfo &L) {
181 assert(Obj.isELF() && "Not an ELF object file.");
182
183 std::unique_ptr<MemoryBuffer> Buffer =
184 MemoryBuffer::getMemBufferCopy(InputData: Obj.getData(), BufferName: Obj.getFileName());
185
186 Expected<std::unique_ptr<ObjectFile>> DebugObj(nullptr);
187 handleAllErrors(E: DebugObj.takeError());
188 if (Obj.getBytesInAddress() == 4 && Obj.isLittleEndian())
189 DebugObj =
190 createRTDyldELFObject<ELF32LE>(Buffer: Buffer->getMemBufferRef(), SourceObject: Obj, L);
191 else if (Obj.getBytesInAddress() == 4 && !Obj.isLittleEndian())
192 DebugObj =
193 createRTDyldELFObject<ELF32BE>(Buffer: Buffer->getMemBufferRef(), SourceObject: Obj, L);
194 else if (Obj.getBytesInAddress() == 8 && !Obj.isLittleEndian())
195 DebugObj =
196 createRTDyldELFObject<ELF64BE>(Buffer: Buffer->getMemBufferRef(), SourceObject: Obj, L);
197 else if (Obj.getBytesInAddress() == 8 && Obj.isLittleEndian())
198 DebugObj =
199 createRTDyldELFObject<ELF64LE>(Buffer: Buffer->getMemBufferRef(), SourceObject: Obj, L);
200 else
201 llvm_unreachable("Unexpected ELF format");
202
203 handleAllErrors(E: DebugObj.takeError());
204 return OwningBinary<ObjectFile>(std::move(*DebugObj), std::move(Buffer));
205}
206
207OwningBinary<ObjectFile>
208LoadedELFObjectInfo::getObjectForDebug(const ObjectFile &Obj) const {
209 return createELFDebugObject(Obj, L: *this);
210}
211
212} // anonymous namespace
213
214namespace llvm {
215
216RuntimeDyldELF::RuntimeDyldELF(RuntimeDyld::MemoryManager &MemMgr,
217 JITSymbolResolver &Resolver)
218 : RuntimeDyldImpl(MemMgr, Resolver), GOTSectionID(0), CurrentGOTIndex(0) {}
219RuntimeDyldELF::~RuntimeDyldELF() = default;
220
221void RuntimeDyldELF::registerEHFrames() {
222 for (SID EHFrameSID : UnregisteredEHFrameSections) {
223 uint8_t *EHFrameAddr = Sections[EHFrameSID].getAddress();
224 uint64_t EHFrameLoadAddr = Sections[EHFrameSID].getLoadAddress();
225 size_t EHFrameSize = Sections[EHFrameSID].getSize();
226 MemMgr.registerEHFrames(Addr: EHFrameAddr, LoadAddr: EHFrameLoadAddr, Size: EHFrameSize);
227 }
228 UnregisteredEHFrameSections.clear();
229}
230
231std::unique_ptr<RuntimeDyldELF>
232llvm::RuntimeDyldELF::create(Triple::ArchType Arch,
233 RuntimeDyld::MemoryManager &MemMgr,
234 JITSymbolResolver &Resolver) {
235 switch (Arch) {
236 default:
237 return std::make_unique<RuntimeDyldELF>(args&: MemMgr, args&: Resolver);
238 case Triple::mips:
239 case Triple::mipsel:
240 case Triple::mips64:
241 case Triple::mips64el:
242 return std::make_unique<RuntimeDyldELFMips>(args&: MemMgr, args&: Resolver);
243 }
244}
245
246std::unique_ptr<RuntimeDyld::LoadedObjectInfo>
247RuntimeDyldELF::loadObject(const object::ObjectFile &O) {
248 if (auto ObjSectionToIDOrErr = loadObjectImpl(Obj: O))
249 return std::make_unique<LoadedELFObjectInfo>(args&: *this, args&: *ObjSectionToIDOrErr);
250 else {
251 HasError = true;
252 raw_string_ostream ErrStream(ErrorStr);
253 logAllUnhandledErrors(E: ObjSectionToIDOrErr.takeError(), OS&: ErrStream);
254 return nullptr;
255 }
256}
257
258void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section,
259 uint64_t Offset, uint64_t Value,
260 uint32_t Type, int64_t Addend,
261 uint64_t SymOffset) {
262 switch (Type) {
263 default:
264 report_fatal_error(reason: "Relocation type not implemented yet!");
265 break;
266 case ELF::R_X86_64_NONE:
267 break;
268 case ELF::R_X86_64_8: {
269 Value += Addend;
270 assert((int64_t)Value <= INT8_MAX && (int64_t)Value >= INT8_MIN);
271 uint8_t TruncatedAddr = (Value & 0xFF);
272 *Section.getAddressWithOffset(OffsetBytes: Offset) = TruncatedAddr;
273 LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at "
274 << format("%p\n", Section.getAddressWithOffset(Offset)));
275 break;
276 }
277 case ELF::R_X86_64_16: {
278 Value += Addend;
279 assert((int64_t)Value <= INT16_MAX && (int64_t)Value >= INT16_MIN);
280 uint16_t TruncatedAddr = (Value & 0xFFFF);
281 support::ulittle16_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
282 TruncatedAddr;
283 LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at "
284 << format("%p\n", Section.getAddressWithOffset(Offset)));
285 break;
286 }
287 case ELF::R_X86_64_64: {
288 support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
289 Value + Addend;
290 LLVM_DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at "
291 << format("%p\n", Section.getAddressWithOffset(Offset)));
292 break;
293 }
294 case ELF::R_X86_64_32:
295 case ELF::R_X86_64_32S: {
296 Value += Addend;
297 assert((Type == ELF::R_X86_64_32 && (Value <= UINT32_MAX)) ||
298 (Type == ELF::R_X86_64_32S &&
299 ((int64_t)Value <= INT32_MAX && (int64_t)Value >= INT32_MIN)));
300 uint32_t TruncatedAddr = (Value & 0xFFFFFFFF);
301 support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
302 TruncatedAddr;
303 LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at "
304 << format("%p\n", Section.getAddressWithOffset(Offset)));
305 break;
306 }
307 case ELF::R_X86_64_PC8: {
308 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
309 int64_t RealOffset = Value + Addend - FinalAddress;
310 assert(isInt<8>(RealOffset));
311 int8_t TruncOffset = (RealOffset & 0xFF);
312 Section.getAddress()[Offset] = TruncOffset;
313 break;
314 }
315 case ELF::R_X86_64_PC32: {
316 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
317 int64_t RealOffset = Value + Addend - FinalAddress;
318 assert(isInt<32>(RealOffset));
319 int32_t TruncOffset = (RealOffset & 0xFFFFFFFF);
320 support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
321 TruncOffset;
322 break;
323 }
324 case ELF::R_X86_64_PC64: {
325 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
326 int64_t RealOffset = Value + Addend - FinalAddress;
327 support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
328 RealOffset;
329 LLVM_DEBUG(dbgs() << "Writing " << format("%p", RealOffset) << " at "
330 << format("%p\n", FinalAddress));
331 break;
332 }
333 case ELF::R_X86_64_GOTOFF64: {
334 // Compute Value - GOTBase.
335 uint64_t GOTBase = 0;
336 for (const auto &Section : Sections) {
337 if (Section.getName() == ".got") {
338 GOTBase = Section.getLoadAddressWithOffset(OffsetBytes: 0);
339 break;
340 }
341 }
342 assert(GOTBase != 0 && "missing GOT");
343 int64_t GOTOffset = Value - GOTBase + Addend;
344 support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) = GOTOffset;
345 break;
346 }
347 case ELF::R_X86_64_DTPMOD64: {
348 // We only have one DSO, so the module id is always 1.
349 support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) = 1;
350 break;
351 }
352 case ELF::R_X86_64_DTPOFF64:
353 case ELF::R_X86_64_TPOFF64: {
354 // DTPOFF64 should resolve to the offset in the TLS block, TPOFF64 to the
355 // offset in the *initial* TLS block. Since we are statically linking, all
356 // TLS blocks already exist in the initial block, so resolve both
357 // relocations equally.
358 support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
359 Value + Addend;
360 break;
361 }
362 case ELF::R_X86_64_DTPOFF32:
363 case ELF::R_X86_64_TPOFF32: {
364 // As for the (D)TPOFF64 relocations above, both DTPOFF32 and TPOFF32 can
365 // be resolved equally.
366 int64_t RealValue = Value + Addend;
367 assert(RealValue >= INT32_MIN && RealValue <= INT32_MAX);
368 int32_t TruncValue = RealValue;
369 support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
370 TruncValue;
371 break;
372 }
373 }
374}
375
376void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section,
377 uint64_t Offset, uint32_t Value,
378 uint32_t Type, int32_t Addend) {
379 switch (Type) {
380 case ELF::R_386_32: {
381 support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
382 Value + Addend;
383 break;
384 }
385 // Handle R_386_PLT32 like R_386_PC32 since it should be able to
386 // reach any 32 bit address.
387 case ELF::R_386_PLT32:
388 case ELF::R_386_PC32: {
389 uint32_t FinalAddress =
390 Section.getLoadAddressWithOffset(OffsetBytes: Offset) & 0xFFFFFFFF;
391 uint32_t RealOffset = Value + Addend - FinalAddress;
392 support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
393 RealOffset;
394 break;
395 }
396 default:
397 // There are other relocation types, but it appears these are the
398 // only ones currently used by the LLVM ELF object writer
399 report_fatal_error(reason: "Relocation type not implemented yet!");
400 break;
401 }
402}
403
404void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section,
405 uint64_t Offset, uint64_t Value,
406 uint32_t Type, int64_t Addend) {
407 uint32_t *TargetPtr =
408 reinterpret_cast<uint32_t *>(Section.getAddressWithOffset(OffsetBytes: Offset));
409 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
410 // Data should use target endian. Code should always use little endian.
411 bool isBE = Arch == Triple::aarch64_be;
412
413 LLVM_DEBUG(dbgs() << "resolveAArch64Relocation, LocalAddress: 0x"
414 << format("%llx", Section.getAddressWithOffset(Offset))
415 << " FinalAddress: 0x" << format("%llx", FinalAddress)
416 << " Value: 0x" << format("%llx", Value) << " Type: 0x"
417 << format("%x", Type) << " Addend: 0x"
418 << format("%llx", Addend) << "\n");
419
420 switch (Type) {
421 default:
422 report_fatal_error(reason: "Relocation type not implemented yet!");
423 break;
424 case ELF::R_AARCH64_NONE:
425 break;
426 case ELF::R_AARCH64_ABS16: {
427 uint64_t Result = Value + Addend;
428 assert(Result == static_cast<uint64_t>(llvm::SignExtend64(Result, 16)) ||
429 (Result >> 16) == 0);
430 write(isBE, P: TargetPtr, V: static_cast<uint16_t>(Result & 0xffffU));
431 break;
432 }
433 case ELF::R_AARCH64_ABS32: {
434 uint64_t Result = Value + Addend;
435 assert(Result == static_cast<uint64_t>(llvm::SignExtend64(Result, 32)) ||
436 (Result >> 32) == 0);
437 write(isBE, P: TargetPtr, V: static_cast<uint32_t>(Result & 0xffffffffU));
438 break;
439 }
440 case ELF::R_AARCH64_ABS64:
441 write(isBE, P: TargetPtr, V: Value + Addend);
442 break;
443 case ELF::R_AARCH64_PLT32: {
444 uint64_t Result = Value + Addend - FinalAddress;
445 assert(static_cast<int64_t>(Result) >= INT32_MIN &&
446 static_cast<int64_t>(Result) <= INT32_MAX);
447 write(isBE, P: TargetPtr, V: static_cast<uint32_t>(Result));
448 break;
449 }
450 case ELF::R_AARCH64_PREL16: {
451 uint64_t Result = Value + Addend - FinalAddress;
452 assert(static_cast<int64_t>(Result) >= INT16_MIN &&
453 static_cast<int64_t>(Result) <= UINT16_MAX);
454 write(isBE, P: TargetPtr, V: static_cast<uint16_t>(Result & 0xffffU));
455 break;
456 }
457 case ELF::R_AARCH64_PREL32: {
458 uint64_t Result = Value + Addend - FinalAddress;
459 assert(static_cast<int64_t>(Result) >= INT32_MIN &&
460 static_cast<int64_t>(Result) <= UINT32_MAX);
461 write(isBE, P: TargetPtr, V: static_cast<uint32_t>(Result & 0xffffffffU));
462 break;
463 }
464 case ELF::R_AARCH64_PREL64:
465 write(isBE, P: TargetPtr, V: Value + Addend - FinalAddress);
466 break;
467 case ELF::R_AARCH64_CONDBR19: {
468 uint64_t BranchImm = Value + Addend - FinalAddress;
469
470 assert(isInt<21>(BranchImm));
471 *TargetPtr &= 0xff00001fU;
472 // Immediate:20:2 goes in bits 23:5 of Bcc, CBZ, CBNZ
473 or32le(P: TargetPtr, V: (BranchImm & 0x001FFFFC) << 3);
474 break;
475 }
476 case ELF::R_AARCH64_TSTBR14: {
477 uint64_t BranchImm = Value + Addend - FinalAddress;
478
479 assert(isInt<16>(BranchImm));
480
481 uint32_t RawInstr = *(support::little32_t *)TargetPtr;
482 *(support::little32_t *)TargetPtr = RawInstr & 0xfff8001fU;
483
484 // Immediate:15:2 goes in bits 18:5 of TBZ, TBNZ
485 or32le(P: TargetPtr, V: (BranchImm & 0x0000FFFC) << 3);
486 break;
487 }
488 case ELF::R_AARCH64_CALL26: // fallthrough
489 case ELF::R_AARCH64_JUMP26: {
490 // Operation: S+A-P. Set Call or B immediate value to bits fff_fffc of the
491 // calculation.
492 uint64_t BranchImm = Value + Addend - FinalAddress;
493
494 // "Check that -2^27 <= result < 2^27".
495 assert(isInt<28>(BranchImm));
496 or32le(P: TargetPtr, V: (BranchImm & 0x0FFFFFFC) >> 2);
497 break;
498 }
499 case ELF::R_AARCH64_MOVW_UABS_G3:
500 or32le(P: TargetPtr, V: ((Value + Addend) & 0xFFFF000000000000) >> 43);
501 break;
502 case ELF::R_AARCH64_MOVW_UABS_G2_NC:
503 or32le(P: TargetPtr, V: ((Value + Addend) & 0xFFFF00000000) >> 27);
504 break;
505 case ELF::R_AARCH64_MOVW_UABS_G1_NC:
506 or32le(P: TargetPtr, V: ((Value + Addend) & 0xFFFF0000) >> 11);
507 break;
508 case ELF::R_AARCH64_MOVW_UABS_G0_NC:
509 or32le(P: TargetPtr, V: ((Value + Addend) & 0xFFFF) << 5);
510 break;
511 case ELF::R_AARCH64_ADR_PREL_PG_HI21: {
512 // Operation: Page(S+A) - Page(P)
513 uint64_t Result =
514 ((Value + Addend) & ~0xfffULL) - (FinalAddress & ~0xfffULL);
515
516 // Check that -2^32 <= X < 2^32
517 assert(isInt<33>(Result) && "overflow check failed for relocation");
518
519 // Immediate goes in bits 30:29 + 5:23 of ADRP instruction, taken
520 // from bits 32:12 of X.
521 write32AArch64Addr(L: TargetPtr, Imm: Result >> 12);
522 break;
523 }
524 case ELF::R_AARCH64_ADD_ABS_LO12_NC:
525 // Operation: S + A
526 // Immediate goes in bits 21:10 of LD/ST instruction, taken
527 // from bits 11:0 of X
528 or32AArch64Imm(L: TargetPtr, Imm: Value + Addend);
529 break;
530 case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
531 // Operation: S + A
532 // Immediate goes in bits 21:10 of LD/ST instruction, taken
533 // from bits 11:0 of X
534 or32AArch64Imm(L: TargetPtr, Imm: getBits(Val: Value + Addend, Start: 0, End: 11));
535 break;
536 case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
537 // Operation: S + A
538 // Immediate goes in bits 21:10 of LD/ST instruction, taken
539 // from bits 11:1 of X
540 or32AArch64Imm(L: TargetPtr, Imm: getBits(Val: Value + Addend, Start: 1, End: 11));
541 break;
542 case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
543 // Operation: S + A
544 // Immediate goes in bits 21:10 of LD/ST instruction, taken
545 // from bits 11:2 of X
546 or32AArch64Imm(L: TargetPtr, Imm: getBits(Val: Value + Addend, Start: 2, End: 11));
547 break;
548 case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
549 // Operation: S + A
550 // Immediate goes in bits 21:10 of LD/ST instruction, taken
551 // from bits 11:3 of X
552 or32AArch64Imm(L: TargetPtr, Imm: getBits(Val: Value + Addend, Start: 3, End: 11));
553 break;
554 case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
555 // Operation: S + A
556 // Immediate goes in bits 21:10 of LD/ST instruction, taken
557 // from bits 11:4 of X
558 or32AArch64Imm(L: TargetPtr, Imm: getBits(Val: Value + Addend, Start: 4, End: 11));
559 break;
560 case ELF::R_AARCH64_LD_PREL_LO19: {
561 // Operation: S + A - P
562 uint64_t Result = Value + Addend - FinalAddress;
563
564 // "Check that -2^20 <= result < 2^20".
565 assert(isInt<21>(Result));
566
567 *TargetPtr &= 0xff00001fU;
568 // Immediate goes in bits 23:5 of LD imm instruction, taken
569 // from bits 20:2 of X
570 *TargetPtr |= ((Result & 0xffc) << (5 - 2));
571 break;
572 }
573 case ELF::R_AARCH64_ADR_PREL_LO21: {
574 // Operation: S + A - P
575 uint64_t Result = Value + Addend - FinalAddress;
576
577 // "Check that -2^20 <= result < 2^20".
578 assert(isInt<21>(Result));
579
580 *TargetPtr &= 0x9f00001fU;
581 // Immediate goes in bits 23:5, 30:29 of ADR imm instruction, taken
582 // from bits 20:0 of X
583 *TargetPtr |= ((Result & 0xffc) << (5 - 2));
584 *TargetPtr |= (Result & 0x3) << 29;
585 break;
586 }
587 }
588}
589
590void RuntimeDyldELF::resolveARMRelocation(const SectionEntry &Section,
591 uint64_t Offset, uint32_t Value,
592 uint32_t Type, int32_t Addend) {
593 // TODO: Add Thumb relocations.
594 uint32_t *TargetPtr =
595 reinterpret_cast<uint32_t *>(Section.getAddressWithOffset(OffsetBytes: Offset));
596 uint32_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset) & 0xFFFFFFFF;
597 Value += Addend;
598
599 LLVM_DEBUG(dbgs() << "resolveARMRelocation, LocalAddress: "
600 << Section.getAddressWithOffset(Offset)
601 << " FinalAddress: " << format("%p", FinalAddress)
602 << " Value: " << format("%x", Value)
603 << " Type: " << format("%x", Type)
604 << " Addend: " << format("%x", Addend) << "\n");
605
606 switch (Type) {
607 default:
608 llvm_unreachable("Not implemented relocation type!");
609
610 case ELF::R_ARM_NONE:
611 break;
612 // Write a 31bit signed offset
613 case ELF::R_ARM_PREL31:
614 support::ulittle32_t::ref{TargetPtr} =
615 (support::ulittle32_t::ref{TargetPtr} & 0x80000000) |
616 ((Value - FinalAddress) & ~0x80000000);
617 break;
618 case ELF::R_ARM_TARGET1:
619 case ELF::R_ARM_ABS32:
620 support::ulittle32_t::ref{TargetPtr} = Value;
621 break;
622 // Write first 16 bit of 32 bit value to the mov instruction.
623 // Last 4 bit should be shifted.
624 case ELF::R_ARM_MOVW_ABS_NC:
625 case ELF::R_ARM_MOVT_ABS:
626 if (Type == ELF::R_ARM_MOVW_ABS_NC)
627 Value = Value & 0xFFFF;
628 else if (Type == ELF::R_ARM_MOVT_ABS)
629 Value = (Value >> 16) & 0xFFFF;
630 support::ulittle32_t::ref{TargetPtr} =
631 (support::ulittle32_t::ref{TargetPtr} & ~0x000F0FFF) | (Value & 0xFFF) |
632 (((Value >> 12) & 0xF) << 16);
633 break;
634 // Write 24 bit relative value to the branch instruction.
635 case ELF::R_ARM_PC24: // Fall through.
636 case ELF::R_ARM_CALL: // Fall through.
637 case ELF::R_ARM_JUMP24:
638 int32_t RelValue = static_cast<int32_t>(Value - FinalAddress - 8);
639 RelValue = (RelValue & 0x03FFFFFC) >> 2;
640 assert((support::ulittle32_t::ref{TargetPtr} & 0xFFFFFF) == 0xFFFFFE);
641 support::ulittle32_t::ref{TargetPtr} =
642 (support::ulittle32_t::ref{TargetPtr} & 0xFF000000) | RelValue;
643 break;
644 }
645}
646
647bool RuntimeDyldELF::resolveLoongArch64ShortBranch(
648 unsigned SectionID, relocation_iterator RelI,
649 const RelocationValueRef &Value) {
650 uint64_t Address;
651 if (Value.SymbolName) {
652 auto Loc = GlobalSymbolTable.find(Key: Value.SymbolName);
653 // Don't create direct branch for external symbols.
654 if (Loc == GlobalSymbolTable.end())
655 return false;
656 const auto &SymInfo = Loc->second;
657 Address = Sections[SymInfo.getSectionID()].getLoadAddressWithOffset(
658 OffsetBytes: SymInfo.getOffset());
659 } else {
660 Address = Sections[Value.SectionID].getLoadAddress();
661 }
662 uint64_t Offset = RelI->getOffset();
663 uint64_t SourceAddress = Sections[SectionID].getLoadAddressWithOffset(OffsetBytes: Offset);
664 uint64_t Delta = Address + Value.Addend - SourceAddress;
665 // Normal call
666 if (RelI->getType() == ELF::R_LARCH_B26) {
667 if (!isInt<28>(x: Delta))
668 return false;
669 resolveRelocation(Section: Sections[SectionID], Offset, Value: Address, Type: RelI->getType(),
670 Addend: Value.Addend);
671 return true;
672 }
673 // Medium call: R_LARCH_CALL36
674 // Range: [-128G - 0x20000, +128G - 0x20000)
675 if (((int64_t)Delta + 0x20000) != llvm::SignExtend64(X: Delta + 0x20000, B: 38))
676 return false;
677 resolveRelocation(Section: Sections[SectionID], Offset, Value: Address, Type: RelI->getType(),
678 Addend: Value.Addend);
679 return true;
680}
681
682void RuntimeDyldELF::resolveLoongArch64Branch(unsigned SectionID,
683 const RelocationValueRef &Value,
684 relocation_iterator RelI,
685 StubMap &Stubs) {
686 LLVM_DEBUG(dbgs() << "\t\tThis is an LoongArch64 branch relocation.\n");
687
688 if (resolveLoongArch64ShortBranch(SectionID, RelI, Value))
689 return;
690
691 SectionEntry &Section = Sections[SectionID];
692 uint64_t Offset = RelI->getOffset();
693 unsigned RelType = RelI->getType();
694 // Look for an existing stub.
695 auto [It, Inserted] = Stubs.try_emplace(k: Value);
696 if (!Inserted) {
697 resolveRelocation(Section, Offset,
698 Value: (uint64_t)Section.getAddressWithOffset(OffsetBytes: It->second),
699 Type: RelType, Addend: 0);
700 LLVM_DEBUG(dbgs() << " Stub function found\n");
701 return;
702 }
703 // Create a new stub function.
704 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
705 It->second = Section.getStubOffset();
706 uint8_t *StubTargetAddr =
707 createStubFunction(Addr: Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset()));
708 RelocationEntry LU12I_W(SectionID, StubTargetAddr - Section.getAddress(),
709 ELF::R_LARCH_ABS_HI20, Value.Addend);
710 RelocationEntry ORI(SectionID, StubTargetAddr - Section.getAddress() + 4,
711 ELF::R_LARCH_ABS_LO12, Value.Addend);
712 RelocationEntry LU32I_D(SectionID, StubTargetAddr - Section.getAddress() + 8,
713 ELF::R_LARCH_ABS64_LO20, Value.Addend);
714 RelocationEntry LU52I_D(SectionID, StubTargetAddr - Section.getAddress() + 12,
715 ELF::R_LARCH_ABS64_HI12, Value.Addend);
716 if (Value.SymbolName) {
717 addRelocationForSymbol(RE: LU12I_W, SymbolName: Value.SymbolName);
718 addRelocationForSymbol(RE: ORI, SymbolName: Value.SymbolName);
719 addRelocationForSymbol(RE: LU32I_D, SymbolName: Value.SymbolName);
720 addRelocationForSymbol(RE: LU52I_D, SymbolName: Value.SymbolName);
721 } else {
722 addRelocationForSection(RE: LU12I_W, SectionID: Value.SectionID);
723 addRelocationForSection(RE: ORI, SectionID: Value.SectionID);
724 addRelocationForSection(RE: LU32I_D, SectionID: Value.SectionID);
725
726 addRelocationForSection(RE: LU52I_D, SectionID: Value.SectionID);
727 }
728 resolveRelocation(Section, Offset,
729 Value: reinterpret_cast<uint64_t>(
730 Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset())),
731 Type: RelType, Addend: 0);
732 Section.advanceStubOffset(StubSize: getMaxStubSize());
733}
734
735// Returns extract bits Val[Hi:Lo].
736static inline uint32_t extractBits(uint64_t Val, uint32_t Hi, uint32_t Lo) {
737 return Hi == 63 ? Val >> Lo : (Val & (((1ULL << (Hi + 1)) - 1))) >> Lo;
738}
739
740// Calculate the adjusted page delta between dest and PC. The code is copied
741// from lld and see comments there for more details.
742static uint64_t getLoongArchPageDelta(uint64_t dest, uint64_t pc,
743 uint32_t type) {
744 uint64_t pcalau12i_pc;
745 switch (type) {
746 case ELF::R_LARCH_PCALA64_LO20:
747 case ELF::R_LARCH_GOT64_PC_LO20:
748 pcalau12i_pc = pc - 8;
749 break;
750 case ELF::R_LARCH_PCALA64_HI12:
751 case ELF::R_LARCH_GOT64_PC_HI12:
752 pcalau12i_pc = pc - 12;
753 break;
754 default:
755 pcalau12i_pc = pc;
756 break;
757 }
758 uint64_t result = (dest & ~0xfffULL) - (pcalau12i_pc & ~0xfffULL);
759 if (dest & 0x800)
760 result += 0x1000 - 0x1'0000'0000;
761 if (result & 0x8000'0000)
762 result += 0x1'0000'0000;
763 return result;
764}
765
766void RuntimeDyldELF::resolveLoongArch64Relocation(const SectionEntry &Section,
767 uint64_t Offset,
768 uint64_t Value, uint32_t Type,
769 int64_t Addend) {
770 auto *TargetPtr = Section.getAddressWithOffset(OffsetBytes: Offset);
771 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
772
773 LLVM_DEBUG(dbgs() << "resolveLoongArch64Relocation, LocalAddress: 0x"
774 << format("%llx", Section.getAddressWithOffset(Offset))
775 << " FinalAddress: 0x" << format("%llx", FinalAddress)
776 << " Value: 0x" << format("%llx", Value) << " Type: 0x"
777 << format("%x", Type) << " Addend: 0x"
778 << format("%llx", Addend) << "\n");
779
780 switch (Type) {
781 default:
782 report_fatal_error(reason: "Relocation type not implemented yet!");
783 break;
784 case ELF::R_LARCH_MARK_LA:
785 // ignore
786 break;
787 case ELF::R_LARCH_32:
788 support::ulittle32_t::ref{TargetPtr} =
789 static_cast<uint32_t>(Value + Addend);
790 break;
791 case ELF::R_LARCH_64:
792 support::ulittle64_t::ref{TargetPtr} = Value + Addend;
793 break;
794 case ELF::R_LARCH_32_PCREL:
795 support::ulittle32_t::ref{TargetPtr} =
796 static_cast<uint32_t>(Value + Addend - FinalAddress);
797 break;
798 case ELF::R_LARCH_B26: {
799 uint64_t B26 = (Value + Addend - FinalAddress) >> 2;
800 auto Instr = support::ulittle32_t::ref(TargetPtr);
801 uint32_t Imm15_0 = extractBits(Val: B26, /*Hi=*/15, /*Lo=*/0) << 10;
802 uint32_t Imm25_16 = extractBits(Val: B26, /*Hi=*/25, /*Lo=*/16);
803 Instr = (Instr & 0xfc000000) | Imm15_0 | Imm25_16;
804 break;
805 }
806 case ELF::R_LARCH_CALL36: {
807 uint64_t Call36 = (Value + Addend - FinalAddress) >> 2;
808 auto Pcaddu18i = support::ulittle32_t::ref(TargetPtr);
809 uint32_t Imm35_16 =
810 extractBits(Val: (Call36 + (1UL << 15)), /*Hi=*/35, /*Lo=*/16) << 5;
811 Pcaddu18i = (Pcaddu18i & 0xfe00001f) | Imm35_16;
812 auto Jirl = support::ulittle32_t::ref(TargetPtr + 4);
813 uint32_t Imm15_0 = extractBits(Val: Call36, /*Hi=*/15, /*Lo=*/0) << 10;
814 Jirl = (Jirl & 0xfc0003ff) | Imm15_0;
815 break;
816 }
817 case ELF::R_LARCH_GOT_PC_HI20:
818 case ELF::R_LARCH_PCALA_HI20: {
819 uint64_t Target = Value + Addend;
820 int64_t PageDelta = getLoongArchPageDelta(dest: Target, pc: FinalAddress, type: Type);
821 auto Instr = support::ulittle32_t::ref(TargetPtr);
822 uint32_t Imm31_12 = extractBits(Val: PageDelta, /*Hi=*/31, /*Lo=*/12) << 5;
823 Instr = (Instr & 0xfe00001f) | Imm31_12;
824 break;
825 }
826 case ELF::R_LARCH_GOT_PC_LO12:
827 case ELF::R_LARCH_PCALA_LO12: {
828 uint64_t TargetOffset = (Value + Addend) & 0xfff;
829 auto Instr = support::ulittle32_t::ref(TargetPtr);
830 uint32_t Imm11_0 = TargetOffset << 10;
831 Instr = (Instr & 0xffc003ff) | Imm11_0;
832 break;
833 }
834 case ELF::R_LARCH_GOT64_PC_LO20:
835 case ELF::R_LARCH_PCALA64_LO20: {
836 uint64_t Target = Value + Addend;
837 int64_t PageDelta = getLoongArchPageDelta(dest: Target, pc: FinalAddress, type: Type);
838 auto Instr = support::ulittle32_t::ref(TargetPtr);
839 uint32_t Imm51_32 = extractBits(Val: PageDelta, /*Hi=*/51, /*Lo=*/32) << 5;
840 Instr = (Instr & 0xfe00001f) | Imm51_32;
841 break;
842 }
843 case ELF::R_LARCH_GOT64_PC_HI12:
844 case ELF::R_LARCH_PCALA64_HI12: {
845 uint64_t Target = Value + Addend;
846 int64_t PageDelta = getLoongArchPageDelta(dest: Target, pc: FinalAddress, type: Type);
847 auto Instr = support::ulittle32_t::ref(TargetPtr);
848 uint32_t Imm63_52 = extractBits(Val: PageDelta, /*Hi=*/63, /*Lo=*/52) << 10;
849 Instr = (Instr & 0xffc003ff) | Imm63_52;
850 break;
851 }
852 case ELF::R_LARCH_ABS_HI20: {
853 uint64_t Target = Value + Addend;
854 auto Instr = support::ulittle32_t::ref(TargetPtr);
855 uint32_t Imm31_12 = extractBits(Val: Target, /*Hi=*/31, /*Lo=*/12) << 5;
856 Instr = (Instr & 0xfe00001f) | Imm31_12;
857 break;
858 }
859 case ELF::R_LARCH_ABS_LO12: {
860 uint64_t Target = Value + Addend;
861 auto Instr = support::ulittle32_t::ref(TargetPtr);
862 uint32_t Imm11_0 = extractBits(Val: Target, /*Hi=*/11, /*Lo=*/0) << 10;
863 Instr = (Instr & 0xffc003ff) | Imm11_0;
864 break;
865 }
866 case ELF::R_LARCH_ABS64_LO20: {
867 uint64_t Target = Value + Addend;
868 auto Instr = support::ulittle32_t::ref(TargetPtr);
869 uint32_t Imm51_32 = extractBits(Val: Target, /*Hi=*/51, /*Lo=*/32) << 5;
870 Instr = (Instr & 0xfe00001f) | Imm51_32;
871 break;
872 }
873 case ELF::R_LARCH_ABS64_HI12: {
874 uint64_t Target = Value + Addend;
875 auto Instr = support::ulittle32_t::ref(TargetPtr);
876 uint32_t Imm63_52 = extractBits(Val: Target, /*Hi=*/63, /*Lo=*/52) << 10;
877 Instr = (Instr & 0xffc003ff) | Imm63_52;
878 break;
879 }
880 case ELF::R_LARCH_ADD32:
881 support::ulittle32_t::ref{TargetPtr} =
882 (support::ulittle32_t::ref{TargetPtr} +
883 static_cast<uint32_t>(Value + Addend));
884 break;
885 case ELF::R_LARCH_SUB32:
886 support::ulittle32_t::ref{TargetPtr} =
887 (support::ulittle32_t::ref{TargetPtr} -
888 static_cast<uint32_t>(Value + Addend));
889 break;
890 case ELF::R_LARCH_ADD64:
891 support::ulittle64_t::ref{TargetPtr} =
892 (support::ulittle64_t::ref{TargetPtr} + Value + Addend);
893 break;
894 case ELF::R_LARCH_SUB64:
895 support::ulittle64_t::ref{TargetPtr} =
896 (support::ulittle64_t::ref{TargetPtr} - Value - Addend);
897 break;
898 }
899}
900
901void RuntimeDyldELF::setMipsABI(const ObjectFile &Obj) {
902 if (Arch == Triple::UnknownArch ||
903 Triple::getArchTypePrefix(Kind: Arch) != "mips") {
904 IsMipsO32ABI = false;
905 IsMipsN32ABI = false;
906 IsMipsN64ABI = false;
907 return;
908 }
909 if (auto *E = dyn_cast<ELFObjectFileBase>(Val: &Obj)) {
910 unsigned AbiVariant = E->getPlatformFlags();
911 IsMipsO32ABI = AbiVariant & ELF::EF_MIPS_ABI_O32;
912 IsMipsN32ABI = AbiVariant & ELF::EF_MIPS_ABI2;
913 }
914 IsMipsN64ABI = Obj.getFileFormatName() == "elf64-mips";
915}
916
917// Return the .TOC. section and offset.
918Error RuntimeDyldELF::findPPC64TOCSection(const ELFObjectFileBase &Obj,
919 ObjSectionToIDMap &LocalSections,
920 RelocationValueRef &Rel) {
921 // Set a default SectionID in case we do not find a TOC section below.
922 // This may happen for references to TOC base base (sym@toc, .odp
923 // relocation) without a .toc directive. In this case just use the
924 // first section (which is usually the .odp) since the code won't
925 // reference the .toc base directly.
926 Rel.SymbolName = nullptr;
927 Rel.SectionID = 0;
928
929 // The TOC consists of sections .got, .toc, .tocbss, .plt in that
930 // order. The TOC starts where the first of these sections starts.
931 for (auto &Section : Obj.sections()) {
932 Expected<StringRef> NameOrErr = Section.getName();
933 if (!NameOrErr)
934 return NameOrErr.takeError();
935 StringRef SectionName = *NameOrErr;
936
937 if (SectionName == ".got"
938 || SectionName == ".toc"
939 || SectionName == ".tocbss"
940 || SectionName == ".plt") {
941 if (auto SectionIDOrErr =
942 findOrEmitSection(Obj, Section, IsCode: false, LocalSections))
943 Rel.SectionID = *SectionIDOrErr;
944 else
945 return SectionIDOrErr.takeError();
946 break;
947 }
948 }
949
950 // Per the ppc64-elf-linux ABI, The TOC base is TOC value plus 0x8000
951 // thus permitting a full 64 Kbytes segment.
952 Rel.Addend = 0x8000;
953
954 return Error::success();
955}
956
957// Returns the sections and offset associated with the ODP entry referenced
958// by Symbol.
959Error RuntimeDyldELF::findOPDEntrySection(const ELFObjectFileBase &Obj,
960 ObjSectionToIDMap &LocalSections,
961 RelocationValueRef &Rel) {
962 // Get the ELF symbol value (st_value) to compare with Relocation offset in
963 // .opd entries
964 for (section_iterator si = Obj.section_begin(), se = Obj.section_end();
965 si != se; ++si) {
966
967 Expected<section_iterator> RelSecOrErr = si->getRelocatedSection();
968 if (!RelSecOrErr)
969 report_fatal_error(reason: Twine(toString(E: RelSecOrErr.takeError())));
970
971 section_iterator RelSecI = *RelSecOrErr;
972 if (RelSecI == Obj.section_end())
973 continue;
974
975 Expected<StringRef> NameOrErr = RelSecI->getName();
976 if (!NameOrErr)
977 return NameOrErr.takeError();
978 StringRef RelSectionName = *NameOrErr;
979
980 if (RelSectionName != ".opd")
981 continue;
982
983 for (elf_relocation_iterator i = si->relocation_begin(),
984 e = si->relocation_end();
985 i != e;) {
986 // The R_PPC64_ADDR64 relocation indicates the first field
987 // of a .opd entry
988 uint64_t TypeFunc = i->getType();
989 if (TypeFunc != ELF::R_PPC64_ADDR64) {
990 ++i;
991 continue;
992 }
993
994 uint64_t TargetSymbolOffset = i->getOffset();
995 symbol_iterator TargetSymbol = i->getSymbol();
996 int64_t Addend;
997 if (auto AddendOrErr = i->getAddend())
998 Addend = *AddendOrErr;
999 else
1000 return AddendOrErr.takeError();
1001
1002 ++i;
1003 if (i == e)
1004 break;
1005
1006 // Just check if following relocation is a R_PPC64_TOC
1007 uint64_t TypeTOC = i->getType();
1008 if (TypeTOC != ELF::R_PPC64_TOC)
1009 continue;
1010
1011 // Finally compares the Symbol value and the target symbol offset
1012 // to check if this .opd entry refers to the symbol the relocation
1013 // points to.
1014 if (Rel.Addend != (int64_t)TargetSymbolOffset)
1015 continue;
1016
1017 section_iterator TSI = Obj.section_end();
1018 if (auto TSIOrErr = TargetSymbol->getSection())
1019 TSI = *TSIOrErr;
1020 else
1021 return TSIOrErr.takeError();
1022 assert(TSI != Obj.section_end() && "TSI should refer to a valid section");
1023
1024 bool IsCode = TSI->isText();
1025 if (auto SectionIDOrErr = findOrEmitSection(Obj, Section: *TSI, IsCode,
1026 LocalSections))
1027 Rel.SectionID = *SectionIDOrErr;
1028 else
1029 return SectionIDOrErr.takeError();
1030 Rel.Addend = (intptr_t)Addend;
1031 return Error::success();
1032 }
1033 }
1034 llvm_unreachable("Attempting to get address of ODP entry!");
1035}
1036
1037// Relocation masks following the #lo(value), #hi(value), #ha(value),
1038// #higher(value), #highera(value), #highest(value), and #highesta(value)
1039// macros defined in section 4.5.1. Relocation Types of the PPC-elf64abi
1040// document.
1041
1042static inline uint16_t applyPPClo(uint64_t value) { return value & 0xffff; }
1043
1044static inline uint16_t applyPPChi(uint64_t value) {
1045 return (value >> 16) & 0xffff;
1046}
1047
1048static inline uint16_t applyPPCha (uint64_t value) {
1049 return ((value + 0x8000) >> 16) & 0xffff;
1050}
1051
1052static inline uint16_t applyPPChigher(uint64_t value) {
1053 return (value >> 32) & 0xffff;
1054}
1055
1056static inline uint16_t applyPPChighera (uint64_t value) {
1057 return ((value + 0x8000) >> 32) & 0xffff;
1058}
1059
1060static inline uint16_t applyPPChighest(uint64_t value) {
1061 return (value >> 48) & 0xffff;
1062}
1063
1064static inline uint16_t applyPPChighesta (uint64_t value) {
1065 return ((value + 0x8000) >> 48) & 0xffff;
1066}
1067
1068void RuntimeDyldELF::resolvePPC32Relocation(const SectionEntry &Section,
1069 uint64_t Offset, uint64_t Value,
1070 uint32_t Type, int64_t Addend) {
1071 uint8_t *LocalAddress = Section.getAddressWithOffset(OffsetBytes: Offset);
1072 uint32_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1073 switch (Type) {
1074 default:
1075 report_fatal_error(reason: "Relocation type not implemented yet!");
1076 break;
1077 case ELF::R_PPC_ADDR16_LO:
1078 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Value + Addend));
1079 break;
1080 case ELF::R_PPC_ADDR16_HI:
1081 writeInt16BE(Addr: LocalAddress, Value: applyPPChi(value: Value + Addend));
1082 break;
1083 case ELF::R_PPC_ADDR16_HA:
1084 writeInt16BE(Addr: LocalAddress, Value: applyPPCha(value: Value + Addend));
1085 break;
1086 case ELF::R_PPC_ADDR32:
1087 writeInt32BE(Addr: LocalAddress, Value: Value + Addend);
1088 break;
1089 case ELF::R_PPC_REL16_LO:
1090 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Value - FinalAddress + Addend));
1091 break;
1092 case ELF::R_PPC_REL16_HA:
1093 writeInt16BE(Addr: LocalAddress, Value: applyPPCha(value: Value - FinalAddress + Addend));
1094 break;
1095 case ELF::R_PPC_REL24: {
1096 int32_t Delta = static_cast<int32_t>(Value - FinalAddress + Addend);
1097 if (SignExtend32<26>(X: Delta) != Delta)
1098 report_fatal_error(reason: "Relocation R_PPC_REL24 overflow");
1099 // Keep the opcode and the AA and LK bits.
1100 uint32_t Inst = readBytesUnaligned(Src: LocalAddress, Size: 4);
1101 writeInt32BE(Addr: LocalAddress, Value: (Inst & 0xFC000003) | (Delta & 0x03FFFFFC));
1102 } break;
1103 case ELF::R_PPC_REL32:
1104 writeInt32BE(Addr: LocalAddress, Value: Value - FinalAddress + Addend);
1105 break;
1106 }
1107}
1108
1109void RuntimeDyldELF::resolvePPC64Relocation(const SectionEntry &Section,
1110 uint64_t Offset, uint64_t Value,
1111 uint32_t Type, int64_t Addend) {
1112 uint8_t *LocalAddress = Section.getAddressWithOffset(OffsetBytes: Offset);
1113 switch (Type) {
1114 default:
1115 report_fatal_error(reason: "Relocation type not implemented yet!");
1116 break;
1117 case ELF::R_PPC64_ADDR16:
1118 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Value + Addend));
1119 break;
1120 case ELF::R_PPC64_ADDR16_DS:
1121 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Value + Addend) & ~3);
1122 break;
1123 case ELF::R_PPC64_ADDR16_LO:
1124 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Value + Addend));
1125 break;
1126 case ELF::R_PPC64_ADDR16_LO_DS:
1127 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Value + Addend) & ~3);
1128 break;
1129 case ELF::R_PPC64_ADDR16_HI:
1130 case ELF::R_PPC64_ADDR16_HIGH:
1131 writeInt16BE(Addr: LocalAddress, Value: applyPPChi(value: Value + Addend));
1132 break;
1133 case ELF::R_PPC64_ADDR16_HA:
1134 case ELF::R_PPC64_ADDR16_HIGHA:
1135 writeInt16BE(Addr: LocalAddress, Value: applyPPCha(value: Value + Addend));
1136 break;
1137 case ELF::R_PPC64_ADDR16_HIGHER:
1138 writeInt16BE(Addr: LocalAddress, Value: applyPPChigher(value: Value + Addend));
1139 break;
1140 case ELF::R_PPC64_ADDR16_HIGHERA:
1141 writeInt16BE(Addr: LocalAddress, Value: applyPPChighera(value: Value + Addend));
1142 break;
1143 case ELF::R_PPC64_ADDR16_HIGHEST:
1144 writeInt16BE(Addr: LocalAddress, Value: applyPPChighest(value: Value + Addend));
1145 break;
1146 case ELF::R_PPC64_ADDR16_HIGHESTA:
1147 writeInt16BE(Addr: LocalAddress, Value: applyPPChighesta(value: Value + Addend));
1148 break;
1149 case ELF::R_PPC64_ADDR14: {
1150 assert(((Value + Addend) & 3) == 0);
1151 // Preserve the AA/LK bits in the branch instruction
1152 uint8_t aalk = *(LocalAddress + 3);
1153 writeInt16BE(Addr: LocalAddress + 2, Value: (aalk & 3) | ((Value + Addend) & 0xfffc));
1154 } break;
1155 case ELF::R_PPC64_REL16_LO: {
1156 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1157 uint64_t Delta = Value - FinalAddress + Addend;
1158 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Delta));
1159 } break;
1160 case ELF::R_PPC64_REL16_HI: {
1161 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1162 uint64_t Delta = Value - FinalAddress + Addend;
1163 writeInt16BE(Addr: LocalAddress, Value: applyPPChi(value: Delta));
1164 } break;
1165 case ELF::R_PPC64_REL16_HA: {
1166 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1167 uint64_t Delta = Value - FinalAddress + Addend;
1168 writeInt16BE(Addr: LocalAddress, Value: applyPPCha(value: Delta));
1169 } break;
1170 case ELF::R_PPC64_ADDR32: {
1171 int64_t Result = static_cast<int64_t>(Value + Addend);
1172 if (SignExtend64<32>(x: Result) != Result)
1173 llvm_unreachable("Relocation R_PPC64_ADDR32 overflow");
1174 writeInt32BE(Addr: LocalAddress, Value: Result);
1175 } break;
1176 case ELF::R_PPC64_REL24: {
1177 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1178 int64_t delta = static_cast<int64_t>(Value - FinalAddress + Addend);
1179 if (SignExtend64<26>(x: delta) != delta)
1180 llvm_unreachable("Relocation R_PPC64_REL24 overflow");
1181 // We preserve bits other than LI field, i.e. PO and AA/LK fields.
1182 uint32_t Inst = readBytesUnaligned(Src: LocalAddress, Size: 4);
1183 writeInt32BE(Addr: LocalAddress, Value: (Inst & 0xFC000003) | (delta & 0x03FFFFFC));
1184 } break;
1185 case ELF::R_PPC64_REL32: {
1186 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1187 int64_t delta = static_cast<int64_t>(Value - FinalAddress + Addend);
1188 if (SignExtend64<32>(x: delta) != delta)
1189 llvm_unreachable("Relocation R_PPC64_REL32 overflow");
1190 writeInt32BE(Addr: LocalAddress, Value: delta);
1191 } break;
1192 case ELF::R_PPC64_REL64: {
1193 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1194 uint64_t Delta = Value - FinalAddress + Addend;
1195 writeInt64BE(Addr: LocalAddress, Value: Delta);
1196 } break;
1197 case ELF::R_PPC64_ADDR64:
1198 writeInt64BE(Addr: LocalAddress, Value: Value + Addend);
1199 break;
1200 }
1201}
1202
1203void RuntimeDyldELF::resolveSystemZRelocation(const SectionEntry &Section,
1204 uint64_t Offset, uint64_t Value,
1205 uint32_t Type, int64_t Addend) {
1206 uint8_t *LocalAddress = Section.getAddressWithOffset(OffsetBytes: Offset);
1207 switch (Type) {
1208 default:
1209 report_fatal_error(reason: "Relocation type not implemented yet!");
1210 break;
1211 case ELF::R_390_PC16DBL:
1212 case ELF::R_390_PLT16DBL: {
1213 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1214 assert(int16_t(Delta / 2) * 2 == Delta && "R_390_PC16DBL overflow");
1215 writeInt16BE(Addr: LocalAddress, Value: Delta / 2);
1216 break;
1217 }
1218 case ELF::R_390_PC32DBL:
1219 case ELF::R_390_PLT32DBL: {
1220 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1221 assert(int32_t(Delta / 2) * 2 == Delta && "R_390_PC32DBL overflow");
1222 writeInt32BE(Addr: LocalAddress, Value: Delta / 2);
1223 break;
1224 }
1225 case ELF::R_390_PC16: {
1226 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1227 assert(int16_t(Delta) == Delta && "R_390_PC16 overflow");
1228 writeInt16BE(Addr: LocalAddress, Value: Delta);
1229 break;
1230 }
1231 case ELF::R_390_PC32: {
1232 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1233 assert(int32_t(Delta) == Delta && "R_390_PC32 overflow");
1234 writeInt32BE(Addr: LocalAddress, Value: Delta);
1235 break;
1236 }
1237 case ELF::R_390_PC64: {
1238 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1239 writeInt64BE(Addr: LocalAddress, Value: Delta);
1240 break;
1241 }
1242 case ELF::R_390_8:
1243 *LocalAddress = (uint8_t)(Value + Addend);
1244 break;
1245 case ELF::R_390_16:
1246 writeInt16BE(Addr: LocalAddress, Value: Value + Addend);
1247 break;
1248 case ELF::R_390_32:
1249 writeInt32BE(Addr: LocalAddress, Value: Value + Addend);
1250 break;
1251 case ELF::R_390_64:
1252 writeInt64BE(Addr: LocalAddress, Value: Value + Addend);
1253 break;
1254 }
1255}
1256
1257void RuntimeDyldELF::resolveBPFRelocation(const SectionEntry &Section,
1258 uint64_t Offset, uint64_t Value,
1259 uint32_t Type, int64_t Addend) {
1260 bool isBE = Arch == Triple::bpfeb;
1261
1262 switch (Type) {
1263 default:
1264 report_fatal_error(reason: "Relocation type not implemented yet!");
1265 break;
1266 case ELF::R_BPF_NONE:
1267 case ELF::R_BPF_64_64:
1268 case ELF::R_BPF_64_32:
1269 case ELF::R_BPF_64_NODYLD32:
1270 break;
1271 case ELF::R_BPF_64_ABS64: {
1272 write(isBE, P: Section.getAddressWithOffset(OffsetBytes: Offset), V: Value + Addend);
1273 LLVM_DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at "
1274 << format("%p\n", Section.getAddressWithOffset(Offset)));
1275 break;
1276 }
1277 case ELF::R_BPF_64_ABS32: {
1278 Value += Addend;
1279 assert(Value <= UINT32_MAX);
1280 write(isBE, P: Section.getAddressWithOffset(OffsetBytes: Offset), V: static_cast<uint32_t>(Value));
1281 LLVM_DEBUG(dbgs() << "Writing " << format("%p", Value) << " at "
1282 << format("%p\n", Section.getAddressWithOffset(Offset)));
1283 break;
1284 }
1285 }
1286}
1287
1288static void applyUTypeImmRISCV(uint8_t *InstrAddr, uint32_t Imm) {
1289 uint32_t UpperImm = (Imm + 0x800) & 0xfffff000;
1290 auto Instr = support::ulittle32_t::ref(InstrAddr);
1291 Instr = (Instr & 0xfff) | UpperImm;
1292}
1293
1294static void applyITypeImmRISCV(uint8_t *InstrAddr, uint32_t Imm) {
1295 uint32_t LowerImm = Imm & 0xfff;
1296 auto Instr = support::ulittle32_t::ref(InstrAddr);
1297 Instr = (Instr & 0xfffff) | (LowerImm << 20);
1298}
1299
1300void RuntimeDyldELF::resolveRISCVRelocation(const SectionEntry &Section,
1301 uint64_t Offset, uint64_t Value,
1302 uint32_t Type, int64_t Addend,
1303 SID SectionID) {
1304 switch (Type) {
1305 default: {
1306 std::string Err = "Unimplemented reloc type: " + std::to_string(val: Type);
1307 llvm::report_fatal_error(reason: Err.c_str());
1308 }
1309 // 32-bit PC-relative function call, macros call, tail (PIC)
1310 // Write first 20 bits of 32 bit value to the auipc instruction
1311 // Last 12 bits to the jalr instruction
1312 case ELF::R_RISCV_CALL:
1313 case ELF::R_RISCV_CALL_PLT: {
1314 uint64_t P = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1315 uint64_t PCOffset = Value + Addend - P;
1316 applyUTypeImmRISCV(InstrAddr: Section.getAddressWithOffset(OffsetBytes: Offset), Imm: PCOffset);
1317 applyITypeImmRISCV(InstrAddr: Section.getAddressWithOffset(OffsetBytes: Offset + 4), Imm: PCOffset);
1318 break;
1319 }
1320 // High 20 bits of 32-bit absolute address, %hi(symbol)
1321 case ELF::R_RISCV_HI20: {
1322 uint64_t PCOffset = Value + Addend;
1323 applyUTypeImmRISCV(InstrAddr: Section.getAddressWithOffset(OffsetBytes: Offset), Imm: PCOffset);
1324 break;
1325 }
1326 // Low 12 bits of 32-bit absolute address, %lo(symbol)
1327 case ELF::R_RISCV_LO12_I: {
1328 uint64_t PCOffset = Value + Addend;
1329 applyITypeImmRISCV(InstrAddr: Section.getAddressWithOffset(OffsetBytes: Offset), Imm: PCOffset);
1330 break;
1331 }
1332 // High 20 bits of 32-bit PC-relative reference, %pcrel_hi(symbol)
1333 case ELF::R_RISCV_GOT_HI20:
1334 case ELF::R_RISCV_PCREL_HI20: {
1335 uint64_t P = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1336 uint64_t PCOffset = Value + Addend - P;
1337 applyUTypeImmRISCV(InstrAddr: Section.getAddressWithOffset(OffsetBytes: Offset), Imm: PCOffset);
1338 break;
1339 }
1340
1341 // label:
1342 // auipc a0, %pcrel_hi(symbol) // R_RISCV_PCREL_HI20
1343 // addi a0, a0, %pcrel_lo(label) // R_RISCV_PCREL_LO12_I
1344 //
1345 // The low 12 bits of relative address between pc and symbol.
1346 // The symbol is related to the high part instruction which is marked by
1347 // label.
1348 case ELF::R_RISCV_PCREL_LO12_I: {
1349 for (auto &&PendingReloc : PendingRelocs) {
1350 const RelocationValueRef &MatchingValue = PendingReloc.first;
1351 RelocationEntry &Reloc = PendingReloc.second;
1352 uint64_t HIRelocPC =
1353 getSectionLoadAddress(SectionID: Reloc.SectionID) + Reloc.Offset;
1354 if (Value + Addend == HIRelocPC) {
1355 uint64_t Symbol = getSectionLoadAddress(SectionID: MatchingValue.SectionID) +
1356 MatchingValue.Addend;
1357 auto PCOffset = Symbol - HIRelocPC;
1358 applyITypeImmRISCV(InstrAddr: Section.getAddressWithOffset(OffsetBytes: Offset), Imm: PCOffset);
1359 return;
1360 }
1361 }
1362
1363 llvm::report_fatal_error(
1364 reason: "R_RISCV_PCREL_LO12_I without matching R_RISCV_PCREL_HI20");
1365 }
1366 case ELF::R_RISCV_32_PCREL: {
1367 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
1368 int64_t RealOffset = Value + Addend - FinalAddress;
1369 int32_t TruncOffset = Lo_32(Value: RealOffset);
1370 support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
1371 TruncOffset;
1372 break;
1373 }
1374 case ELF::R_RISCV_32: {
1375 auto Ref = support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1376 Ref = Value + Addend;
1377 break;
1378 }
1379 case ELF::R_RISCV_64: {
1380 auto Ref = support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1381 Ref = Value + Addend;
1382 break;
1383 }
1384 case ELF::R_RISCV_ADD8: {
1385 auto Ref = support::ulittle8_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1386 Ref = Ref + Value + Addend;
1387 break;
1388 }
1389 case ELF::R_RISCV_ADD16: {
1390 auto Ref = support::ulittle16_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1391 Ref = Ref + Value + Addend;
1392 break;
1393 }
1394 case ELF::R_RISCV_ADD32: {
1395 auto Ref = support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1396 Ref = Ref + Value + Addend;
1397 break;
1398 }
1399 case ELF::R_RISCV_ADD64: {
1400 auto Ref = support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1401 Ref = Ref + Value + Addend;
1402 break;
1403 }
1404 case ELF::R_RISCV_SUB8: {
1405 auto Ref = support::ulittle8_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1406 Ref = Ref - Value - Addend;
1407 break;
1408 }
1409 case ELF::R_RISCV_SUB16: {
1410 auto Ref = support::ulittle16_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1411 Ref = Ref - Value - Addend;
1412 break;
1413 }
1414 case ELF::R_RISCV_SUB32: {
1415 auto Ref = support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1416 Ref = Ref - Value - Addend;
1417 break;
1418 }
1419 case ELF::R_RISCV_SUB64: {
1420 auto Ref = support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1421 Ref = Ref - Value - Addend;
1422 break;
1423 }
1424 case ELF::R_RISCV_SET8: {
1425 auto Ref = support::ulittle8_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1426 Ref = Value + Addend;
1427 break;
1428 }
1429 case ELF::R_RISCV_SET16: {
1430 auto Ref = support::ulittle16_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1431 Ref = Value + Addend;
1432 break;
1433 }
1434 case ELF::R_RISCV_SET32: {
1435 auto Ref = support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset));
1436 Ref = Value + Addend;
1437 break;
1438 }
1439 }
1440}
1441
1442// The target location for the relocation is described by RE.SectionID and
1443// RE.Offset. RE.SectionID can be used to find the SectionEntry. Each
1444// SectionEntry has three members describing its location.
1445// SectionEntry::Address is the address at which the section has been loaded
1446// into memory in the current (host) process. SectionEntry::LoadAddress is the
1447// address that the section will have in the target process.
1448// SectionEntry::ObjAddress is the address of the bits for this section in the
1449// original emitted object image (also in the current address space).
1450//
1451// Relocations will be applied as if the section were loaded at
1452// SectionEntry::LoadAddress, but they will be applied at an address based
1453// on SectionEntry::Address. SectionEntry::ObjAddress will be used to refer to
1454// Target memory contents if they are required for value calculations.
1455//
1456// The Value parameter here is the load address of the symbol for the
1457// relocation to be applied. For relocations which refer to symbols in the
1458// current object Value will be the LoadAddress of the section in which
1459// the symbol resides (RE.Addend provides additional information about the
1460// symbol location). For external symbols, Value will be the address of the
1461// symbol in the target address space.
1462void RuntimeDyldELF::resolveRelocation(const RelocationEntry &RE,
1463 uint64_t Value) {
1464 const SectionEntry &Section = Sections[RE.SectionID];
1465 return resolveRelocation(Section, Offset: RE.Offset, Value, Type: RE.RelType, Addend: RE.Addend,
1466 SymOffset: RE.SymOffset, SectionID: RE.SectionID);
1467}
1468
1469void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section,
1470 uint64_t Offset, uint64_t Value,
1471 uint32_t Type, int64_t Addend,
1472 uint64_t SymOffset, SID SectionID) {
1473 switch (Arch) {
1474 case Triple::x86_64:
1475 resolveX86_64Relocation(Section, Offset, Value, Type, Addend, SymOffset);
1476 break;
1477 case Triple::x86:
1478 resolveX86Relocation(Section, Offset, Value: (uint32_t)(Value & 0xffffffffL), Type,
1479 Addend: (uint32_t)(Addend & 0xffffffffL));
1480 break;
1481 case Triple::aarch64:
1482 case Triple::aarch64_be:
1483 resolveAArch64Relocation(Section, Offset, Value, Type, Addend);
1484 break;
1485 case Triple::arm: // Fall through.
1486 case Triple::armeb:
1487 case Triple::thumb:
1488 case Triple::thumbeb:
1489 resolveARMRelocation(Section, Offset, Value: (uint32_t)(Value & 0xffffffffL), Type,
1490 Addend: (uint32_t)(Addend & 0xffffffffL));
1491 break;
1492 case Triple::loongarch64:
1493 resolveLoongArch64Relocation(Section, Offset, Value, Type, Addend);
1494 break;
1495 case Triple::ppc: // Fall through.
1496 case Triple::ppcle:
1497 resolvePPC32Relocation(Section, Offset, Value, Type, Addend);
1498 break;
1499 case Triple::ppc64: // Fall through.
1500 case Triple::ppc64le:
1501 resolvePPC64Relocation(Section, Offset, Value, Type, Addend);
1502 break;
1503 case Triple::systemz:
1504 resolveSystemZRelocation(Section, Offset, Value, Type, Addend);
1505 break;
1506 case Triple::bpfel:
1507 case Triple::bpfeb:
1508 resolveBPFRelocation(Section, Offset, Value, Type, Addend);
1509 break;
1510 case Triple::riscv32: // Fall through.
1511 case Triple::riscv64:
1512 resolveRISCVRelocation(Section, Offset, Value, Type, Addend, SectionID);
1513 break;
1514 default:
1515 llvm_unreachable("Unsupported CPU type!");
1516 }
1517}
1518
1519void *RuntimeDyldELF::computePlaceholderAddress(unsigned SectionID,
1520 uint64_t Offset) const {
1521 return (void *)(Sections[SectionID].getObjAddress() + Offset);
1522}
1523
1524void RuntimeDyldELF::processSimpleRelocation(unsigned SectionID, uint64_t Offset, unsigned RelType, RelocationValueRef Value) {
1525 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend, Value.Offset);
1526 if (Value.SymbolName)
1527 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1528 else
1529 addRelocationForSection(RE, SectionID: Value.SectionID);
1530}
1531
1532uint32_t RuntimeDyldELF::getMatchingLoRelocation(uint32_t RelType,
1533 bool IsLocal) const {
1534 switch (RelType) {
1535 case ELF::R_MICROMIPS_GOT16:
1536 if (IsLocal)
1537 return ELF::R_MICROMIPS_LO16;
1538 break;
1539 case ELF::R_MICROMIPS_HI16:
1540 return ELF::R_MICROMIPS_LO16;
1541 case ELF::R_MIPS_GOT16:
1542 if (IsLocal)
1543 return ELF::R_MIPS_LO16;
1544 break;
1545 case ELF::R_MIPS_HI16:
1546 return ELF::R_MIPS_LO16;
1547 case ELF::R_MIPS_PCHI16:
1548 return ELF::R_MIPS_PCLO16;
1549 default:
1550 break;
1551 }
1552 return ELF::R_MIPS_NONE;
1553}
1554
1555// Sometimes we don't need to create thunk for a branch.
1556// This typically happens when branch target is located
1557// in the same object file. In such case target is either
1558// a weak symbol or symbol in a different executable section.
1559// This function checks if branch target is located in the
1560// same object file and if distance between source and target
1561// fits R_AARCH64_CALL26 relocation. If both conditions are
1562// met, it emits direct jump to the target and returns true.
1563// Otherwise false is returned and thunk is created.
1564bool RuntimeDyldELF::resolveAArch64ShortBranch(
1565 unsigned SectionID, relocation_iterator RelI,
1566 const RelocationValueRef &Value) {
1567 uint64_t TargetOffset;
1568 unsigned TargetSectionID;
1569 if (Value.SymbolName) {
1570 auto Loc = GlobalSymbolTable.find(Key: Value.SymbolName);
1571
1572 // Don't create direct branch for external symbols.
1573 if (Loc == GlobalSymbolTable.end())
1574 return false;
1575
1576 const auto &SymInfo = Loc->second;
1577
1578 TargetSectionID = SymInfo.getSectionID();
1579 TargetOffset = SymInfo.getOffset();
1580 } else {
1581 TargetSectionID = Value.SectionID;
1582 TargetOffset = 0;
1583 }
1584
1585 // We don't actually know the load addresses at this point, so if the
1586 // branch is cross-section, we don't know exactly how far away it is.
1587 if (TargetSectionID != SectionID)
1588 return false;
1589
1590 uint64_t SourceOffset = RelI->getOffset();
1591
1592 // R_AARCH64_CALL26 requires immediate to be in range -2^27 <= imm < 2^27
1593 // If distance between source and target is out of range then we should
1594 // create thunk.
1595 if (!isInt<28>(x: TargetOffset + Value.Addend - SourceOffset))
1596 return false;
1597
1598 RelocationEntry RE(SectionID, SourceOffset, RelI->getType(), Value.Addend);
1599 if (Value.SymbolName)
1600 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1601 else
1602 addRelocationForSection(RE, SectionID: Value.SectionID);
1603
1604 return true;
1605}
1606
1607void RuntimeDyldELF::resolveAArch64Branch(unsigned SectionID,
1608 const RelocationValueRef &Value,
1609 relocation_iterator RelI,
1610 StubMap &Stubs) {
1611
1612 LLVM_DEBUG(dbgs() << "\t\tThis is an AArch64 branch relocation.");
1613 SectionEntry &Section = Sections[SectionID];
1614
1615 uint64_t Offset = RelI->getOffset();
1616 unsigned RelType = RelI->getType();
1617 // Look for an existing stub.
1618 StubMap::const_iterator i = Stubs.find(x: Value);
1619 if (i != Stubs.end()) {
1620 resolveRelocation(Section, Offset,
1621 Value: Section.getLoadAddressWithOffset(OffsetBytes: i->second), Type: RelType, Addend: 0);
1622 LLVM_DEBUG(dbgs() << " Stub function found\n");
1623 } else if (!resolveAArch64ShortBranch(SectionID, RelI, Value)) {
1624 // Create a new stub function.
1625 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
1626 Stubs[Value] = Section.getStubOffset();
1627 uint8_t *StubTargetAddr = createStubFunction(
1628 Addr: Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset()));
1629
1630 RelocationEntry REmovz_g3(SectionID, StubTargetAddr - Section.getAddress(),
1631 ELF::R_AARCH64_MOVW_UABS_G3, Value.Addend);
1632 RelocationEntry REmovk_g2(SectionID,
1633 StubTargetAddr - Section.getAddress() + 4,
1634 ELF::R_AARCH64_MOVW_UABS_G2_NC, Value.Addend);
1635 RelocationEntry REmovk_g1(SectionID,
1636 StubTargetAddr - Section.getAddress() + 8,
1637 ELF::R_AARCH64_MOVW_UABS_G1_NC, Value.Addend);
1638 RelocationEntry REmovk_g0(SectionID,
1639 StubTargetAddr - Section.getAddress() + 12,
1640 ELF::R_AARCH64_MOVW_UABS_G0_NC, Value.Addend);
1641
1642 if (Value.SymbolName) {
1643 addRelocationForSymbol(RE: REmovz_g3, SymbolName: Value.SymbolName);
1644 addRelocationForSymbol(RE: REmovk_g2, SymbolName: Value.SymbolName);
1645 addRelocationForSymbol(RE: REmovk_g1, SymbolName: Value.SymbolName);
1646 addRelocationForSymbol(RE: REmovk_g0, SymbolName: Value.SymbolName);
1647 } else {
1648 addRelocationForSection(RE: REmovz_g3, SectionID: Value.SectionID);
1649 addRelocationForSection(RE: REmovk_g2, SectionID: Value.SectionID);
1650 addRelocationForSection(RE: REmovk_g1, SectionID: Value.SectionID);
1651 addRelocationForSection(RE: REmovk_g0, SectionID: Value.SectionID);
1652 }
1653 resolveRelocation(Section, Offset,
1654 Value: Section.getLoadAddressWithOffset(OffsetBytes: Section.getStubOffset()),
1655 Type: RelType, Addend: 0);
1656 Section.advanceStubOffset(StubSize: getMaxStubSize());
1657 }
1658}
1659
1660Expected<relocation_iterator>
1661RuntimeDyldELF::processRelocationRef(
1662 unsigned SectionID, relocation_iterator RelI, const ObjectFile &O,
1663 ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) {
1664 const auto &Obj = cast<ELFObjectFileBase>(Val: O);
1665 uint64_t RelType = RelI->getType();
1666 int64_t Addend = 0;
1667 if (Expected<int64_t> AddendOrErr = ELFRelocationRef(*RelI).getAddend())
1668 Addend = *AddendOrErr;
1669 else
1670 consumeError(Err: AddendOrErr.takeError());
1671 elf_symbol_iterator Symbol = RelI->getSymbol();
1672
1673 // Obtain the symbol name which is referenced in the relocation
1674 StringRef TargetName;
1675 if (Symbol != Obj.symbol_end()) {
1676 if (auto TargetNameOrErr = Symbol->getName())
1677 TargetName = *TargetNameOrErr;
1678 else
1679 return TargetNameOrErr.takeError();
1680 }
1681 LLVM_DEBUG(dbgs() << "\t\tRelType: " << RelType << " Addend: " << Addend
1682 << " TargetName: " << TargetName << "\n");
1683 RelocationValueRef Value;
1684 // First search for the symbol in the local symbol table
1685 SymbolRef::Type SymType = SymbolRef::ST_Unknown;
1686
1687 // Search for the symbol in the global symbol table
1688 RTDyldSymbolTable::const_iterator gsi = GlobalSymbolTable.end();
1689 if (Symbol != Obj.symbol_end()) {
1690 gsi = GlobalSymbolTable.find(Key: TargetName.data());
1691 Expected<SymbolRef::Type> SymTypeOrErr = Symbol->getType();
1692 if (!SymTypeOrErr) {
1693 std::string Buf;
1694 raw_string_ostream OS(Buf);
1695 logAllUnhandledErrors(E: SymTypeOrErr.takeError(), OS);
1696 report_fatal_error(reason: Twine(Buf));
1697 }
1698 SymType = *SymTypeOrErr;
1699 }
1700 if (gsi != GlobalSymbolTable.end()) {
1701 const auto &SymInfo = gsi->second;
1702 Value.SectionID = SymInfo.getSectionID();
1703 Value.Offset = SymInfo.getOffset();
1704 Value.Addend = SymInfo.getOffset() + Addend;
1705 } else {
1706 switch (SymType) {
1707 case SymbolRef::ST_Debug: {
1708 // TODO: Now ELF SymbolRef::ST_Debug = STT_SECTION, it's not obviously
1709 // and can be changed by another developers. Maybe best way is add
1710 // a new symbol type ST_Section to SymbolRef and use it.
1711 auto SectionOrErr = Symbol->getSection();
1712 if (!SectionOrErr) {
1713 std::string Buf;
1714 raw_string_ostream OS(Buf);
1715 logAllUnhandledErrors(E: SectionOrErr.takeError(), OS);
1716 report_fatal_error(reason: Twine(Buf));
1717 }
1718 section_iterator si = *SectionOrErr;
1719 if (si == Obj.section_end())
1720 llvm_unreachable("Symbol section not found, bad object file format!");
1721 LLVM_DEBUG(dbgs() << "\t\tThis is section symbol\n");
1722 bool isCode = si->isText();
1723 if (auto SectionIDOrErr = findOrEmitSection(Obj, Section: (*si), IsCode: isCode,
1724 LocalSections&: ObjSectionToID))
1725 Value.SectionID = *SectionIDOrErr;
1726 else
1727 return SectionIDOrErr.takeError();
1728 Value.Addend = Addend;
1729 break;
1730 }
1731 case SymbolRef::ST_Data:
1732 case SymbolRef::ST_Function:
1733 case SymbolRef::ST_Other:
1734 case SymbolRef::ST_Unknown: {
1735 Value.SymbolName = TargetName.data();
1736 Value.Addend = Addend;
1737
1738 // Absolute relocations will have a zero symbol ID (STN_UNDEF), which
1739 // will manifest here as a NULL symbol name.
1740 // We can set this as a valid (but empty) symbol name, and rely
1741 // on addRelocationForSymbol to handle this.
1742 if (!Value.SymbolName)
1743 Value.SymbolName = "";
1744 break;
1745 }
1746 default:
1747 llvm_unreachable("Unresolved symbol type!");
1748 break;
1749 }
1750 }
1751
1752 uint64_t Offset = RelI->getOffset();
1753
1754 LLVM_DEBUG(dbgs() << "\t\tSectionID: " << SectionID << " Offset: " << Offset
1755 << "\n");
1756 if ((Arch == Triple::aarch64 || Arch == Triple::aarch64_be)) {
1757 if ((RelType == ELF::R_AARCH64_CALL26 ||
1758 RelType == ELF::R_AARCH64_JUMP26) &&
1759 MemMgr.allowStubAllocation()) {
1760 resolveAArch64Branch(SectionID, Value, RelI, Stubs);
1761 } else if (RelType == ELF::R_AARCH64_ADR_GOT_PAGE) {
1762 // Create new GOT entry or find existing one. If GOT entry is
1763 // to be created, then we also emit ABS64 relocation for it.
1764 uint64_t GOTOffset = findOrAllocGOTEntry(Value, GOTRelType: ELF::R_AARCH64_ABS64);
1765 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: GOTOffset + Addend,
1766 Type: ELF::R_AARCH64_ADR_PREL_PG_HI21);
1767
1768 } else if (RelType == ELF::R_AARCH64_LD64_GOT_LO12_NC) {
1769 uint64_t GOTOffset = findOrAllocGOTEntry(Value, GOTRelType: ELF::R_AARCH64_ABS64);
1770 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: GOTOffset + Addend,
1771 Type: ELF::R_AARCH64_LDST64_ABS_LO12_NC);
1772 } else {
1773 processSimpleRelocation(SectionID, Offset, RelType, Value);
1774 }
1775 } else if (Arch == Triple::arm) {
1776 if (RelType == ELF::R_ARM_PC24 || RelType == ELF::R_ARM_CALL ||
1777 RelType == ELF::R_ARM_JUMP24) {
1778 // This is an ARM branch relocation, need to use a stub function.
1779 LLVM_DEBUG(dbgs() << "\t\tThis is an ARM branch relocation.\n");
1780 SectionEntry &Section = Sections[SectionID];
1781
1782 // Look for an existing stub.
1783 auto [It, Inserted] = Stubs.try_emplace(k: Value);
1784 if (!Inserted) {
1785 resolveRelocation(Section, Offset,
1786 Value: Section.getLoadAddressWithOffset(OffsetBytes: It->second), Type: RelType,
1787 Addend: 0);
1788 LLVM_DEBUG(dbgs() << " Stub function found\n");
1789 } else {
1790 // Create a new stub function.
1791 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
1792 It->second = Section.getStubOffset();
1793 uint8_t *StubTargetAddr = createStubFunction(
1794 Addr: Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset()));
1795 RelocationEntry RE(SectionID, StubTargetAddr - Section.getAddress(),
1796 ELF::R_ARM_ABS32, Value.Addend);
1797 if (Value.SymbolName)
1798 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1799 else
1800 addRelocationForSection(RE, SectionID: Value.SectionID);
1801
1802 resolveRelocation(
1803 Section, Offset,
1804 Value: Section.getLoadAddressWithOffset(OffsetBytes: Section.getStubOffset()), Type: RelType,
1805 Addend: 0);
1806 Section.advanceStubOffset(StubSize: getMaxStubSize());
1807 }
1808 } else {
1809 uint32_t *Placeholder =
1810 reinterpret_cast<uint32_t*>(computePlaceholderAddress(SectionID, Offset));
1811 if (RelType == ELF::R_ARM_PREL31 || RelType == ELF::R_ARM_TARGET1 ||
1812 RelType == ELF::R_ARM_ABS32) {
1813 Value.Addend += *Placeholder;
1814 } else if (RelType == ELF::R_ARM_MOVW_ABS_NC || RelType == ELF::R_ARM_MOVT_ABS) {
1815 // See ELF for ARM documentation
1816 Value.Addend += (int16_t)((*Placeholder & 0xFFF) | (((*Placeholder >> 16) & 0xF) << 12));
1817 }
1818 processSimpleRelocation(SectionID, Offset, RelType, Value);
1819 }
1820 } else if (Arch == Triple::loongarch64) {
1821 if ((RelType == ELF::R_LARCH_B26 || RelType == ELF::R_LARCH_CALL36) &&
1822 MemMgr.allowStubAllocation()) {
1823 resolveLoongArch64Branch(SectionID, Value, RelI, Stubs);
1824 } else if (RelType == ELF::R_LARCH_GOT_PC_HI20 ||
1825 RelType == ELF::R_LARCH_GOT_PC_LO12 ||
1826 RelType == ELF::R_LARCH_GOT64_PC_HI12 ||
1827 RelType == ELF::R_LARCH_GOT64_PC_LO20) {
1828 uint64_t GOTOffset = findOrAllocGOTEntry(Value, GOTRelType: ELF::R_LARCH_64);
1829 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: GOTOffset + Addend,
1830 Type: RelType);
1831 } else {
1832 processSimpleRelocation(SectionID, Offset, RelType, Value);
1833 }
1834 } else if (IsMipsO32ABI) {
1835 uint8_t *Placeholder = reinterpret_cast<uint8_t *>(
1836 computePlaceholderAddress(SectionID, Offset));
1837 uint32_t Opcode = readBytesUnaligned(Src: Placeholder, Size: 4);
1838 if (RelType == ELF::R_MIPS_26) {
1839 // This is an Mips branch relocation, need to use a stub function.
1840 LLVM_DEBUG(dbgs() << "\t\tThis is a Mips branch relocation.");
1841 SectionEntry &Section = Sections[SectionID];
1842
1843 // Extract the addend from the instruction.
1844 // We shift up by two since the Value will be down shifted again
1845 // when applying the relocation.
1846 uint32_t Addend = (Opcode & 0x03ffffff) << 2;
1847
1848 Value.Addend += Addend;
1849
1850 // Look up for existing stub.
1851 auto [It, Inserted] = Stubs.try_emplace(k: Value);
1852 if (!Inserted) {
1853 RelocationEntry RE(SectionID, Offset, RelType, It->second);
1854 addRelocationForSection(RE, SectionID);
1855 LLVM_DEBUG(dbgs() << " Stub function found\n");
1856 } else {
1857 // Create a new stub function.
1858 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
1859 It->second = Section.getStubOffset();
1860
1861 unsigned AbiVariant = Obj.getPlatformFlags();
1862
1863 uint8_t *StubTargetAddr = createStubFunction(
1864 Addr: Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset()), AbiVariant);
1865
1866 // Creating Hi and Lo relocations for the filled stub instructions.
1867 RelocationEntry REHi(SectionID, StubTargetAddr - Section.getAddress(),
1868 ELF::R_MIPS_HI16, Value.Addend);
1869 RelocationEntry RELo(SectionID,
1870 StubTargetAddr - Section.getAddress() + 4,
1871 ELF::R_MIPS_LO16, Value.Addend);
1872
1873 if (Value.SymbolName) {
1874 addRelocationForSymbol(RE: REHi, SymbolName: Value.SymbolName);
1875 addRelocationForSymbol(RE: RELo, SymbolName: Value.SymbolName);
1876 } else {
1877 addRelocationForSection(RE: REHi, SectionID: Value.SectionID);
1878 addRelocationForSection(RE: RELo, SectionID: Value.SectionID);
1879 }
1880
1881 RelocationEntry RE(SectionID, Offset, RelType, Section.getStubOffset());
1882 addRelocationForSection(RE, SectionID);
1883 Section.advanceStubOffset(StubSize: getMaxStubSize());
1884 }
1885 } else if (RelType == ELF::R_MIPS_HI16 || RelType == ELF::R_MIPS_PCHI16) {
1886 int64_t Addend = (Opcode & 0x0000ffff) << 16;
1887 RelocationEntry RE(SectionID, Offset, RelType, Addend);
1888 PendingRelocs.push_back(Elt: std::make_pair(x&: Value, y&: RE));
1889 } else if (RelType == ELF::R_MIPS_LO16 || RelType == ELF::R_MIPS_PCLO16) {
1890 int64_t Addend = Value.Addend + SignExtend32<16>(X: Opcode & 0x0000ffff);
1891 for (auto I = PendingRelocs.begin(); I != PendingRelocs.end();) {
1892 const RelocationValueRef &MatchingValue = I->first;
1893 RelocationEntry &Reloc = I->second;
1894 if (MatchingValue == Value &&
1895 RelType == getMatchingLoRelocation(RelType: Reloc.RelType) &&
1896 SectionID == Reloc.SectionID) {
1897 Reloc.Addend += Addend;
1898 if (Value.SymbolName)
1899 addRelocationForSymbol(RE: Reloc, SymbolName: Value.SymbolName);
1900 else
1901 addRelocationForSection(RE: Reloc, SectionID: Value.SectionID);
1902 I = PendingRelocs.erase(CI: I);
1903 } else
1904 ++I;
1905 }
1906 RelocationEntry RE(SectionID, Offset, RelType, Addend);
1907 if (Value.SymbolName)
1908 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1909 else
1910 addRelocationForSection(RE, SectionID: Value.SectionID);
1911 } else {
1912 if (RelType == ELF::R_MIPS_32)
1913 Value.Addend += Opcode;
1914 else if (RelType == ELF::R_MIPS_PC16)
1915 Value.Addend += SignExtend32<18>(X: (Opcode & 0x0000ffff) << 2);
1916 else if (RelType == ELF::R_MIPS_PC19_S2)
1917 Value.Addend += SignExtend32<21>(X: (Opcode & 0x0007ffff) << 2);
1918 else if (RelType == ELF::R_MIPS_PC21_S2)
1919 Value.Addend += SignExtend32<23>(X: (Opcode & 0x001fffff) << 2);
1920 else if (RelType == ELF::R_MIPS_PC26_S2)
1921 Value.Addend += SignExtend32<28>(X: (Opcode & 0x03ffffff) << 2);
1922 processSimpleRelocation(SectionID, Offset, RelType, Value);
1923 }
1924 } else if (IsMipsN32ABI || IsMipsN64ABI) {
1925 uint32_t r_type = RelType & 0xff;
1926 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend);
1927 if (r_type == ELF::R_MIPS_CALL16 || r_type == ELF::R_MIPS_GOT_PAGE
1928 || r_type == ELF::R_MIPS_GOT_DISP) {
1929 auto [I, Inserted] = GOTSymbolOffsets.try_emplace(Key: TargetName);
1930 if (Inserted)
1931 I->second = allocateGOTEntries(no: 1);
1932 RE.SymOffset = I->second;
1933 if (Value.SymbolName)
1934 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1935 else
1936 addRelocationForSection(RE, SectionID: Value.SectionID);
1937 } else if (RelType == ELF::R_MIPS_26) {
1938 // This is an Mips branch relocation, need to use a stub function.
1939 LLVM_DEBUG(dbgs() << "\t\tThis is a Mips branch relocation.");
1940 SectionEntry &Section = Sections[SectionID];
1941
1942 // Look up for existing stub.
1943 StubMap::const_iterator i = Stubs.find(x: Value);
1944 if (i != Stubs.end()) {
1945 RelocationEntry RE(SectionID, Offset, RelType, i->second);
1946 addRelocationForSection(RE, SectionID);
1947 LLVM_DEBUG(dbgs() << " Stub function found\n");
1948 } else {
1949 // Create a new stub function.
1950 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
1951 Stubs[Value] = Section.getStubOffset();
1952
1953 unsigned AbiVariant = Obj.getPlatformFlags();
1954
1955 uint8_t *StubTargetAddr = createStubFunction(
1956 Addr: Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset()), AbiVariant);
1957
1958 if (IsMipsN32ABI) {
1959 // Creating Hi and Lo relocations for the filled stub instructions.
1960 RelocationEntry REHi(SectionID, StubTargetAddr - Section.getAddress(),
1961 ELF::R_MIPS_HI16, Value.Addend);
1962 RelocationEntry RELo(SectionID,
1963 StubTargetAddr - Section.getAddress() + 4,
1964 ELF::R_MIPS_LO16, Value.Addend);
1965 if (Value.SymbolName) {
1966 addRelocationForSymbol(RE: REHi, SymbolName: Value.SymbolName);
1967 addRelocationForSymbol(RE: RELo, SymbolName: Value.SymbolName);
1968 } else {
1969 addRelocationForSection(RE: REHi, SectionID: Value.SectionID);
1970 addRelocationForSection(RE: RELo, SectionID: Value.SectionID);
1971 }
1972 } else {
1973 // Creating Highest, Higher, Hi and Lo relocations for the filled stub
1974 // instructions.
1975 RelocationEntry REHighest(SectionID,
1976 StubTargetAddr - Section.getAddress(),
1977 ELF::R_MIPS_HIGHEST, Value.Addend);
1978 RelocationEntry REHigher(SectionID,
1979 StubTargetAddr - Section.getAddress() + 4,
1980 ELF::R_MIPS_HIGHER, Value.Addend);
1981 RelocationEntry REHi(SectionID,
1982 StubTargetAddr - Section.getAddress() + 12,
1983 ELF::R_MIPS_HI16, Value.Addend);
1984 RelocationEntry RELo(SectionID,
1985 StubTargetAddr - Section.getAddress() + 20,
1986 ELF::R_MIPS_LO16, Value.Addend);
1987 if (Value.SymbolName) {
1988 addRelocationForSymbol(RE: REHighest, SymbolName: Value.SymbolName);
1989 addRelocationForSymbol(RE: REHigher, SymbolName: Value.SymbolName);
1990 addRelocationForSymbol(RE: REHi, SymbolName: Value.SymbolName);
1991 addRelocationForSymbol(RE: RELo, SymbolName: Value.SymbolName);
1992 } else {
1993 addRelocationForSection(RE: REHighest, SectionID: Value.SectionID);
1994 addRelocationForSection(RE: REHigher, SectionID: Value.SectionID);
1995 addRelocationForSection(RE: REHi, SectionID: Value.SectionID);
1996 addRelocationForSection(RE: RELo, SectionID: Value.SectionID);
1997 }
1998 }
1999 RelocationEntry RE(SectionID, Offset, RelType, Section.getStubOffset());
2000 addRelocationForSection(RE, SectionID);
2001 Section.advanceStubOffset(StubSize: getMaxStubSize());
2002 }
2003 } else {
2004 processSimpleRelocation(SectionID, Offset, RelType, Value);
2005 }
2006
2007 } else if (Arch == Triple::ppc) {
2008 if (RelType == ELF::R_PPC_PLTREL24) {
2009 // The addend selects the GOT pointer of a PLT call stub. It is not an
2010 // offset from the symbol.
2011 Value.Addend -= Addend;
2012 RelType = ELF::R_PPC_REL24;
2013 }
2014
2015 // A branch needs a stub if its target is external or out of range.
2016 SectionEntry &Section = Sections[SectionID];
2017 bool NeedsStub = RelType == ELF::R_PPC_REL24 &&
2018 (Value.SymbolName ||
2019 !isInt<26>(x: Sections[Value.SectionID].getAddressWithOffset(
2020 OffsetBytes: Value.Addend) -
2021 Section.getAddressWithOffset(OffsetBytes: Offset)));
2022 if (!NeedsStub) {
2023 processSimpleRelocation(SectionID, Offset, RelType, Value);
2024 } else {
2025 auto [It, Inserted] = Stubs.try_emplace(k: Value, args: Section.getStubOffset());
2026 if (Inserted) {
2027 createStubFunction(Addr: Section.getAddressWithOffset(OffsetBytes: It->second));
2028 // The relocations apply to the low half of the lis and the ori.
2029 processSimpleRelocation(SectionID, Offset: It->second + 2, RelType: ELF::R_PPC_ADDR16_HI,
2030 Value);
2031 processSimpleRelocation(SectionID, Offset: It->second + 6, RelType: ELF::R_PPC_ADDR16_LO,
2032 Value);
2033 Section.advanceStubOffset(StubSize: getMaxStubSize());
2034 }
2035 resolveRelocation(Section, Offset,
2036 Value: Section.getLoadAddressWithOffset(OffsetBytes: It->second), Type: RelType,
2037 Addend: 0);
2038 }
2039 } else if (Arch == Triple::ppc64 || Arch == Triple::ppc64le) {
2040 if (RelType == ELF::R_PPC64_REL24) {
2041 // Determine ABI variant in use for this object.
2042 unsigned AbiVariant = Obj.getPlatformFlags();
2043 AbiVariant &= ELF::EF_PPC64_ABI;
2044 // A PPC branch relocation will need a stub function if the target is
2045 // an external symbol (either Value.SymbolName is set, or SymType is
2046 // Symbol::ST_Unknown) or if the target address is not within the
2047 // signed 24-bits branch address.
2048 SectionEntry &Section = Sections[SectionID];
2049 uint8_t *Target = Section.getAddressWithOffset(OffsetBytes: Offset);
2050 bool RangeOverflow = false;
2051 bool IsExtern = Value.SymbolName || SymType == SymbolRef::ST_Unknown;
2052 if (!IsExtern) {
2053 if (AbiVariant != 2) {
2054 // In the ELFv1 ABI, a function call may point to the .opd entry,
2055 // so the final symbol value is calculated based on the relocation
2056 // values in the .opd section.
2057 if (auto Err = findOPDEntrySection(Obj, LocalSections&: ObjSectionToID, Rel&: Value))
2058 return std::move(Err);
2059 } else {
2060 // In the ELFv2 ABI, a function symbol may provide a local entry
2061 // point, which must be used for direct calls.
2062 if (Value.SectionID == SectionID){
2063 uint8_t SymOther = Symbol->getOther();
2064 Value.Addend += ELF::decodePPC64LocalEntryOffset(Other: SymOther);
2065 }
2066 }
2067 uint8_t *RelocTarget =
2068 Sections[Value.SectionID].getAddressWithOffset(OffsetBytes: Value.Addend);
2069 int64_t delta = static_cast<int64_t>(Target - RelocTarget);
2070 // If it is within 26-bits branch range, just set the branch target
2071 if (SignExtend64<26>(x: delta) != delta) {
2072 RangeOverflow = true;
2073 } else if ((AbiVariant != 2) ||
2074 (AbiVariant == 2 && Value.SectionID == SectionID)) {
2075 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend);
2076 addRelocationForSection(RE, SectionID: Value.SectionID);
2077 }
2078 }
2079 if (IsExtern || (AbiVariant == 2 && Value.SectionID != SectionID) ||
2080 RangeOverflow) {
2081 // It is an external symbol (either Value.SymbolName is set, or
2082 // SymType is SymbolRef::ST_Unknown) or out of range.
2083 auto [It, Inserted] = Stubs.try_emplace(k: Value);
2084 if (!Inserted) {
2085 // Symbol function stub already created, just relocate to it
2086 resolveRelocation(Section, Offset,
2087 Value: Section.getLoadAddressWithOffset(OffsetBytes: It->second),
2088 Type: RelType, Addend: 0);
2089 LLVM_DEBUG(dbgs() << " Stub function found\n");
2090 } else {
2091 // Create a new stub function.
2092 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
2093 It->second = Section.getStubOffset();
2094 uint8_t *StubTargetAddr = createStubFunction(
2095 Addr: Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset()),
2096 AbiVariant);
2097 RelocationEntry RE(SectionID, StubTargetAddr - Section.getAddress(),
2098 ELF::R_PPC64_ADDR64, Value.Addend);
2099
2100 // Generates the 64-bits address loads as exemplified in section
2101 // 4.5.1 in PPC64 ELF ABI. Note that the relocations need to
2102 // apply to the low part of the instructions, so we have to update
2103 // the offset according to the target endianness.
2104 uint64_t StubRelocOffset = StubTargetAddr - Section.getAddress();
2105 if (!IsTargetLittleEndian)
2106 StubRelocOffset += 2;
2107
2108 RelocationEntry REhst(SectionID, StubRelocOffset + 0,
2109 ELF::R_PPC64_ADDR16_HIGHEST, Value.Addend);
2110 RelocationEntry REhr(SectionID, StubRelocOffset + 4,
2111 ELF::R_PPC64_ADDR16_HIGHER, Value.Addend);
2112 RelocationEntry REh(SectionID, StubRelocOffset + 12,
2113 ELF::R_PPC64_ADDR16_HI, Value.Addend);
2114 RelocationEntry REl(SectionID, StubRelocOffset + 16,
2115 ELF::R_PPC64_ADDR16_LO, Value.Addend);
2116
2117 if (Value.SymbolName) {
2118 addRelocationForSymbol(RE: REhst, SymbolName: Value.SymbolName);
2119 addRelocationForSymbol(RE: REhr, SymbolName: Value.SymbolName);
2120 addRelocationForSymbol(RE: REh, SymbolName: Value.SymbolName);
2121 addRelocationForSymbol(RE: REl, SymbolName: Value.SymbolName);
2122 } else {
2123 addRelocationForSection(RE: REhst, SectionID: Value.SectionID);
2124 addRelocationForSection(RE: REhr, SectionID: Value.SectionID);
2125 addRelocationForSection(RE: REh, SectionID: Value.SectionID);
2126 addRelocationForSection(RE: REl, SectionID: Value.SectionID);
2127 }
2128
2129 resolveRelocation(
2130 Section, Offset,
2131 Value: Section.getLoadAddressWithOffset(OffsetBytes: Section.getStubOffset()),
2132 Type: RelType, Addend: 0);
2133 Section.advanceStubOffset(StubSize: getMaxStubSize());
2134 }
2135 if (IsExtern || (AbiVariant == 2 && Value.SectionID != SectionID)) {
2136 // Restore the TOC for external calls
2137 if (AbiVariant == 2)
2138 writeInt32BE(Addr: Target + 4, Value: 0xE8410018); // ld r2,24(r1)
2139 else
2140 writeInt32BE(Addr: Target + 4, Value: 0xE8410028); // ld r2,40(r1)
2141 }
2142 }
2143 } else if (RelType == ELF::R_PPC64_TOC16 ||
2144 RelType == ELF::R_PPC64_TOC16_DS ||
2145 RelType == ELF::R_PPC64_TOC16_LO ||
2146 RelType == ELF::R_PPC64_TOC16_LO_DS ||
2147 RelType == ELF::R_PPC64_TOC16_HI ||
2148 RelType == ELF::R_PPC64_TOC16_HA) {
2149 // These relocations are supposed to subtract the TOC address from
2150 // the final value. This does not fit cleanly into the RuntimeDyld
2151 // scheme, since there may be *two* sections involved in determining
2152 // the relocation value (the section of the symbol referred to by the
2153 // relocation, and the TOC section associated with the current module).
2154 //
2155 // Fortunately, these relocations are currently only ever generated
2156 // referring to symbols that themselves reside in the TOC, which means
2157 // that the two sections are actually the same. Thus they cancel out
2158 // and we can immediately resolve the relocation right now.
2159 switch (RelType) {
2160 case ELF::R_PPC64_TOC16: RelType = ELF::R_PPC64_ADDR16; break;
2161 case ELF::R_PPC64_TOC16_DS: RelType = ELF::R_PPC64_ADDR16_DS; break;
2162 case ELF::R_PPC64_TOC16_LO: RelType = ELF::R_PPC64_ADDR16_LO; break;
2163 case ELF::R_PPC64_TOC16_LO_DS: RelType = ELF::R_PPC64_ADDR16_LO_DS; break;
2164 case ELF::R_PPC64_TOC16_HI: RelType = ELF::R_PPC64_ADDR16_HI; break;
2165 case ELF::R_PPC64_TOC16_HA: RelType = ELF::R_PPC64_ADDR16_HA; break;
2166 default: llvm_unreachable("Wrong relocation type.");
2167 }
2168
2169 RelocationValueRef TOCValue;
2170 if (auto Err = findPPC64TOCSection(Obj, LocalSections&: ObjSectionToID, Rel&: TOCValue))
2171 return std::move(Err);
2172 if (Value.SymbolName || Value.SectionID != TOCValue.SectionID)
2173 llvm_unreachable("Unsupported TOC relocation.");
2174 Value.Addend -= TOCValue.Addend;
2175 resolveRelocation(Section: Sections[SectionID], Offset, Value: Value.Addend, Type: RelType, Addend: 0);
2176 } else {
2177 // There are two ways to refer to the TOC address directly: either
2178 // via a ELF::R_PPC64_TOC relocation (where both symbol and addend are
2179 // ignored), or via any relocation that refers to the magic ".TOC."
2180 // symbols (in which case the addend is respected).
2181 if (RelType == ELF::R_PPC64_TOC) {
2182 RelType = ELF::R_PPC64_ADDR64;
2183 if (auto Err = findPPC64TOCSection(Obj, LocalSections&: ObjSectionToID, Rel&: Value))
2184 return std::move(Err);
2185 } else if (TargetName == ".TOC.") {
2186 if (auto Err = findPPC64TOCSection(Obj, LocalSections&: ObjSectionToID, Rel&: Value))
2187 return std::move(Err);
2188 Value.Addend += Addend;
2189 }
2190
2191 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend);
2192
2193 if (Value.SymbolName)
2194 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
2195 else
2196 addRelocationForSection(RE, SectionID: Value.SectionID);
2197 }
2198 } else if (Arch == Triple::systemz &&
2199 (RelType == ELF::R_390_PLT32DBL || RelType == ELF::R_390_GOTENT)) {
2200 // Create function stubs for both PLT and GOT references, regardless of
2201 // whether the GOT reference is to data or code. The stub contains the
2202 // full address of the symbol, as needed by GOT references, and the
2203 // executable part only adds an overhead of 8 bytes.
2204 //
2205 // We could try to conserve space by allocating the code and data
2206 // parts of the stub separately. However, as things stand, we allocate
2207 // a stub for every relocation, so using a GOT in JIT code should be
2208 // no less space efficient than using an explicit constant pool.
2209 LLVM_DEBUG(dbgs() << "\t\tThis is a SystemZ indirect relocation.");
2210 SectionEntry &Section = Sections[SectionID];
2211
2212 // Look for an existing stub.
2213 StubMap::const_iterator i = Stubs.find(x: Value);
2214 uintptr_t StubAddress;
2215 if (i != Stubs.end()) {
2216 StubAddress = uintptr_t(Section.getAddressWithOffset(OffsetBytes: i->second));
2217 LLVM_DEBUG(dbgs() << " Stub function found\n");
2218 } else {
2219 // Create a new stub function.
2220 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
2221
2222 uintptr_t BaseAddress = uintptr_t(Section.getAddress());
2223 StubAddress =
2224 alignTo(Size: BaseAddress + Section.getStubOffset(), A: getStubAlignment());
2225 unsigned StubOffset = StubAddress - BaseAddress;
2226
2227 Stubs[Value] = StubOffset;
2228 createStubFunction(Addr: (uint8_t *)StubAddress);
2229 RelocationEntry RE(SectionID, StubOffset + 8, ELF::R_390_64,
2230 Value.Offset);
2231 if (Value.SymbolName)
2232 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
2233 else
2234 addRelocationForSection(RE, SectionID: Value.SectionID);
2235 Section.advanceStubOffset(StubSize: getMaxStubSize());
2236 }
2237
2238 if (RelType == ELF::R_390_GOTENT)
2239 resolveRelocation(Section, Offset, Value: StubAddress + 8, Type: ELF::R_390_PC32DBL,
2240 Addend);
2241 else
2242 resolveRelocation(Section, Offset, Value: StubAddress, Type: RelType, Addend);
2243 } else if (Arch == Triple::x86_64) {
2244 if (RelType == ELF::R_X86_64_PLT32) {
2245 // The way the PLT relocations normally work is that the linker allocates
2246 // the
2247 // PLT and this relocation makes a PC-relative call into the PLT. The PLT
2248 // entry will then jump to an address provided by the GOT. On first call,
2249 // the
2250 // GOT address will point back into PLT code that resolves the symbol. After
2251 // the first call, the GOT entry points to the actual function.
2252 //
2253 // For local functions we're ignoring all of that here and just replacing
2254 // the PLT32 relocation type with PC32, which will translate the relocation
2255 // into a PC-relative call directly to the function. For external symbols we
2256 // can't be sure the function will be within 2^32 bytes of the call site, so
2257 // we need to create a stub, which calls into the GOT. This case is
2258 // equivalent to the usual PLT implementation except that we use the stub
2259 // mechanism in RuntimeDyld (which puts stubs at the end of the section)
2260 // rather than allocating a PLT section.
2261 if (Value.SymbolName && MemMgr.allowStubAllocation()) {
2262 // This is a call to an external function.
2263 // Look for an existing stub.
2264 SectionEntry *Section = &Sections[SectionID];
2265 auto [It, Inserted] = Stubs.try_emplace(k: Value);
2266 uintptr_t StubAddress;
2267 if (!Inserted) {
2268 StubAddress = uintptr_t(Section->getAddress()) + It->second;
2269 LLVM_DEBUG(dbgs() << " Stub function found\n");
2270 } else {
2271 // Create a new stub function (equivalent to a PLT entry).
2272 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
2273
2274 uintptr_t BaseAddress = uintptr_t(Section->getAddress());
2275 StubAddress = alignTo(Size: BaseAddress + Section->getStubOffset(),
2276 A: getStubAlignment());
2277 unsigned StubOffset = StubAddress - BaseAddress;
2278 It->second = StubOffset;
2279 createStubFunction(Addr: (uint8_t *)StubAddress);
2280
2281 // Bump our stub offset counter
2282 Section->advanceStubOffset(StubSize: getMaxStubSize());
2283
2284 // Allocate a GOT Entry
2285 uint64_t GOTOffset = allocateGOTEntries(no: 1);
2286 // This potentially creates a new Section which potentially
2287 // invalidates the Section pointer, so reload it.
2288 Section = &Sections[SectionID];
2289
2290 // The load of the GOT address has an addend of -4
2291 resolveGOTOffsetRelocation(SectionID, Offset: StubOffset + 2, GOTOffset: GOTOffset - 4,
2292 Type: ELF::R_X86_64_PC32);
2293
2294 // Fill in the value of the symbol we're targeting into the GOT
2295 addRelocationForSymbol(
2296 RE: computeGOTOffsetRE(GOTOffset, SymbolOffset: 0, Type: ELF::R_X86_64_64),
2297 SymbolName: Value.SymbolName);
2298 }
2299
2300 // Make the target call a call into the stub table.
2301 resolveRelocation(Section: *Section, Offset, Value: StubAddress, Type: ELF::R_X86_64_PC32,
2302 Addend);
2303 } else {
2304 Value.Addend += support::ulittle32_t::ref(
2305 computePlaceholderAddress(SectionID, Offset));
2306 processSimpleRelocation(SectionID, Offset, RelType: ELF::R_X86_64_PC32, Value);
2307 }
2308 } else if (RelType == ELF::R_X86_64_GOTPCREL ||
2309 RelType == ELF::R_X86_64_GOTPCRELX ||
2310 RelType == ELF::R_X86_64_REX_GOTPCRELX) {
2311 uint64_t GOTOffset = allocateGOTEntries(no: 1);
2312 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: GOTOffset + Addend,
2313 Type: ELF::R_X86_64_PC32);
2314
2315 // Fill in the value of the symbol we're targeting into the GOT
2316 RelocationEntry RE =
2317 computeGOTOffsetRE(GOTOffset, SymbolOffset: Value.Offset, Type: ELF::R_X86_64_64);
2318 if (Value.SymbolName)
2319 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
2320 else
2321 addRelocationForSection(RE, SectionID: Value.SectionID);
2322 } else if (RelType == ELF::R_X86_64_GOT64) {
2323 // Fill in a 64-bit GOT offset.
2324 uint64_t GOTOffset = allocateGOTEntries(no: 1);
2325 resolveRelocation(Section: Sections[SectionID], Offset, Value: GOTOffset,
2326 Type: ELF::R_X86_64_64, Addend: 0);
2327
2328 // Fill in the value of the symbol we're targeting into the GOT
2329 RelocationEntry RE =
2330 computeGOTOffsetRE(GOTOffset, SymbolOffset: Value.Offset, Type: ELF::R_X86_64_64);
2331 if (Value.SymbolName)
2332 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
2333 else
2334 addRelocationForSection(RE, SectionID: Value.SectionID);
2335 } else if (RelType == ELF::R_X86_64_GOTPC32) {
2336 // Materialize the address of the base of the GOT relative to the PC.
2337 // This doesn't create a GOT entry, but it does mean we need a GOT
2338 // section.
2339 (void)allocateGOTEntries(no: 0);
2340 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: Addend, Type: ELF::R_X86_64_PC32);
2341 } else if (RelType == ELF::R_X86_64_GOTPC64) {
2342 (void)allocateGOTEntries(no: 0);
2343 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: Addend, Type: ELF::R_X86_64_PC64);
2344 } else if (RelType == ELF::R_X86_64_GOTOFF64) {
2345 // GOTOFF relocations ultimately require a section difference relocation.
2346 (void)allocateGOTEntries(no: 0);
2347 processSimpleRelocation(SectionID, Offset, RelType, Value);
2348 } else if (RelType == ELF::R_X86_64_PC32) {
2349 Value.Addend += support::ulittle32_t::ref(computePlaceholderAddress(SectionID, Offset));
2350 processSimpleRelocation(SectionID, Offset, RelType, Value);
2351 } else if (RelType == ELF::R_X86_64_PC64) {
2352 Value.Addend += support::ulittle64_t::ref(
2353 computePlaceholderAddress(SectionID, Offset));
2354 processSimpleRelocation(SectionID, Offset, RelType, Value);
2355 } else if (RelType == ELF::R_X86_64_GOTTPOFF) {
2356 processX86_64GOTTPOFFRelocation(SectionID, Offset, Value, Addend);
2357 } else if (RelType == ELF::R_X86_64_TLSGD ||
2358 RelType == ELF::R_X86_64_TLSLD) {
2359 // The next relocation must be the relocation for __tls_get_addr.
2360 ++RelI;
2361 auto &GetAddrRelocation = *RelI;
2362 processX86_64TLSRelocation(SectionID, Offset, RelType, Value, Addend,
2363 GetAddrRelocation);
2364 } else {
2365 processSimpleRelocation(SectionID, Offset, RelType, Value);
2366 }
2367 } else if (Arch == Triple::riscv32 || Arch == Triple::riscv64) {
2368 // *_LO12 relocation receive information about a symbol from the
2369 // corresponding *_HI20 relocation, so we have to collect this information
2370 // before resolving
2371 if (RelType == ELF::R_RISCV_GOT_HI20 ||
2372 RelType == ELF::R_RISCV_PCREL_HI20 ||
2373 RelType == ELF::R_RISCV_TPREL_HI20 ||
2374 RelType == ELF::R_RISCV_TLS_GD_HI20 ||
2375 RelType == ELF::R_RISCV_TLS_GOT_HI20) {
2376 RelocationEntry RE(SectionID, Offset, RelType, Addend);
2377 PendingRelocs.push_back(Elt: {Value, RE});
2378 }
2379 processSimpleRelocation(SectionID, Offset, RelType, Value);
2380 } else {
2381 if (Arch == Triple::x86) {
2382 Value.Addend += support::ulittle32_t::ref(
2383 computePlaceholderAddress(SectionID, Offset));
2384 }
2385 processSimpleRelocation(SectionID, Offset, RelType, Value);
2386 }
2387 return ++RelI;
2388}
2389
2390void RuntimeDyldELF::processX86_64GOTTPOFFRelocation(unsigned SectionID,
2391 uint64_t Offset,
2392 RelocationValueRef Value,
2393 int64_t Addend) {
2394 // Use the approach from "x86-64 Linker Optimizations" from the TLS spec
2395 // to replace the GOTTPOFF relocation with a TPOFF relocation. The spec
2396 // only mentions one optimization even though there are two different
2397 // code sequences for the Initial Exec TLS Model. We match the code to
2398 // find out which one was used.
2399
2400 // A possible TLS code sequence and its replacement
2401 struct CodeSequence {
2402 // The expected code sequence
2403 ArrayRef<uint8_t> ExpectedCodeSequence;
2404 // The negative offset of the GOTTPOFF relocation to the beginning of
2405 // the sequence
2406 uint64_t TLSSequenceOffset;
2407 // The new code sequence
2408 ArrayRef<uint8_t> NewCodeSequence;
2409 // The offset of the new TPOFF relocation
2410 uint64_t TpoffRelocationOffset;
2411 };
2412
2413 std::array<CodeSequence, 2> CodeSequences;
2414
2415 // Initial Exec Code Model Sequence
2416 {
2417 static const std::initializer_list<uint8_t> ExpectedCodeSequenceList = {
2418 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00,
2419 0x00, // mov %fs:0, %rax
2420 0x48, 0x03, 0x05, 0x00, 0x00, 0x00, 0x00 // add x@gotpoff(%rip),
2421 // %rax
2422 };
2423 CodeSequences[0].ExpectedCodeSequence =
2424 ArrayRef<uint8_t>(ExpectedCodeSequenceList);
2425 CodeSequences[0].TLSSequenceOffset = 12;
2426
2427 static const std::initializer_list<uint8_t> NewCodeSequenceList = {
2428 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0, %rax
2429 0x48, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00 // lea x@tpoff(%rax), %rax
2430 };
2431 CodeSequences[0].NewCodeSequence = ArrayRef<uint8_t>(NewCodeSequenceList);
2432 CodeSequences[0].TpoffRelocationOffset = 12;
2433 }
2434
2435 // Initial Exec Code Model Sequence, II
2436 {
2437 static const std::initializer_list<uint8_t> ExpectedCodeSequenceList = {
2438 0x48, 0x8b, 0x05, 0x00, 0x00, 0x00, 0x00, // mov x@gotpoff(%rip), %rax
2439 0x64, 0x48, 0x8b, 0x00, 0x00, 0x00, 0x00 // mov %fs:(%rax), %rax
2440 };
2441 CodeSequences[1].ExpectedCodeSequence =
2442 ArrayRef<uint8_t>(ExpectedCodeSequenceList);
2443 CodeSequences[1].TLSSequenceOffset = 3;
2444
2445 static const std::initializer_list<uint8_t> NewCodeSequenceList = {
2446 0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00, // 6 byte nop
2447 0x64, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:x@tpoff, %rax
2448 };
2449 CodeSequences[1].NewCodeSequence = ArrayRef<uint8_t>(NewCodeSequenceList);
2450 CodeSequences[1].TpoffRelocationOffset = 10;
2451 }
2452
2453 bool Resolved = false;
2454 auto &Section = Sections[SectionID];
2455 for (const auto &C : CodeSequences) {
2456 assert(C.ExpectedCodeSequence.size() == C.NewCodeSequence.size() &&
2457 "Old and new code sequences must have the same size");
2458
2459 if (Offset < C.TLSSequenceOffset ||
2460 (Offset - C.TLSSequenceOffset + C.NewCodeSequence.size()) >
2461 Section.getSize()) {
2462 // This can't be a matching sequence as it doesn't fit in the current
2463 // section
2464 continue;
2465 }
2466
2467 auto TLSSequenceStartOffset = Offset - C.TLSSequenceOffset;
2468 auto *TLSSequence = Section.getAddressWithOffset(OffsetBytes: TLSSequenceStartOffset);
2469 if (ArrayRef<uint8_t>(TLSSequence, C.ExpectedCodeSequence.size()) !=
2470 C.ExpectedCodeSequence) {
2471 continue;
2472 }
2473
2474 memcpy(dest: TLSSequence, src: C.NewCodeSequence.data(), n: C.NewCodeSequence.size());
2475
2476 // The original GOTTPOFF relocation has an addend as it is PC relative,
2477 // so it needs to be corrected. The TPOFF32 relocation is used as an
2478 // absolute value (which is an offset from %fs:0), so remove the addend
2479 // again.
2480 RelocationEntry RE(SectionID,
2481 TLSSequenceStartOffset + C.TpoffRelocationOffset,
2482 ELF::R_X86_64_TPOFF32, Value.Addend - Addend);
2483
2484 if (Value.SymbolName)
2485 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
2486 else
2487 addRelocationForSection(RE, SectionID: Value.SectionID);
2488
2489 Resolved = true;
2490 break;
2491 }
2492
2493 if (!Resolved) {
2494 // The GOTTPOFF relocation was not used in one of the sequences
2495 // described in the spec, so we can't optimize it to a TPOFF
2496 // relocation.
2497 uint64_t GOTOffset = allocateGOTEntries(no: 1);
2498 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: GOTOffset + Addend,
2499 Type: ELF::R_X86_64_PC32);
2500 RelocationEntry RE =
2501 computeGOTOffsetRE(GOTOffset, SymbolOffset: Value.Offset, Type: ELF::R_X86_64_TPOFF64);
2502 if (Value.SymbolName)
2503 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
2504 else
2505 addRelocationForSection(RE, SectionID: Value.SectionID);
2506 }
2507}
2508
2509void RuntimeDyldELF::processX86_64TLSRelocation(
2510 unsigned SectionID, uint64_t Offset, uint64_t RelType,
2511 RelocationValueRef Value, int64_t Addend,
2512 const RelocationRef &GetAddrRelocation) {
2513 // Since we are statically linking and have no additional DSOs, we can resolve
2514 // the relocation directly without using __tls_get_addr.
2515 // Use the approach from "x86-64 Linker Optimizations" from the TLS spec
2516 // to replace it with the Local Exec relocation variant.
2517
2518 // Find out whether the code was compiled with the large or small memory
2519 // model. For this we look at the next relocation which is the relocation
2520 // for the __tls_get_addr function. If it's a 32 bit relocation, it's the
2521 // small code model, with a 64 bit relocation it's the large code model.
2522 bool IsSmallCodeModel;
2523 // Is the relocation for the __tls_get_addr a PC-relative GOT relocation?
2524 bool IsGOTPCRel = false;
2525
2526 switch (GetAddrRelocation.getType()) {
2527 case ELF::R_X86_64_GOTPCREL:
2528 case ELF::R_X86_64_REX_GOTPCRELX:
2529 case ELF::R_X86_64_GOTPCRELX:
2530 IsGOTPCRel = true;
2531 [[fallthrough]];
2532 case ELF::R_X86_64_PLT32:
2533 IsSmallCodeModel = true;
2534 break;
2535 case ELF::R_X86_64_PLTOFF64:
2536 IsSmallCodeModel = false;
2537 break;
2538 default:
2539 report_fatal_error(
2540 reason: "invalid TLS relocations for General/Local Dynamic TLS Model: "
2541 "expected PLT or GOT relocation for __tls_get_addr function");
2542 }
2543
2544 // The negative offset to the start of the TLS code sequence relative to
2545 // the offset of the TLSGD/TLSLD relocation
2546 uint64_t TLSSequenceOffset;
2547 // The expected start of the code sequence
2548 ArrayRef<uint8_t> ExpectedCodeSequence;
2549 // The new TLS code sequence that will replace the existing code
2550 ArrayRef<uint8_t> NewCodeSequence;
2551
2552 if (RelType == ELF::R_X86_64_TLSGD) {
2553 // The offset of the new TPOFF32 relocation (offset starting from the
2554 // beginning of the whole TLS sequence)
2555 uint64_t TpoffRelocOffset;
2556
2557 if (IsSmallCodeModel) {
2558 if (!IsGOTPCRel) {
2559 static const std::initializer_list<uint8_t> CodeSequence = {
2560 0x66, // data16 (no-op prefix)
2561 0x48, 0x8d, 0x3d, 0x00, 0x00,
2562 0x00, 0x00, // lea <disp32>(%rip), %rdi
2563 0x66, 0x66, // two data16 prefixes
2564 0x48, // rex64 (no-op prefix)
2565 0xe8, 0x00, 0x00, 0x00, 0x00 // call __tls_get_addr@plt
2566 };
2567 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2568 TLSSequenceOffset = 4;
2569 } else {
2570 // This code sequence is not described in the TLS spec but gcc
2571 // generates it sometimes.
2572 static const std::initializer_list<uint8_t> CodeSequence = {
2573 0x66, // data16 (no-op prefix)
2574 0x48, 0x8d, 0x3d, 0x00, 0x00,
2575 0x00, 0x00, // lea <disp32>(%rip), %rdi
2576 0x66, // data16 prefix (no-op prefix)
2577 0x48, // rex64 (no-op prefix)
2578 0xff, 0x15, 0x00, 0x00, 0x00,
2579 0x00 // call *__tls_get_addr@gotpcrel(%rip)
2580 };
2581 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2582 TLSSequenceOffset = 4;
2583 }
2584
2585 // The replacement code for the small code model. It's the same for
2586 // both sequences.
2587 static const std::initializer_list<uint8_t> SmallSequence = {
2588 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00,
2589 0x00, // mov %fs:0, %rax
2590 0x48, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00 // lea x@tpoff(%rax),
2591 // %rax
2592 };
2593 NewCodeSequence = ArrayRef<uint8_t>(SmallSequence);
2594 TpoffRelocOffset = 12;
2595 } else {
2596 static const std::initializer_list<uint8_t> CodeSequence = {
2597 0x48, 0x8d, 0x3d, 0x00, 0x00, 0x00, 0x00, // lea <disp32>(%rip),
2598 // %rdi
2599 0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2600 0x00, // movabs $__tls_get_addr@pltoff, %rax
2601 0x48, 0x01, 0xd8, // add %rbx, %rax
2602 0xff, 0xd0 // call *%rax
2603 };
2604 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2605 TLSSequenceOffset = 3;
2606
2607 // The replacement code for the large code model
2608 static const std::initializer_list<uint8_t> LargeSequence = {
2609 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00,
2610 0x00, // mov %fs:0, %rax
2611 0x48, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00, // lea x@tpoff(%rax),
2612 // %rax
2613 0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00 // nopw 0x0(%rax,%rax,1)
2614 };
2615 NewCodeSequence = ArrayRef<uint8_t>(LargeSequence);
2616 TpoffRelocOffset = 12;
2617 }
2618
2619 // The TLSGD/TLSLD relocations are PC-relative, so they have an addend.
2620 // The new TPOFF32 relocations is used as an absolute offset from
2621 // %fs:0, so remove the TLSGD/TLSLD addend again.
2622 RelocationEntry RE(SectionID, Offset - TLSSequenceOffset + TpoffRelocOffset,
2623 ELF::R_X86_64_TPOFF32, Value.Addend - Addend);
2624 if (Value.SymbolName)
2625 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
2626 else
2627 addRelocationForSection(RE, SectionID: Value.SectionID);
2628 } else if (RelType == ELF::R_X86_64_TLSLD) {
2629 if (IsSmallCodeModel) {
2630 if (!IsGOTPCRel) {
2631 static const std::initializer_list<uint8_t> CodeSequence = {
2632 0x48, 0x8d, 0x3d, 0x00, 0x00, 0x00, // leaq <disp32>(%rip), %rdi
2633 0x00, 0xe8, 0x00, 0x00, 0x00, 0x00 // call __tls_get_addr@plt
2634 };
2635 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2636 TLSSequenceOffset = 3;
2637
2638 // The replacement code for the small code model
2639 static const std::initializer_list<uint8_t> SmallSequence = {
2640 0x66, 0x66, 0x66, // three data16 prefixes (no-op)
2641 0x64, 0x48, 0x8b, 0x04, 0x25,
2642 0x00, 0x00, 0x00, 0x00 // mov %fs:0, %rax
2643 };
2644 NewCodeSequence = ArrayRef<uint8_t>(SmallSequence);
2645 } else {
2646 // This code sequence is not described in the TLS spec but gcc
2647 // generates it sometimes.
2648 static const std::initializer_list<uint8_t> CodeSequence = {
2649 0x48, 0x8d, 0x3d, 0x00,
2650 0x00, 0x00, 0x00, // leaq <disp32>(%rip), %rdi
2651 0xff, 0x15, 0x00, 0x00,
2652 0x00, 0x00 // call
2653 // *__tls_get_addr@gotpcrel(%rip)
2654 };
2655 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2656 TLSSequenceOffset = 3;
2657
2658 // The replacement is code is just like above but it needs to be
2659 // one byte longer.
2660 static const std::initializer_list<uint8_t> SmallSequence = {
2661 0x0f, 0x1f, 0x40, 0x00, // 4 byte nop
2662 0x64, 0x48, 0x8b, 0x04, 0x25,
2663 0x00, 0x00, 0x00, 0x00 // mov %fs:0, %rax
2664 };
2665 NewCodeSequence = ArrayRef<uint8_t>(SmallSequence);
2666 }
2667 } else {
2668 // This is the same sequence as for the TLSGD sequence with the large
2669 // memory model above
2670 static const std::initializer_list<uint8_t> CodeSequence = {
2671 0x48, 0x8d, 0x3d, 0x00, 0x00, 0x00, 0x00, // lea <disp32>(%rip),
2672 // %rdi
2673 0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2674 0x48, // movabs $__tls_get_addr@pltoff, %rax
2675 0x01, 0xd8, // add %rbx, %rax
2676 0xff, 0xd0 // call *%rax
2677 };
2678 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2679 TLSSequenceOffset = 3;
2680
2681 // The replacement code for the large code model
2682 static const std::initializer_list<uint8_t> LargeSequence = {
2683 0x66, 0x66, 0x66, // three data16 prefixes (no-op)
2684 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00,
2685 0x00, // 10 byte nop
2686 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00 // mov %fs:0,%rax
2687 };
2688 NewCodeSequence = ArrayRef<uint8_t>(LargeSequence);
2689 }
2690 } else {
2691 llvm_unreachable("both TLS relocations handled above");
2692 }
2693
2694 assert(ExpectedCodeSequence.size() == NewCodeSequence.size() &&
2695 "Old and new code sequences must have the same size");
2696
2697 auto &Section = Sections[SectionID];
2698 if (Offset < TLSSequenceOffset ||
2699 (Offset - TLSSequenceOffset + NewCodeSequence.size()) >
2700 Section.getSize()) {
2701 report_fatal_error(reason: "unexpected end of section in TLS sequence");
2702 }
2703
2704 auto *TLSSequence = Section.getAddressWithOffset(OffsetBytes: Offset - TLSSequenceOffset);
2705 if (ArrayRef<uint8_t>(TLSSequence, ExpectedCodeSequence.size()) !=
2706 ExpectedCodeSequence) {
2707 report_fatal_error(
2708 reason: "invalid TLS sequence for Global/Local Dynamic TLS Model");
2709 }
2710
2711 memcpy(dest: TLSSequence, src: NewCodeSequence.data(), n: NewCodeSequence.size());
2712}
2713
2714size_t RuntimeDyldELF::getGOTEntrySize() {
2715 // We don't use the GOT in all of these cases, but it's essentially free
2716 // to put them all here.
2717 size_t Result = 0;
2718 switch (Arch) {
2719 case Triple::x86_64:
2720 case Triple::aarch64:
2721 case Triple::aarch64_be:
2722 case Triple::loongarch64:
2723 case Triple::ppc64:
2724 case Triple::ppc64le:
2725 case Triple::systemz:
2726 Result = sizeof(uint64_t);
2727 break;
2728 case Triple::x86:
2729 case Triple::arm:
2730 case Triple::ppc:
2731 case Triple::thumb:
2732 Result = sizeof(uint32_t);
2733 break;
2734 case Triple::mips:
2735 case Triple::mipsel:
2736 case Triple::mips64:
2737 case Triple::mips64el:
2738 if (IsMipsO32ABI || IsMipsN32ABI)
2739 Result = sizeof(uint32_t);
2740 else if (IsMipsN64ABI)
2741 Result = sizeof(uint64_t);
2742 else
2743 llvm_unreachable("Mips ABI not handled");
2744 break;
2745 default:
2746 llvm_unreachable("Unsupported CPU type!");
2747 }
2748 return Result;
2749}
2750
2751uint64_t RuntimeDyldELF::allocateGOTEntries(unsigned no) {
2752 if (GOTSectionID == 0) {
2753 GOTSectionID = Sections.size();
2754 // Reserve a section id. We'll allocate the section later
2755 // once we know the total size
2756 Sections.push_back(x: SectionEntry(".got", nullptr, 0, 0, 0));
2757 }
2758 uint64_t StartOffset = CurrentGOTIndex * getGOTEntrySize();
2759 CurrentGOTIndex += no;
2760 return StartOffset;
2761}
2762
2763uint64_t RuntimeDyldELF::findOrAllocGOTEntry(const RelocationValueRef &Value,
2764 unsigned GOTRelType) {
2765 auto E = GOTOffsetMap.insert(x: {Value, 0});
2766 if (E.second) {
2767 uint64_t GOTOffset = allocateGOTEntries(no: 1);
2768
2769 // Create relocation for newly created GOT entry
2770 RelocationEntry RE =
2771 computeGOTOffsetRE(GOTOffset, SymbolOffset: Value.Offset, Type: GOTRelType);
2772 if (Value.SymbolName)
2773 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
2774 else
2775 addRelocationForSection(RE, SectionID: Value.SectionID);
2776
2777 E.first->second = GOTOffset;
2778 }
2779
2780 return E.first->second;
2781}
2782
2783void RuntimeDyldELF::resolveGOTOffsetRelocation(unsigned SectionID,
2784 uint64_t Offset,
2785 uint64_t GOTOffset,
2786 uint32_t Type) {
2787 // Fill in the relative address of the GOT Entry into the stub
2788 RelocationEntry GOTRE(SectionID, Offset, Type, GOTOffset);
2789 addRelocationForSection(RE: GOTRE, SectionID: GOTSectionID);
2790}
2791
2792RelocationEntry RuntimeDyldELF::computeGOTOffsetRE(uint64_t GOTOffset,
2793 uint64_t SymbolOffset,
2794 uint32_t Type) {
2795 return RelocationEntry(GOTSectionID, GOTOffset, Type, SymbolOffset);
2796}
2797
2798void RuntimeDyldELF::processNewSymbol(const SymbolRef &ObjSymbol, SymbolTableEntry& Symbol) {
2799 // This should never return an error as `processNewSymbol` wouldn't have been
2800 // called if getFlags() returned an error before.
2801 auto ObjSymbolFlags = cantFail(ValOrErr: ObjSymbol.getFlags());
2802
2803 if (ObjSymbolFlags & SymbolRef::SF_Indirect) {
2804 if (IFuncStubSectionID == 0) {
2805 // Create a dummy section for the ifunc stubs. It will be actually
2806 // allocated in finalizeLoad() below.
2807 IFuncStubSectionID = Sections.size();
2808 Sections.push_back(
2809 x: SectionEntry(".text.__llvm_IFuncStubs", nullptr, 0, 0, 0));
2810 // First 64B are reserverd for the IFunc resolver
2811 IFuncStubOffset = 64;
2812 }
2813
2814 IFuncStubs.push_back(Elt: IFuncStub{.StubOffset: IFuncStubOffset, .OriginalSymbol: Symbol});
2815 // Modify the symbol so that it points to the ifunc stub instead of to the
2816 // resolver function.
2817 Symbol = SymbolTableEntry(IFuncStubSectionID, IFuncStubOffset,
2818 Symbol.getFlags());
2819 IFuncStubOffset += getMaxIFuncStubSize();
2820 }
2821}
2822
2823Error RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj,
2824 ObjSectionToIDMap &SectionMap) {
2825 if (IsMipsO32ABI)
2826 if (!PendingRelocs.empty())
2827 return make_error<RuntimeDyldError>(Args: "Can't find matching LO16 reloc");
2828
2829 // Create the IFunc stubs if necessary. This must be done before processing
2830 // the GOT entries, as the IFunc stubs may create some.
2831 if (IFuncStubSectionID != 0) {
2832 uint8_t *IFuncStubsAddr = MemMgr.allocateCodeSection(
2833 Size: IFuncStubOffset, Alignment: 1, SectionID: IFuncStubSectionID, SectionName: ".text.__llvm_IFuncStubs");
2834 if (!IFuncStubsAddr)
2835 return make_error<RuntimeDyldError>(
2836 Args: "Unable to allocate memory for IFunc stubs!");
2837 Sections[IFuncStubSectionID] =
2838 SectionEntry(".text.__llvm_IFuncStubs", IFuncStubsAddr, IFuncStubOffset,
2839 IFuncStubOffset, 0);
2840
2841 createIFuncResolver(Addr: IFuncStubsAddr);
2842
2843 LLVM_DEBUG(dbgs() << "Creating IFunc stubs SectionID: "
2844 << IFuncStubSectionID << " Addr: "
2845 << Sections[IFuncStubSectionID].getAddress() << '\n');
2846 for (auto &IFuncStub : IFuncStubs) {
2847 auto &Symbol = IFuncStub.OriginalSymbol;
2848 LLVM_DEBUG(dbgs() << "\tSectionID: " << Symbol.getSectionID()
2849 << " Offset: " << format("%p", Symbol.getOffset())
2850 << " IFuncStubOffset: "
2851 << format("%p\n", IFuncStub.StubOffset));
2852 createIFuncStub(IFuncStubSectionID, IFuncResolverOffset: 0, IFuncStubOffset: IFuncStub.StubOffset,
2853 IFuncSectionID: Symbol.getSectionID(), IFuncOffset: Symbol.getOffset());
2854 }
2855
2856 IFuncStubSectionID = 0;
2857 IFuncStubOffset = 0;
2858 IFuncStubs.clear();
2859 }
2860
2861 // If necessary, allocate the global offset table
2862 if (GOTSectionID != 0) {
2863 // Allocate memory for the section
2864 size_t TotalSize = CurrentGOTIndex * getGOTEntrySize();
2865 uint8_t *Addr = MemMgr.allocateDataSection(Size: TotalSize, Alignment: getGOTEntrySize(),
2866 SectionID: GOTSectionID, SectionName: ".got", IsReadOnly: false);
2867 if (!Addr)
2868 return make_error<RuntimeDyldError>(Args: "Unable to allocate memory for GOT!");
2869
2870 Sections[GOTSectionID] =
2871 SectionEntry(".got", Addr, TotalSize, TotalSize, 0);
2872
2873 // For now, initialize all GOT entries to zero. We'll fill them in as
2874 // needed when GOT-based relocations are applied.
2875 memset(s: Addr, c: 0, n: TotalSize);
2876 if (IsMipsN32ABI || IsMipsN64ABI) {
2877 // To correctly resolve Mips GOT relocations, we need a mapping from
2878 // object's sections to GOTs.
2879 for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end();
2880 SI != SE; ++SI) {
2881 if (!SI->relocations().empty()) {
2882 Expected<section_iterator> RelSecOrErr = SI->getRelocatedSection();
2883 if (!RelSecOrErr)
2884 return make_error<RuntimeDyldError>(
2885 Args: toString(E: RelSecOrErr.takeError()));
2886
2887 section_iterator RelocatedSection = *RelSecOrErr;
2888 ObjSectionToIDMap::iterator i = SectionMap.find(x: *RelocatedSection);
2889 assert(i != SectionMap.end());
2890 SectionToGOTMap[i->second] = GOTSectionID;
2891 }
2892 }
2893 GOTSymbolOffsets.clear();
2894 }
2895 }
2896
2897 // Look for and record the EH frame section.
2898 ObjSectionToIDMap::iterator i, e;
2899 for (i = SectionMap.begin(), e = SectionMap.end(); i != e; ++i) {
2900 const SectionRef &Section = i->first;
2901
2902 StringRef Name;
2903 Expected<StringRef> NameOrErr = Section.getName();
2904 if (NameOrErr)
2905 Name = *NameOrErr;
2906 else
2907 consumeError(Err: NameOrErr.takeError());
2908
2909 if (Name == ".eh_frame") {
2910 UnregisteredEHFrameSections.push_back(Elt: i->second);
2911 break;
2912 }
2913 }
2914
2915 GOTOffsetMap.clear();
2916 GOTSectionID = 0;
2917 CurrentGOTIndex = 0;
2918
2919 return Error::success();
2920}
2921
2922bool RuntimeDyldELF::isCompatibleFile(const object::ObjectFile &Obj) const {
2923 return Obj.isELF();
2924}
2925
2926void RuntimeDyldELF::createIFuncResolver(uint8_t *Addr) const {
2927 if (Arch == Triple::x86_64) {
2928 // The adddres of the GOT1 entry is in %r11, the GOT2 entry is in %r11+8
2929 // (see createIFuncStub() for details)
2930 // The following code first saves all registers that contain the original
2931 // function arguments as those registers are not saved by the resolver
2932 // function. %r11 is saved as well so that the GOT2 entry can be updated
2933 // afterwards. Then it calls the actual IFunc resolver function whose
2934 // address is stored in GOT2. After the resolver function returns, all
2935 // saved registers are restored and the return value is written to GOT1.
2936 // Finally, jump to the now resolved function.
2937 // clang-format off
2938 const uint8_t StubCode[] = {
2939 0x57, // push %rdi
2940 0x56, // push %rsi
2941 0x52, // push %rdx
2942 0x51, // push %rcx
2943 0x41, 0x50, // push %r8
2944 0x41, 0x51, // push %r9
2945 0x41, 0x53, // push %r11
2946 0x41, 0xff, 0x53, 0x08, // call *0x8(%r11)
2947 0x41, 0x5b, // pop %r11
2948 0x41, 0x59, // pop %r9
2949 0x41, 0x58, // pop %r8
2950 0x59, // pop %rcx
2951 0x5a, // pop %rdx
2952 0x5e, // pop %rsi
2953 0x5f, // pop %rdi
2954 0x49, 0x89, 0x03, // mov %rax,(%r11)
2955 0xff, 0xe0 // jmp *%rax
2956 };
2957 // clang-format on
2958 static_assert(sizeof(StubCode) <= 64,
2959 "maximum size of the IFunc resolver is 64B");
2960 memcpy(dest: Addr, src: StubCode, n: sizeof(StubCode));
2961 } else {
2962 report_fatal_error(
2963 reason: "IFunc resolver is not supported for target architecture");
2964 }
2965}
2966
2967void RuntimeDyldELF::createIFuncStub(unsigned IFuncStubSectionID,
2968 uint64_t IFuncResolverOffset,
2969 uint64_t IFuncStubOffset,
2970 unsigned IFuncSectionID,
2971 uint64_t IFuncOffset) {
2972 auto &IFuncStubSection = Sections[IFuncStubSectionID];
2973 auto *Addr = IFuncStubSection.getAddressWithOffset(OffsetBytes: IFuncStubOffset);
2974
2975 if (Arch == Triple::x86_64) {
2976 // The first instruction loads a PC-relative address into %r11 which is a
2977 // GOT entry for this stub. This initially contains the address to the
2978 // IFunc resolver. We can use %r11 here as it's caller saved but not used
2979 // to pass any arguments. In fact, x86_64 ABI even suggests using %r11 for
2980 // code in the PLT. The IFunc resolver will use %r11 to update the GOT
2981 // entry.
2982 //
2983 // The next instruction just jumps to the address contained in the GOT
2984 // entry. As mentioned above, we do this two-step jump by first setting
2985 // %r11 so that the IFunc resolver has access to it.
2986 //
2987 // The IFunc resolver of course also needs to know the actual address of
2988 // the actual IFunc resolver function. This will be stored in a GOT entry
2989 // right next to the first one for this stub. So, the IFunc resolver will
2990 // be able to call it with %r11+8.
2991 //
2992 // In total, two adjacent GOT entries (+relocation) and one additional
2993 // relocation are required:
2994 // GOT1: Address of the IFunc resolver.
2995 // GOT2: Address of the IFunc resolver function.
2996 // IFuncStubOffset+3: 32-bit PC-relative address of GOT1.
2997 uint64_t GOT1 = allocateGOTEntries(no: 2);
2998 uint64_t GOT2 = GOT1 + getGOTEntrySize();
2999
3000 RelocationEntry RE1(GOTSectionID, GOT1, ELF::R_X86_64_64,
3001 IFuncResolverOffset, {});
3002 addRelocationForSection(RE: RE1, SectionID: IFuncStubSectionID);
3003 RelocationEntry RE2(GOTSectionID, GOT2, ELF::R_X86_64_64, IFuncOffset, {});
3004 addRelocationForSection(RE: RE2, SectionID: IFuncSectionID);
3005
3006 const uint8_t StubCode[] = {
3007 0x4c, 0x8d, 0x1d, 0x00, 0x00, 0x00, 0x00, // leaq 0x0(%rip),%r11
3008 0x41, 0xff, 0x23 // jmpq *(%r11)
3009 };
3010 assert(sizeof(StubCode) <= getMaxIFuncStubSize() &&
3011 "IFunc stub size must not exceed getMaxIFuncStubSize()");
3012 memcpy(dest: Addr, src: StubCode, n: sizeof(StubCode));
3013
3014 // The PC-relative value starts 4 bytes from the end of the leaq
3015 // instruction, so the addend is -4.
3016 resolveGOTOffsetRelocation(SectionID: IFuncStubSectionID, Offset: IFuncStubOffset + 3,
3017 GOTOffset: GOT1 - 4, Type: ELF::R_X86_64_PC32);
3018 } else {
3019 report_fatal_error(reason: "IFunc stub is not supported for target architecture");
3020 }
3021}
3022
3023unsigned RuntimeDyldELF::getMaxIFuncStubSize() const {
3024 if (Arch == Triple::x86_64) {
3025 return 10;
3026 }
3027 return 0;
3028}
3029
3030bool RuntimeDyldELF::relocationNeedsGot(const RelocationRef &R) const {
3031 unsigned RelTy = R.getType();
3032 if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be)
3033 return RelTy == ELF::R_AARCH64_ADR_GOT_PAGE ||
3034 RelTy == ELF::R_AARCH64_LD64_GOT_LO12_NC;
3035
3036 if (Arch == Triple::loongarch64)
3037 return RelTy == ELF::R_LARCH_GOT_PC_HI20 ||
3038 RelTy == ELF::R_LARCH_GOT_PC_LO12 ||
3039 RelTy == ELF::R_LARCH_GOT64_PC_HI12 ||
3040 RelTy == ELF::R_LARCH_GOT64_PC_LO20;
3041
3042 if (Arch == Triple::x86_64)
3043 return RelTy == ELF::R_X86_64_GOTPCREL ||
3044 RelTy == ELF::R_X86_64_GOTPCRELX ||
3045 RelTy == ELF::R_X86_64_GOT64 ||
3046 RelTy == ELF::R_X86_64_REX_GOTPCRELX;
3047 return false;
3048}
3049
3050bool RuntimeDyldELF::relocationNeedsStub(const RelocationRef &R) const {
3051 if (Arch != Triple::x86_64)
3052 return true; // Conservative answer
3053
3054 switch (R.getType()) {
3055 default:
3056 return true; // Conservative answer
3057
3058
3059 case ELF::R_X86_64_GOTPCREL:
3060 case ELF::R_X86_64_GOTPCRELX:
3061 case ELF::R_X86_64_REX_GOTPCRELX:
3062 case ELF::R_X86_64_GOTPC64:
3063 case ELF::R_X86_64_GOT64:
3064 case ELF::R_X86_64_GOTOFF64:
3065 case ELF::R_X86_64_PC32:
3066 case ELF::R_X86_64_PC64:
3067 case ELF::R_X86_64_64:
3068 // We know that these reloation types won't need a stub function. This list
3069 // can be extended as needed.
3070 return false;
3071 }
3072}
3073
3074} // namespace llvm
3075