1//===-- RuntimeDyldELF.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Implementation of ELF support for the MC-JIT runtime dynamic linker.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RuntimeDyldELF.h"
14#include "RuntimeDyldCheckerImpl.h"
15#include "Targets/RuntimeDyldELFMips.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/BinaryFormat/ELF.h"
19#include "llvm/Object/ELFObjectFile.h"
20#include "llvm/Object/ObjectFile.h"
21#include "llvm/Support/Endian.h"
22#include "llvm/Support/MemoryBuffer.h"
23#include "llvm/TargetParser/Triple.h"
24
25using namespace llvm;
26using namespace llvm::object;
27using namespace llvm::support::endian;
28
29#define DEBUG_TYPE "dyld"
30
31static void or32le(void *P, int32_t V) { write32le(P, V: read32le(P) | V); }
32
33static void or32AArch64Imm(void *L, uint64_t Imm) {
34 or32le(P: L, V: (Imm & 0xFFF) << 10);
35}
36
37template <class T> static void write(bool isBE, void *P, T V) {
38 isBE ? write<T, llvm::endianness::big>(P, V)
39 : write<T, llvm::endianness::little>(P, V);
40}
41
42static void write32AArch64Addr(void *L, uint64_t Imm) {
43 uint32_t ImmLo = (Imm & 0x3) << 29;
44 uint32_t ImmHi = (Imm & 0x1FFFFC) << 3;
45 uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3);
46 write32le(P: L, V: (read32le(P: L) & ~Mask) | ImmLo | ImmHi);
47}
48
49// Return the bits [Start, End] from Val shifted Start bits.
50// For instance, getBits(0xF0, 4, 8) returns 0xF.
51static uint64_t getBits(uint64_t Val, int Start, int End) {
52 uint64_t Mask = ((uint64_t)1 << (End + 1 - Start)) - 1;
53 return (Val >> Start) & Mask;
54}
55
56namespace {
57
58template <class ELFT> class DyldELFObject : public ELFObjectFile<ELFT> {
59 LLVM_ELF_IMPORT_TYPES_ELFT(ELFT)
60
61 typedef typename ELFT::uint addr_type;
62
63 DyldELFObject(ELFObjectFile<ELFT> &&Obj);
64
65public:
66 static Expected<std::unique_ptr<DyldELFObject>>
67 create(MemoryBufferRef Wrapper);
68
69 void updateSectionAddress(const SectionRef &Sec, uint64_t Addr);
70
71 void updateSymbolAddress(const SymbolRef &SymRef, uint64_t Addr);
72
73 // Methods for type inquiry through isa, cast and dyn_cast
74 static bool classof(const Binary *v) {
75 return (isa<ELFObjectFile<ELFT>>(v) &&
76 classof(cast<ELFObjectFile<ELFT>>(v)));
77 }
78 static bool classof(const ELFObjectFile<ELFT> *v) {
79 return v->isDyldType();
80 }
81};
82
83
84
85// The MemoryBuffer passed into this constructor is just a wrapper around the
86// actual memory. Ultimately, the Binary parent class will take ownership of
87// this MemoryBuffer object but not the underlying memory.
88template <class ELFT>
89DyldELFObject<ELFT>::DyldELFObject(ELFObjectFile<ELFT> &&Obj)
90 : ELFObjectFile<ELFT>(std::move(Obj)) {
91 this->isDyldELFObject = true;
92}
93
94template <class ELFT>
95Expected<std::unique_ptr<DyldELFObject<ELFT>>>
96DyldELFObject<ELFT>::create(MemoryBufferRef Wrapper) {
97 auto Obj = ELFObjectFile<ELFT>::create(Wrapper);
98 if (auto E = Obj.takeError())
99 return std::move(E);
100 std::unique_ptr<DyldELFObject<ELFT>> Ret(
101 new DyldELFObject<ELFT>(std::move(*Obj)));
102 return std::move(Ret);
103}
104
105template <class ELFT>
106void DyldELFObject<ELFT>::updateSectionAddress(const SectionRef &Sec,
107 uint64_t Addr) {
108 DataRefImpl ShdrRef = Sec.getRawDataRefImpl();
109 Elf_Shdr *shdr =
110 const_cast<Elf_Shdr *>(reinterpret_cast<const Elf_Shdr *>(ShdrRef.p));
111
112 // This assumes the address passed in matches the target address bitness
113 // The template-based type cast handles everything else.
114 shdr->sh_addr = static_cast<addr_type>(Addr);
115}
116
117template <class ELFT>
118void DyldELFObject<ELFT>::updateSymbolAddress(const SymbolRef &SymRef,
119 uint64_t Addr) {
120
121 Elf_Sym *sym = const_cast<Elf_Sym *>(
122 ELFObjectFile<ELFT>::getSymbol(SymRef.getRawDataRefImpl()));
123
124 // This assumes the address passed in matches the target address bitness
125 // The template-based type cast handles everything else.
126 sym->st_value = static_cast<addr_type>(Addr);
127}
128
129class LoadedELFObjectInfo final
130 : public LoadedObjectInfoHelper<LoadedELFObjectInfo,
131 RuntimeDyld::LoadedObjectInfo> {
132public:
133 LoadedELFObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap)
134 : LoadedObjectInfoHelper(RTDyld, std::move(ObjSecToIDMap)) {}
135
136 OwningBinary<ObjectFile>
137 getObjectForDebug(const ObjectFile &Obj) const override;
138};
139
140template <typename ELFT>
141static Expected<std::unique_ptr<DyldELFObject<ELFT>>>
142createRTDyldELFObject(MemoryBufferRef Buffer, const ObjectFile &SourceObject,
143 const LoadedELFObjectInfo &L) {
144 typedef typename ELFT::Shdr Elf_Shdr;
145 typedef typename ELFT::uint addr_type;
146
147 Expected<std::unique_ptr<DyldELFObject<ELFT>>> ObjOrErr =
148 DyldELFObject<ELFT>::create(Buffer);
149 if (Error E = ObjOrErr.takeError())
150 return std::move(E);
151
152 std::unique_ptr<DyldELFObject<ELFT>> Obj = std::move(*ObjOrErr);
153
154 // Iterate over all sections in the object.
155 auto SI = SourceObject.section_begin();
156 for (const auto &Sec : Obj->sections()) {
157 Expected<StringRef> NameOrErr = Sec.getName();
158 if (!NameOrErr) {
159 consumeError(Err: NameOrErr.takeError());
160 continue;
161 }
162
163 if (*NameOrErr != "") {
164 DataRefImpl ShdrRef = Sec.getRawDataRefImpl();
165 Elf_Shdr *shdr = const_cast<Elf_Shdr *>(
166 reinterpret_cast<const Elf_Shdr *>(ShdrRef.p));
167
168 if (uint64_t SecLoadAddr = L.getSectionLoadAddress(Sec: *SI)) {
169 // This assumes that the address passed in matches the target address
170 // bitness. The template-based type cast handles everything else.
171 shdr->sh_addr = static_cast<addr_type>(SecLoadAddr);
172 }
173 }
174 ++SI;
175 }
176
177 return std::move(Obj);
178}
179
180static OwningBinary<ObjectFile>
181createELFDebugObject(const ObjectFile &Obj, const LoadedELFObjectInfo &L) {
182 assert(Obj.isELF() && "Not an ELF object file.");
183
184 std::unique_ptr<MemoryBuffer> Buffer =
185 MemoryBuffer::getMemBufferCopy(InputData: Obj.getData(), BufferName: Obj.getFileName());
186
187 Expected<std::unique_ptr<ObjectFile>> DebugObj(nullptr);
188 handleAllErrors(E: DebugObj.takeError());
189 if (Obj.getBytesInAddress() == 4 && Obj.isLittleEndian())
190 DebugObj =
191 createRTDyldELFObject<ELF32LE>(Buffer: Buffer->getMemBufferRef(), SourceObject: Obj, L);
192 else if (Obj.getBytesInAddress() == 4 && !Obj.isLittleEndian())
193 DebugObj =
194 createRTDyldELFObject<ELF32BE>(Buffer: Buffer->getMemBufferRef(), SourceObject: Obj, L);
195 else if (Obj.getBytesInAddress() == 8 && !Obj.isLittleEndian())
196 DebugObj =
197 createRTDyldELFObject<ELF64BE>(Buffer: Buffer->getMemBufferRef(), SourceObject: Obj, L);
198 else if (Obj.getBytesInAddress() == 8 && Obj.isLittleEndian())
199 DebugObj =
200 createRTDyldELFObject<ELF64LE>(Buffer: Buffer->getMemBufferRef(), SourceObject: Obj, L);
201 else
202 llvm_unreachable("Unexpected ELF format");
203
204 handleAllErrors(E: DebugObj.takeError());
205 return OwningBinary<ObjectFile>(std::move(*DebugObj), std::move(Buffer));
206}
207
208OwningBinary<ObjectFile>
209LoadedELFObjectInfo::getObjectForDebug(const ObjectFile &Obj) const {
210 return createELFDebugObject(Obj, L: *this);
211}
212
213} // anonymous namespace
214
215namespace llvm {
216
217RuntimeDyldELF::RuntimeDyldELF(RuntimeDyld::MemoryManager &MemMgr,
218 JITSymbolResolver &Resolver)
219 : RuntimeDyldImpl(MemMgr, Resolver), GOTSectionID(0), CurrentGOTIndex(0) {}
220RuntimeDyldELF::~RuntimeDyldELF() = default;
221
222void RuntimeDyldELF::registerEHFrames() {
223 for (SID EHFrameSID : UnregisteredEHFrameSections) {
224 uint8_t *EHFrameAddr = Sections[EHFrameSID].getAddress();
225 uint64_t EHFrameLoadAddr = Sections[EHFrameSID].getLoadAddress();
226 size_t EHFrameSize = Sections[EHFrameSID].getSize();
227 MemMgr.registerEHFrames(Addr: EHFrameAddr, LoadAddr: EHFrameLoadAddr, Size: EHFrameSize);
228 }
229 UnregisteredEHFrameSections.clear();
230}
231
232std::unique_ptr<RuntimeDyldELF>
233llvm::RuntimeDyldELF::create(Triple::ArchType Arch,
234 RuntimeDyld::MemoryManager &MemMgr,
235 JITSymbolResolver &Resolver) {
236 switch (Arch) {
237 default:
238 return std::make_unique<RuntimeDyldELF>(args&: MemMgr, args&: Resolver);
239 case Triple::mips:
240 case Triple::mipsel:
241 case Triple::mips64:
242 case Triple::mips64el:
243 return std::make_unique<RuntimeDyldELFMips>(args&: MemMgr, args&: Resolver);
244 }
245}
246
247std::unique_ptr<RuntimeDyld::LoadedObjectInfo>
248RuntimeDyldELF::loadObject(const object::ObjectFile &O) {
249 if (auto ObjSectionToIDOrErr = loadObjectImpl(Obj: O))
250 return std::make_unique<LoadedELFObjectInfo>(args&: *this, args&: *ObjSectionToIDOrErr);
251 else {
252 HasError = true;
253 raw_string_ostream ErrStream(ErrorStr);
254 logAllUnhandledErrors(E: ObjSectionToIDOrErr.takeError(), OS&: ErrStream);
255 return nullptr;
256 }
257}
258
259void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section,
260 uint64_t Offset, uint64_t Value,
261 uint32_t Type, int64_t Addend,
262 uint64_t SymOffset) {
263 switch (Type) {
264 default:
265 report_fatal_error(reason: "Relocation type not implemented yet!");
266 break;
267 case ELF::R_X86_64_NONE:
268 break;
269 case ELF::R_X86_64_8: {
270 Value += Addend;
271 assert((int64_t)Value <= INT8_MAX && (int64_t)Value >= INT8_MIN);
272 uint8_t TruncatedAddr = (Value & 0xFF);
273 *Section.getAddressWithOffset(OffsetBytes: Offset) = TruncatedAddr;
274 LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at "
275 << format("%p\n", Section.getAddressWithOffset(Offset)));
276 break;
277 }
278 case ELF::R_X86_64_16: {
279 Value += Addend;
280 assert((int64_t)Value <= INT16_MAX && (int64_t)Value >= INT16_MIN);
281 uint16_t TruncatedAddr = (Value & 0xFFFF);
282 support::ulittle16_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
283 TruncatedAddr;
284 LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at "
285 << format("%p\n", Section.getAddressWithOffset(Offset)));
286 break;
287 }
288 case ELF::R_X86_64_64: {
289 support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
290 Value + Addend;
291 LLVM_DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at "
292 << format("%p\n", Section.getAddressWithOffset(Offset)));
293 break;
294 }
295 case ELF::R_X86_64_32:
296 case ELF::R_X86_64_32S: {
297 Value += Addend;
298 assert((Type == ELF::R_X86_64_32 && (Value <= UINT32_MAX)) ||
299 (Type == ELF::R_X86_64_32S &&
300 ((int64_t)Value <= INT32_MAX && (int64_t)Value >= INT32_MIN)));
301 uint32_t TruncatedAddr = (Value & 0xFFFFFFFF);
302 support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
303 TruncatedAddr;
304 LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at "
305 << format("%p\n", Section.getAddressWithOffset(Offset)));
306 break;
307 }
308 case ELF::R_X86_64_PC8: {
309 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
310 int64_t RealOffset = Value + Addend - FinalAddress;
311 assert(isInt<8>(RealOffset));
312 int8_t TruncOffset = (RealOffset & 0xFF);
313 Section.getAddress()[Offset] = TruncOffset;
314 break;
315 }
316 case ELF::R_X86_64_PC32: {
317 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
318 int64_t RealOffset = Value + Addend - FinalAddress;
319 assert(isInt<32>(RealOffset));
320 int32_t TruncOffset = (RealOffset & 0xFFFFFFFF);
321 support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
322 TruncOffset;
323 break;
324 }
325 case ELF::R_X86_64_PC64: {
326 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
327 int64_t RealOffset = Value + Addend - FinalAddress;
328 support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
329 RealOffset;
330 LLVM_DEBUG(dbgs() << "Writing " << format("%p", RealOffset) << " at "
331 << format("%p\n", FinalAddress));
332 break;
333 }
334 case ELF::R_X86_64_GOTOFF64: {
335 // Compute Value - GOTBase.
336 uint64_t GOTBase = 0;
337 for (const auto &Section : Sections) {
338 if (Section.getName() == ".got") {
339 GOTBase = Section.getLoadAddressWithOffset(OffsetBytes: 0);
340 break;
341 }
342 }
343 assert(GOTBase != 0 && "missing GOT");
344 int64_t GOTOffset = Value - GOTBase + Addend;
345 support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) = GOTOffset;
346 break;
347 }
348 case ELF::R_X86_64_DTPMOD64: {
349 // We only have one DSO, so the module id is always 1.
350 support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) = 1;
351 break;
352 }
353 case ELF::R_X86_64_DTPOFF64:
354 case ELF::R_X86_64_TPOFF64: {
355 // DTPOFF64 should resolve to the offset in the TLS block, TPOFF64 to the
356 // offset in the *initial* TLS block. Since we are statically linking, all
357 // TLS blocks already exist in the initial block, so resolve both
358 // relocations equally.
359 support::ulittle64_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
360 Value + Addend;
361 break;
362 }
363 case ELF::R_X86_64_DTPOFF32:
364 case ELF::R_X86_64_TPOFF32: {
365 // As for the (D)TPOFF64 relocations above, both DTPOFF32 and TPOFF32 can
366 // be resolved equally.
367 int64_t RealValue = Value + Addend;
368 assert(RealValue >= INT32_MIN && RealValue <= INT32_MAX);
369 int32_t TruncValue = RealValue;
370 support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
371 TruncValue;
372 break;
373 }
374 }
375}
376
377void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section,
378 uint64_t Offset, uint32_t Value,
379 uint32_t Type, int32_t Addend) {
380 switch (Type) {
381 case ELF::R_386_32: {
382 support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
383 Value + Addend;
384 break;
385 }
386 // Handle R_386_PLT32 like R_386_PC32 since it should be able to
387 // reach any 32 bit address.
388 case ELF::R_386_PLT32:
389 case ELF::R_386_PC32: {
390 uint32_t FinalAddress =
391 Section.getLoadAddressWithOffset(OffsetBytes: Offset) & 0xFFFFFFFF;
392 uint32_t RealOffset = Value + Addend - FinalAddress;
393 support::ulittle32_t::ref(Section.getAddressWithOffset(OffsetBytes: Offset)) =
394 RealOffset;
395 break;
396 }
397 default:
398 // There are other relocation types, but it appears these are the
399 // only ones currently used by the LLVM ELF object writer
400 report_fatal_error(reason: "Relocation type not implemented yet!");
401 break;
402 }
403}
404
405void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section,
406 uint64_t Offset, uint64_t Value,
407 uint32_t Type, int64_t Addend) {
408 uint32_t *TargetPtr =
409 reinterpret_cast<uint32_t *>(Section.getAddressWithOffset(OffsetBytes: Offset));
410 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
411 // Data should use target endian. Code should always use little endian.
412 bool isBE = Arch == Triple::aarch64_be;
413
414 LLVM_DEBUG(dbgs() << "resolveAArch64Relocation, LocalAddress: 0x"
415 << format("%llx", Section.getAddressWithOffset(Offset))
416 << " FinalAddress: 0x" << format("%llx", FinalAddress)
417 << " Value: 0x" << format("%llx", Value) << " Type: 0x"
418 << format("%x", Type) << " Addend: 0x"
419 << format("%llx", Addend) << "\n");
420
421 switch (Type) {
422 default:
423 report_fatal_error(reason: "Relocation type not implemented yet!");
424 break;
425 case ELF::R_AARCH64_NONE:
426 break;
427 case ELF::R_AARCH64_ABS16: {
428 uint64_t Result = Value + Addend;
429 assert(Result == static_cast<uint64_t>(llvm::SignExtend64(Result, 16)) ||
430 (Result >> 16) == 0);
431 write(isBE, P: TargetPtr, V: static_cast<uint16_t>(Result & 0xffffU));
432 break;
433 }
434 case ELF::R_AARCH64_ABS32: {
435 uint64_t Result = Value + Addend;
436 assert(Result == static_cast<uint64_t>(llvm::SignExtend64(Result, 32)) ||
437 (Result >> 32) == 0);
438 write(isBE, P: TargetPtr, V: static_cast<uint32_t>(Result & 0xffffffffU));
439 break;
440 }
441 case ELF::R_AARCH64_ABS64:
442 write(isBE, P: TargetPtr, V: Value + Addend);
443 break;
444 case ELF::R_AARCH64_PLT32: {
445 uint64_t Result = Value + Addend - FinalAddress;
446 assert(static_cast<int64_t>(Result) >= INT32_MIN &&
447 static_cast<int64_t>(Result) <= INT32_MAX);
448 write(isBE, P: TargetPtr, V: static_cast<uint32_t>(Result));
449 break;
450 }
451 case ELF::R_AARCH64_PREL16: {
452 uint64_t Result = Value + Addend - FinalAddress;
453 assert(static_cast<int64_t>(Result) >= INT16_MIN &&
454 static_cast<int64_t>(Result) <= UINT16_MAX);
455 write(isBE, P: TargetPtr, V: static_cast<uint16_t>(Result & 0xffffU));
456 break;
457 }
458 case ELF::R_AARCH64_PREL32: {
459 uint64_t Result = Value + Addend - FinalAddress;
460 assert(static_cast<int64_t>(Result) >= INT32_MIN &&
461 static_cast<int64_t>(Result) <= UINT32_MAX);
462 write(isBE, P: TargetPtr, V: static_cast<uint32_t>(Result & 0xffffffffU));
463 break;
464 }
465 case ELF::R_AARCH64_PREL64:
466 write(isBE, P: TargetPtr, V: Value + Addend - FinalAddress);
467 break;
468 case ELF::R_AARCH64_CONDBR19: {
469 uint64_t BranchImm = Value + Addend - FinalAddress;
470
471 assert(isInt<21>(BranchImm));
472 *TargetPtr &= 0xff00001fU;
473 // Immediate:20:2 goes in bits 23:5 of Bcc, CBZ, CBNZ
474 or32le(P: TargetPtr, V: (BranchImm & 0x001FFFFC) << 3);
475 break;
476 }
477 case ELF::R_AARCH64_TSTBR14: {
478 uint64_t BranchImm = Value + Addend - FinalAddress;
479
480 assert(isInt<16>(BranchImm));
481
482 uint32_t RawInstr = *(support::little32_t *)TargetPtr;
483 *(support::little32_t *)TargetPtr = RawInstr & 0xfff8001fU;
484
485 // Immediate:15:2 goes in bits 18:5 of TBZ, TBNZ
486 or32le(P: TargetPtr, V: (BranchImm & 0x0000FFFC) << 3);
487 break;
488 }
489 case ELF::R_AARCH64_CALL26: // fallthrough
490 case ELF::R_AARCH64_JUMP26: {
491 // Operation: S+A-P. Set Call or B immediate value to bits fff_fffc of the
492 // calculation.
493 uint64_t BranchImm = Value + Addend - FinalAddress;
494
495 // "Check that -2^27 <= result < 2^27".
496 assert(isInt<28>(BranchImm));
497 or32le(P: TargetPtr, V: (BranchImm & 0x0FFFFFFC) >> 2);
498 break;
499 }
500 case ELF::R_AARCH64_MOVW_UABS_G3:
501 or32le(P: TargetPtr, V: ((Value + Addend) & 0xFFFF000000000000) >> 43);
502 break;
503 case ELF::R_AARCH64_MOVW_UABS_G2_NC:
504 or32le(P: TargetPtr, V: ((Value + Addend) & 0xFFFF00000000) >> 27);
505 break;
506 case ELF::R_AARCH64_MOVW_UABS_G1_NC:
507 or32le(P: TargetPtr, V: ((Value + Addend) & 0xFFFF0000) >> 11);
508 break;
509 case ELF::R_AARCH64_MOVW_UABS_G0_NC:
510 or32le(P: TargetPtr, V: ((Value + Addend) & 0xFFFF) << 5);
511 break;
512 case ELF::R_AARCH64_ADR_PREL_PG_HI21: {
513 // Operation: Page(S+A) - Page(P)
514 uint64_t Result =
515 ((Value + Addend) & ~0xfffULL) - (FinalAddress & ~0xfffULL);
516
517 // Check that -2^32 <= X < 2^32
518 assert(isInt<33>(Result) && "overflow check failed for relocation");
519
520 // Immediate goes in bits 30:29 + 5:23 of ADRP instruction, taken
521 // from bits 32:12 of X.
522 write32AArch64Addr(L: TargetPtr, Imm: Result >> 12);
523 break;
524 }
525 case ELF::R_AARCH64_ADD_ABS_LO12_NC:
526 // Operation: S + A
527 // Immediate goes in bits 21:10 of LD/ST instruction, taken
528 // from bits 11:0 of X
529 or32AArch64Imm(L: TargetPtr, Imm: Value + Addend);
530 break;
531 case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
532 // Operation: S + A
533 // Immediate goes in bits 21:10 of LD/ST instruction, taken
534 // from bits 11:0 of X
535 or32AArch64Imm(L: TargetPtr, Imm: getBits(Val: Value + Addend, Start: 0, End: 11));
536 break;
537 case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
538 // Operation: S + A
539 // Immediate goes in bits 21:10 of LD/ST instruction, taken
540 // from bits 11:1 of X
541 or32AArch64Imm(L: TargetPtr, Imm: getBits(Val: Value + Addend, Start: 1, End: 11));
542 break;
543 case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
544 // Operation: S + A
545 // Immediate goes in bits 21:10 of LD/ST instruction, taken
546 // from bits 11:2 of X
547 or32AArch64Imm(L: TargetPtr, Imm: getBits(Val: Value + Addend, Start: 2, End: 11));
548 break;
549 case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
550 // Operation: S + A
551 // Immediate goes in bits 21:10 of LD/ST instruction, taken
552 // from bits 11:3 of X
553 or32AArch64Imm(L: TargetPtr, Imm: getBits(Val: Value + Addend, Start: 3, End: 11));
554 break;
555 case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
556 // Operation: S + A
557 // Immediate goes in bits 21:10 of LD/ST instruction, taken
558 // from bits 11:4 of X
559 or32AArch64Imm(L: TargetPtr, Imm: getBits(Val: Value + Addend, Start: 4, End: 11));
560 break;
561 case ELF::R_AARCH64_LD_PREL_LO19: {
562 // Operation: S + A - P
563 uint64_t Result = Value + Addend - FinalAddress;
564
565 // "Check that -2^20 <= result < 2^20".
566 assert(isInt<21>(Result));
567
568 *TargetPtr &= 0xff00001fU;
569 // Immediate goes in bits 23:5 of LD imm instruction, taken
570 // from bits 20:2 of X
571 *TargetPtr |= ((Result & 0xffc) << (5 - 2));
572 break;
573 }
574 case ELF::R_AARCH64_ADR_PREL_LO21: {
575 // Operation: S + A - P
576 uint64_t Result = Value + Addend - FinalAddress;
577
578 // "Check that -2^20 <= result < 2^20".
579 assert(isInt<21>(Result));
580
581 *TargetPtr &= 0x9f00001fU;
582 // Immediate goes in bits 23:5, 30:29 of ADR imm instruction, taken
583 // from bits 20:0 of X
584 *TargetPtr |= ((Result & 0xffc) << (5 - 2));
585 *TargetPtr |= (Result & 0x3) << 29;
586 break;
587 }
588 }
589}
590
591void RuntimeDyldELF::resolveARMRelocation(const SectionEntry &Section,
592 uint64_t Offset, uint32_t Value,
593 uint32_t Type, int32_t Addend) {
594 // TODO: Add Thumb relocations.
595 uint32_t *TargetPtr =
596 reinterpret_cast<uint32_t *>(Section.getAddressWithOffset(OffsetBytes: Offset));
597 uint32_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset) & 0xFFFFFFFF;
598 Value += Addend;
599
600 LLVM_DEBUG(dbgs() << "resolveARMRelocation, LocalAddress: "
601 << Section.getAddressWithOffset(Offset)
602 << " FinalAddress: " << format("%p", FinalAddress)
603 << " Value: " << format("%x", Value)
604 << " Type: " << format("%x", Type)
605 << " Addend: " << format("%x", Addend) << "\n");
606
607 switch (Type) {
608 default:
609 llvm_unreachable("Not implemented relocation type!");
610
611 case ELF::R_ARM_NONE:
612 break;
613 // Write a 31bit signed offset
614 case ELF::R_ARM_PREL31:
615 support::ulittle32_t::ref{TargetPtr} =
616 (support::ulittle32_t::ref{TargetPtr} & 0x80000000) |
617 ((Value - FinalAddress) & ~0x80000000);
618 break;
619 case ELF::R_ARM_TARGET1:
620 case ELF::R_ARM_ABS32:
621 support::ulittle32_t::ref{TargetPtr} = Value;
622 break;
623 // Write first 16 bit of 32 bit value to the mov instruction.
624 // Last 4 bit should be shifted.
625 case ELF::R_ARM_MOVW_ABS_NC:
626 case ELF::R_ARM_MOVT_ABS:
627 if (Type == ELF::R_ARM_MOVW_ABS_NC)
628 Value = Value & 0xFFFF;
629 else if (Type == ELF::R_ARM_MOVT_ABS)
630 Value = (Value >> 16) & 0xFFFF;
631 support::ulittle32_t::ref{TargetPtr} =
632 (support::ulittle32_t::ref{TargetPtr} & ~0x000F0FFF) | (Value & 0xFFF) |
633 (((Value >> 12) & 0xF) << 16);
634 break;
635 // Write 24 bit relative value to the branch instruction.
636 case ELF::R_ARM_PC24: // Fall through.
637 case ELF::R_ARM_CALL: // Fall through.
638 case ELF::R_ARM_JUMP24:
639 int32_t RelValue = static_cast<int32_t>(Value - FinalAddress - 8);
640 RelValue = (RelValue & 0x03FFFFFC) >> 2;
641 assert((support::ulittle32_t::ref{TargetPtr} & 0xFFFFFF) == 0xFFFFFE);
642 support::ulittle32_t::ref{TargetPtr} =
643 (support::ulittle32_t::ref{TargetPtr} & 0xFF000000) | RelValue;
644 break;
645 }
646}
647
648void RuntimeDyldELF::setMipsABI(const ObjectFile &Obj) {
649 if (Arch == Triple::UnknownArch ||
650 Triple::getArchTypePrefix(Kind: Arch) != "mips") {
651 IsMipsO32ABI = false;
652 IsMipsN32ABI = false;
653 IsMipsN64ABI = false;
654 return;
655 }
656 if (auto *E = dyn_cast<ELFObjectFileBase>(Val: &Obj)) {
657 unsigned AbiVariant = E->getPlatformFlags();
658 IsMipsO32ABI = AbiVariant & ELF::EF_MIPS_ABI_O32;
659 IsMipsN32ABI = AbiVariant & ELF::EF_MIPS_ABI2;
660 }
661 IsMipsN64ABI = Obj.getFileFormatName() == "elf64-mips";
662}
663
664// Return the .TOC. section and offset.
665Error RuntimeDyldELF::findPPC64TOCSection(const ELFObjectFileBase &Obj,
666 ObjSectionToIDMap &LocalSections,
667 RelocationValueRef &Rel) {
668 // Set a default SectionID in case we do not find a TOC section below.
669 // This may happen for references to TOC base base (sym@toc, .odp
670 // relocation) without a .toc directive. In this case just use the
671 // first section (which is usually the .odp) since the code won't
672 // reference the .toc base directly.
673 Rel.SymbolName = nullptr;
674 Rel.SectionID = 0;
675
676 // The TOC consists of sections .got, .toc, .tocbss, .plt in that
677 // order. The TOC starts where the first of these sections starts.
678 for (auto &Section : Obj.sections()) {
679 Expected<StringRef> NameOrErr = Section.getName();
680 if (!NameOrErr)
681 return NameOrErr.takeError();
682 StringRef SectionName = *NameOrErr;
683
684 if (SectionName == ".got"
685 || SectionName == ".toc"
686 || SectionName == ".tocbss"
687 || SectionName == ".plt") {
688 if (auto SectionIDOrErr =
689 findOrEmitSection(Obj, Section, IsCode: false, LocalSections))
690 Rel.SectionID = *SectionIDOrErr;
691 else
692 return SectionIDOrErr.takeError();
693 break;
694 }
695 }
696
697 // Per the ppc64-elf-linux ABI, The TOC base is TOC value plus 0x8000
698 // thus permitting a full 64 Kbytes segment.
699 Rel.Addend = 0x8000;
700
701 return Error::success();
702}
703
704// Returns the sections and offset associated with the ODP entry referenced
705// by Symbol.
706Error RuntimeDyldELF::findOPDEntrySection(const ELFObjectFileBase &Obj,
707 ObjSectionToIDMap &LocalSections,
708 RelocationValueRef &Rel) {
709 // Get the ELF symbol value (st_value) to compare with Relocation offset in
710 // .opd entries
711 for (section_iterator si = Obj.section_begin(), se = Obj.section_end();
712 si != se; ++si) {
713
714 Expected<section_iterator> RelSecOrErr = si->getRelocatedSection();
715 if (!RelSecOrErr)
716 report_fatal_error(reason: Twine(toString(E: RelSecOrErr.takeError())));
717
718 section_iterator RelSecI = *RelSecOrErr;
719 if (RelSecI == Obj.section_end())
720 continue;
721
722 Expected<StringRef> NameOrErr = RelSecI->getName();
723 if (!NameOrErr)
724 return NameOrErr.takeError();
725 StringRef RelSectionName = *NameOrErr;
726
727 if (RelSectionName != ".opd")
728 continue;
729
730 for (elf_relocation_iterator i = si->relocation_begin(),
731 e = si->relocation_end();
732 i != e;) {
733 // The R_PPC64_ADDR64 relocation indicates the first field
734 // of a .opd entry
735 uint64_t TypeFunc = i->getType();
736 if (TypeFunc != ELF::R_PPC64_ADDR64) {
737 ++i;
738 continue;
739 }
740
741 uint64_t TargetSymbolOffset = i->getOffset();
742 symbol_iterator TargetSymbol = i->getSymbol();
743 int64_t Addend;
744 if (auto AddendOrErr = i->getAddend())
745 Addend = *AddendOrErr;
746 else
747 return AddendOrErr.takeError();
748
749 ++i;
750 if (i == e)
751 break;
752
753 // Just check if following relocation is a R_PPC64_TOC
754 uint64_t TypeTOC = i->getType();
755 if (TypeTOC != ELF::R_PPC64_TOC)
756 continue;
757
758 // Finally compares the Symbol value and the target symbol offset
759 // to check if this .opd entry refers to the symbol the relocation
760 // points to.
761 if (Rel.Addend != (int64_t)TargetSymbolOffset)
762 continue;
763
764 section_iterator TSI = Obj.section_end();
765 if (auto TSIOrErr = TargetSymbol->getSection())
766 TSI = *TSIOrErr;
767 else
768 return TSIOrErr.takeError();
769 assert(TSI != Obj.section_end() && "TSI should refer to a valid section");
770
771 bool IsCode = TSI->isText();
772 if (auto SectionIDOrErr = findOrEmitSection(Obj, Section: *TSI, IsCode,
773 LocalSections))
774 Rel.SectionID = *SectionIDOrErr;
775 else
776 return SectionIDOrErr.takeError();
777 Rel.Addend = (intptr_t)Addend;
778 return Error::success();
779 }
780 }
781 llvm_unreachable("Attempting to get address of ODP entry!");
782}
783
784// Relocation masks following the #lo(value), #hi(value), #ha(value),
785// #higher(value), #highera(value), #highest(value), and #highesta(value)
786// macros defined in section 4.5.1. Relocation Types of the PPC-elf64abi
787// document.
788
789static inline uint16_t applyPPClo(uint64_t value) { return value & 0xffff; }
790
791static inline uint16_t applyPPChi(uint64_t value) {
792 return (value >> 16) & 0xffff;
793}
794
795static inline uint16_t applyPPCha (uint64_t value) {
796 return ((value + 0x8000) >> 16) & 0xffff;
797}
798
799static inline uint16_t applyPPChigher(uint64_t value) {
800 return (value >> 32) & 0xffff;
801}
802
803static inline uint16_t applyPPChighera (uint64_t value) {
804 return ((value + 0x8000) >> 32) & 0xffff;
805}
806
807static inline uint16_t applyPPChighest(uint64_t value) {
808 return (value >> 48) & 0xffff;
809}
810
811static inline uint16_t applyPPChighesta (uint64_t value) {
812 return ((value + 0x8000) >> 48) & 0xffff;
813}
814
815void RuntimeDyldELF::resolvePPC32Relocation(const SectionEntry &Section,
816 uint64_t Offset, uint64_t Value,
817 uint32_t Type, int64_t Addend) {
818 uint8_t *LocalAddress = Section.getAddressWithOffset(OffsetBytes: Offset);
819 switch (Type) {
820 default:
821 report_fatal_error(reason: "Relocation type not implemented yet!");
822 break;
823 case ELF::R_PPC_ADDR16_LO:
824 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Value + Addend));
825 break;
826 case ELF::R_PPC_ADDR16_HI:
827 writeInt16BE(Addr: LocalAddress, Value: applyPPChi(value: Value + Addend));
828 break;
829 case ELF::R_PPC_ADDR16_HA:
830 writeInt16BE(Addr: LocalAddress, Value: applyPPCha(value: Value + Addend));
831 break;
832 }
833}
834
835void RuntimeDyldELF::resolvePPC64Relocation(const SectionEntry &Section,
836 uint64_t Offset, uint64_t Value,
837 uint32_t Type, int64_t Addend) {
838 uint8_t *LocalAddress = Section.getAddressWithOffset(OffsetBytes: Offset);
839 switch (Type) {
840 default:
841 report_fatal_error(reason: "Relocation type not implemented yet!");
842 break;
843 case ELF::R_PPC64_ADDR16:
844 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Value + Addend));
845 break;
846 case ELF::R_PPC64_ADDR16_DS:
847 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Value + Addend) & ~3);
848 break;
849 case ELF::R_PPC64_ADDR16_LO:
850 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Value + Addend));
851 break;
852 case ELF::R_PPC64_ADDR16_LO_DS:
853 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Value + Addend) & ~3);
854 break;
855 case ELF::R_PPC64_ADDR16_HI:
856 case ELF::R_PPC64_ADDR16_HIGH:
857 writeInt16BE(Addr: LocalAddress, Value: applyPPChi(value: Value + Addend));
858 break;
859 case ELF::R_PPC64_ADDR16_HA:
860 case ELF::R_PPC64_ADDR16_HIGHA:
861 writeInt16BE(Addr: LocalAddress, Value: applyPPCha(value: Value + Addend));
862 break;
863 case ELF::R_PPC64_ADDR16_HIGHER:
864 writeInt16BE(Addr: LocalAddress, Value: applyPPChigher(value: Value + Addend));
865 break;
866 case ELF::R_PPC64_ADDR16_HIGHERA:
867 writeInt16BE(Addr: LocalAddress, Value: applyPPChighera(value: Value + Addend));
868 break;
869 case ELF::R_PPC64_ADDR16_HIGHEST:
870 writeInt16BE(Addr: LocalAddress, Value: applyPPChighest(value: Value + Addend));
871 break;
872 case ELF::R_PPC64_ADDR16_HIGHESTA:
873 writeInt16BE(Addr: LocalAddress, Value: applyPPChighesta(value: Value + Addend));
874 break;
875 case ELF::R_PPC64_ADDR14: {
876 assert(((Value + Addend) & 3) == 0);
877 // Preserve the AA/LK bits in the branch instruction
878 uint8_t aalk = *(LocalAddress + 3);
879 writeInt16BE(Addr: LocalAddress + 2, Value: (aalk & 3) | ((Value + Addend) & 0xfffc));
880 } break;
881 case ELF::R_PPC64_REL16_LO: {
882 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
883 uint64_t Delta = Value - FinalAddress + Addend;
884 writeInt16BE(Addr: LocalAddress, Value: applyPPClo(value: Delta));
885 } break;
886 case ELF::R_PPC64_REL16_HI: {
887 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
888 uint64_t Delta = Value - FinalAddress + Addend;
889 writeInt16BE(Addr: LocalAddress, Value: applyPPChi(value: Delta));
890 } break;
891 case ELF::R_PPC64_REL16_HA: {
892 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
893 uint64_t Delta = Value - FinalAddress + Addend;
894 writeInt16BE(Addr: LocalAddress, Value: applyPPCha(value: Delta));
895 } break;
896 case ELF::R_PPC64_ADDR32: {
897 int64_t Result = static_cast<int64_t>(Value + Addend);
898 if (SignExtend64<32>(x: Result) != Result)
899 llvm_unreachable("Relocation R_PPC64_ADDR32 overflow");
900 writeInt32BE(Addr: LocalAddress, Value: Result);
901 } break;
902 case ELF::R_PPC64_REL24: {
903 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
904 int64_t delta = static_cast<int64_t>(Value - FinalAddress + Addend);
905 if (SignExtend64<26>(x: delta) != delta)
906 llvm_unreachable("Relocation R_PPC64_REL24 overflow");
907 // We preserve bits other than LI field, i.e. PO and AA/LK fields.
908 uint32_t Inst = readBytesUnaligned(Src: LocalAddress, Size: 4);
909 writeInt32BE(Addr: LocalAddress, Value: (Inst & 0xFC000003) | (delta & 0x03FFFFFC));
910 } break;
911 case ELF::R_PPC64_REL32: {
912 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
913 int64_t delta = static_cast<int64_t>(Value - FinalAddress + Addend);
914 if (SignExtend64<32>(x: delta) != delta)
915 llvm_unreachable("Relocation R_PPC64_REL32 overflow");
916 writeInt32BE(Addr: LocalAddress, Value: delta);
917 } break;
918 case ELF::R_PPC64_REL64: {
919 uint64_t FinalAddress = Section.getLoadAddressWithOffset(OffsetBytes: Offset);
920 uint64_t Delta = Value - FinalAddress + Addend;
921 writeInt64BE(Addr: LocalAddress, Value: Delta);
922 } break;
923 case ELF::R_PPC64_ADDR64:
924 writeInt64BE(Addr: LocalAddress, Value: Value + Addend);
925 break;
926 }
927}
928
929void RuntimeDyldELF::resolveSystemZRelocation(const SectionEntry &Section,
930 uint64_t Offset, uint64_t Value,
931 uint32_t Type, int64_t Addend) {
932 uint8_t *LocalAddress = Section.getAddressWithOffset(OffsetBytes: Offset);
933 switch (Type) {
934 default:
935 report_fatal_error(reason: "Relocation type not implemented yet!");
936 break;
937 case ELF::R_390_PC16DBL:
938 case ELF::R_390_PLT16DBL: {
939 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(OffsetBytes: Offset);
940 assert(int16_t(Delta / 2) * 2 == Delta && "R_390_PC16DBL overflow");
941 writeInt16BE(Addr: LocalAddress, Value: Delta / 2);
942 break;
943 }
944 case ELF::R_390_PC32DBL:
945 case ELF::R_390_PLT32DBL: {
946 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(OffsetBytes: Offset);
947 assert(int32_t(Delta / 2) * 2 == Delta && "R_390_PC32DBL overflow");
948 writeInt32BE(Addr: LocalAddress, Value: Delta / 2);
949 break;
950 }
951 case ELF::R_390_PC16: {
952 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(OffsetBytes: Offset);
953 assert(int16_t(Delta) == Delta && "R_390_PC16 overflow");
954 writeInt16BE(Addr: LocalAddress, Value: Delta);
955 break;
956 }
957 case ELF::R_390_PC32: {
958 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(OffsetBytes: Offset);
959 assert(int32_t(Delta) == Delta && "R_390_PC32 overflow");
960 writeInt32BE(Addr: LocalAddress, Value: Delta);
961 break;
962 }
963 case ELF::R_390_PC64: {
964 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(OffsetBytes: Offset);
965 writeInt64BE(Addr: LocalAddress, Value: Delta);
966 break;
967 }
968 case ELF::R_390_8:
969 *LocalAddress = (uint8_t)(Value + Addend);
970 break;
971 case ELF::R_390_16:
972 writeInt16BE(Addr: LocalAddress, Value: Value + Addend);
973 break;
974 case ELF::R_390_32:
975 writeInt32BE(Addr: LocalAddress, Value: Value + Addend);
976 break;
977 case ELF::R_390_64:
978 writeInt64BE(Addr: LocalAddress, Value: Value + Addend);
979 break;
980 }
981}
982
983void RuntimeDyldELF::resolveBPFRelocation(const SectionEntry &Section,
984 uint64_t Offset, uint64_t Value,
985 uint32_t Type, int64_t Addend) {
986 bool isBE = Arch == Triple::bpfeb;
987
988 switch (Type) {
989 default:
990 report_fatal_error(reason: "Relocation type not implemented yet!");
991 break;
992 case ELF::R_BPF_NONE:
993 case ELF::R_BPF_64_64:
994 case ELF::R_BPF_64_32:
995 case ELF::R_BPF_64_NODYLD32:
996 break;
997 case ELF::R_BPF_64_ABS64: {
998 write(isBE, P: Section.getAddressWithOffset(OffsetBytes: Offset), V: Value + Addend);
999 LLVM_DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at "
1000 << format("%p\n", Section.getAddressWithOffset(Offset)));
1001 break;
1002 }
1003 case ELF::R_BPF_64_ABS32: {
1004 Value += Addend;
1005 assert(Value <= UINT32_MAX);
1006 write(isBE, P: Section.getAddressWithOffset(OffsetBytes: Offset), V: static_cast<uint32_t>(Value));
1007 LLVM_DEBUG(dbgs() << "Writing " << format("%p", Value) << " at "
1008 << format("%p\n", Section.getAddressWithOffset(Offset)));
1009 break;
1010 }
1011 }
1012}
1013
1014// The target location for the relocation is described by RE.SectionID and
1015// RE.Offset. RE.SectionID can be used to find the SectionEntry. Each
1016// SectionEntry has three members describing its location.
1017// SectionEntry::Address is the address at which the section has been loaded
1018// into memory in the current (host) process. SectionEntry::LoadAddress is the
1019// address that the section will have in the target process.
1020// SectionEntry::ObjAddress is the address of the bits for this section in the
1021// original emitted object image (also in the current address space).
1022//
1023// Relocations will be applied as if the section were loaded at
1024// SectionEntry::LoadAddress, but they will be applied at an address based
1025// on SectionEntry::Address. SectionEntry::ObjAddress will be used to refer to
1026// Target memory contents if they are required for value calculations.
1027//
1028// The Value parameter here is the load address of the symbol for the
1029// relocation to be applied. For relocations which refer to symbols in the
1030// current object Value will be the LoadAddress of the section in which
1031// the symbol resides (RE.Addend provides additional information about the
1032// symbol location). For external symbols, Value will be the address of the
1033// symbol in the target address space.
1034void RuntimeDyldELF::resolveRelocation(const RelocationEntry &RE,
1035 uint64_t Value) {
1036 const SectionEntry &Section = Sections[RE.SectionID];
1037 return resolveRelocation(Section, Offset: RE.Offset, Value, Type: RE.RelType, Addend: RE.Addend,
1038 SymOffset: RE.SymOffset, SectionID: RE.SectionID);
1039}
1040
1041void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section,
1042 uint64_t Offset, uint64_t Value,
1043 uint32_t Type, int64_t Addend,
1044 uint64_t SymOffset, SID SectionID) {
1045 switch (Arch) {
1046 case Triple::x86_64:
1047 resolveX86_64Relocation(Section, Offset, Value, Type, Addend, SymOffset);
1048 break;
1049 case Triple::x86:
1050 resolveX86Relocation(Section, Offset, Value: (uint32_t)(Value & 0xffffffffL), Type,
1051 Addend: (uint32_t)(Addend & 0xffffffffL));
1052 break;
1053 case Triple::aarch64:
1054 case Triple::aarch64_be:
1055 resolveAArch64Relocation(Section, Offset, Value, Type, Addend);
1056 break;
1057 case Triple::arm: // Fall through.
1058 case Triple::armeb:
1059 case Triple::thumb:
1060 case Triple::thumbeb:
1061 resolveARMRelocation(Section, Offset, Value: (uint32_t)(Value & 0xffffffffL), Type,
1062 Addend: (uint32_t)(Addend & 0xffffffffL));
1063 break;
1064 case Triple::ppc: // Fall through.
1065 case Triple::ppcle:
1066 resolvePPC32Relocation(Section, Offset, Value, Type, Addend);
1067 break;
1068 case Triple::ppc64: // Fall through.
1069 case Triple::ppc64le:
1070 resolvePPC64Relocation(Section, Offset, Value, Type, Addend);
1071 break;
1072 case Triple::systemz:
1073 resolveSystemZRelocation(Section, Offset, Value, Type, Addend);
1074 break;
1075 case Triple::bpfel:
1076 case Triple::bpfeb:
1077 resolveBPFRelocation(Section, Offset, Value, Type, Addend);
1078 break;
1079 default:
1080 llvm_unreachable("Unsupported CPU type!");
1081 }
1082}
1083
1084void *RuntimeDyldELF::computePlaceholderAddress(unsigned SectionID, uint64_t Offset) const {
1085 return (void *)(Sections[SectionID].getObjAddress() + Offset);
1086}
1087
1088void RuntimeDyldELF::processSimpleRelocation(unsigned SectionID, uint64_t Offset, unsigned RelType, RelocationValueRef Value) {
1089 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend, Value.Offset);
1090 if (Value.SymbolName)
1091 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1092 else
1093 addRelocationForSection(RE, SectionID: Value.SectionID);
1094}
1095
1096uint32_t RuntimeDyldELF::getMatchingLoRelocation(uint32_t RelType,
1097 bool IsLocal) const {
1098 switch (RelType) {
1099 case ELF::R_MICROMIPS_GOT16:
1100 if (IsLocal)
1101 return ELF::R_MICROMIPS_LO16;
1102 break;
1103 case ELF::R_MICROMIPS_HI16:
1104 return ELF::R_MICROMIPS_LO16;
1105 case ELF::R_MIPS_GOT16:
1106 if (IsLocal)
1107 return ELF::R_MIPS_LO16;
1108 break;
1109 case ELF::R_MIPS_HI16:
1110 return ELF::R_MIPS_LO16;
1111 case ELF::R_MIPS_PCHI16:
1112 return ELF::R_MIPS_PCLO16;
1113 default:
1114 break;
1115 }
1116 return ELF::R_MIPS_NONE;
1117}
1118
1119// Sometimes we don't need to create thunk for a branch.
1120// This typically happens when branch target is located
1121// in the same object file. In such case target is either
1122// a weak symbol or symbol in a different executable section.
1123// This function checks if branch target is located in the
1124// same object file and if distance between source and target
1125// fits R_AARCH64_CALL26 relocation. If both conditions are
1126// met, it emits direct jump to the target and returns true.
1127// Otherwise false is returned and thunk is created.
1128bool RuntimeDyldELF::resolveAArch64ShortBranch(
1129 unsigned SectionID, relocation_iterator RelI,
1130 const RelocationValueRef &Value) {
1131 uint64_t TargetOffset;
1132 unsigned TargetSectionID;
1133 if (Value.SymbolName) {
1134 auto Loc = GlobalSymbolTable.find(Key: Value.SymbolName);
1135
1136 // Don't create direct branch for external symbols.
1137 if (Loc == GlobalSymbolTable.end())
1138 return false;
1139
1140 const auto &SymInfo = Loc->second;
1141
1142 TargetSectionID = SymInfo.getSectionID();
1143 TargetOffset = SymInfo.getOffset();
1144 } else {
1145 TargetSectionID = Value.SectionID;
1146 TargetOffset = 0;
1147 }
1148
1149 // We don't actually know the load addresses at this point, so if the
1150 // branch is cross-section, we don't know exactly how far away it is.
1151 if (TargetSectionID != SectionID)
1152 return false;
1153
1154 uint64_t SourceOffset = RelI->getOffset();
1155
1156 // R_AARCH64_CALL26 requires immediate to be in range -2^27 <= imm < 2^27
1157 // If distance between source and target is out of range then we should
1158 // create thunk.
1159 if (!isInt<28>(x: TargetOffset + Value.Addend - SourceOffset))
1160 return false;
1161
1162 RelocationEntry RE(SectionID, SourceOffset, RelI->getType(), Value.Addend);
1163 if (Value.SymbolName)
1164 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1165 else
1166 addRelocationForSection(RE, SectionID: Value.SectionID);
1167
1168 return true;
1169}
1170
1171void RuntimeDyldELF::resolveAArch64Branch(unsigned SectionID,
1172 const RelocationValueRef &Value,
1173 relocation_iterator RelI,
1174 StubMap &Stubs) {
1175
1176 LLVM_DEBUG(dbgs() << "\t\tThis is an AArch64 branch relocation.");
1177 SectionEntry &Section = Sections[SectionID];
1178
1179 uint64_t Offset = RelI->getOffset();
1180 unsigned RelType = RelI->getType();
1181 // Look for an existing stub.
1182 StubMap::const_iterator i = Stubs.find(x: Value);
1183 if (i != Stubs.end()) {
1184 resolveRelocation(Section, Offset,
1185 Value: Section.getLoadAddressWithOffset(OffsetBytes: i->second), Type: RelType, Addend: 0);
1186 LLVM_DEBUG(dbgs() << " Stub function found\n");
1187 } else if (!resolveAArch64ShortBranch(SectionID, RelI, Value)) {
1188 // Create a new stub function.
1189 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
1190 Stubs[Value] = Section.getStubOffset();
1191 uint8_t *StubTargetAddr = createStubFunction(
1192 Addr: Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset()));
1193
1194 RelocationEntry REmovz_g3(SectionID, StubTargetAddr - Section.getAddress(),
1195 ELF::R_AARCH64_MOVW_UABS_G3, Value.Addend);
1196 RelocationEntry REmovk_g2(SectionID,
1197 StubTargetAddr - Section.getAddress() + 4,
1198 ELF::R_AARCH64_MOVW_UABS_G2_NC, Value.Addend);
1199 RelocationEntry REmovk_g1(SectionID,
1200 StubTargetAddr - Section.getAddress() + 8,
1201 ELF::R_AARCH64_MOVW_UABS_G1_NC, Value.Addend);
1202 RelocationEntry REmovk_g0(SectionID,
1203 StubTargetAddr - Section.getAddress() + 12,
1204 ELF::R_AARCH64_MOVW_UABS_G0_NC, Value.Addend);
1205
1206 if (Value.SymbolName) {
1207 addRelocationForSymbol(RE: REmovz_g3, SymbolName: Value.SymbolName);
1208 addRelocationForSymbol(RE: REmovk_g2, SymbolName: Value.SymbolName);
1209 addRelocationForSymbol(RE: REmovk_g1, SymbolName: Value.SymbolName);
1210 addRelocationForSymbol(RE: REmovk_g0, SymbolName: Value.SymbolName);
1211 } else {
1212 addRelocationForSection(RE: REmovz_g3, SectionID: Value.SectionID);
1213 addRelocationForSection(RE: REmovk_g2, SectionID: Value.SectionID);
1214 addRelocationForSection(RE: REmovk_g1, SectionID: Value.SectionID);
1215 addRelocationForSection(RE: REmovk_g0, SectionID: Value.SectionID);
1216 }
1217 resolveRelocation(Section, Offset,
1218 Value: Section.getLoadAddressWithOffset(OffsetBytes: Section.getStubOffset()),
1219 Type: RelType, Addend: 0);
1220 Section.advanceStubOffset(StubSize: getMaxStubSize());
1221 }
1222}
1223
1224Expected<relocation_iterator>
1225RuntimeDyldELF::processRelocationRef(
1226 unsigned SectionID, relocation_iterator RelI, const ObjectFile &O,
1227 ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) {
1228 const auto &Obj = cast<ELFObjectFileBase>(Val: O);
1229 uint64_t RelType = RelI->getType();
1230 int64_t Addend = 0;
1231 if (Expected<int64_t> AddendOrErr = ELFRelocationRef(*RelI).getAddend())
1232 Addend = *AddendOrErr;
1233 else
1234 consumeError(Err: AddendOrErr.takeError());
1235 elf_symbol_iterator Symbol = RelI->getSymbol();
1236
1237 // Obtain the symbol name which is referenced in the relocation
1238 StringRef TargetName;
1239 if (Symbol != Obj.symbol_end()) {
1240 if (auto TargetNameOrErr = Symbol->getName())
1241 TargetName = *TargetNameOrErr;
1242 else
1243 return TargetNameOrErr.takeError();
1244 }
1245 LLVM_DEBUG(dbgs() << "\t\tRelType: " << RelType << " Addend: " << Addend
1246 << " TargetName: " << TargetName << "\n");
1247 RelocationValueRef Value;
1248 // First search for the symbol in the local symbol table
1249 SymbolRef::Type SymType = SymbolRef::ST_Unknown;
1250
1251 // Search for the symbol in the global symbol table
1252 RTDyldSymbolTable::const_iterator gsi = GlobalSymbolTable.end();
1253 if (Symbol != Obj.symbol_end()) {
1254 gsi = GlobalSymbolTable.find(Key: TargetName.data());
1255 Expected<SymbolRef::Type> SymTypeOrErr = Symbol->getType();
1256 if (!SymTypeOrErr) {
1257 std::string Buf;
1258 raw_string_ostream OS(Buf);
1259 logAllUnhandledErrors(E: SymTypeOrErr.takeError(), OS);
1260 report_fatal_error(reason: Twine(OS.str()));
1261 }
1262 SymType = *SymTypeOrErr;
1263 }
1264 if (gsi != GlobalSymbolTable.end()) {
1265 const auto &SymInfo = gsi->second;
1266 Value.SectionID = SymInfo.getSectionID();
1267 Value.Offset = SymInfo.getOffset();
1268 Value.Addend = SymInfo.getOffset() + Addend;
1269 } else {
1270 switch (SymType) {
1271 case SymbolRef::ST_Debug: {
1272 // TODO: Now ELF SymbolRef::ST_Debug = STT_SECTION, it's not obviously
1273 // and can be changed by another developers. Maybe best way is add
1274 // a new symbol type ST_Section to SymbolRef and use it.
1275 auto SectionOrErr = Symbol->getSection();
1276 if (!SectionOrErr) {
1277 std::string Buf;
1278 raw_string_ostream OS(Buf);
1279 logAllUnhandledErrors(E: SectionOrErr.takeError(), OS);
1280 report_fatal_error(reason: Twine(OS.str()));
1281 }
1282 section_iterator si = *SectionOrErr;
1283 if (si == Obj.section_end())
1284 llvm_unreachable("Symbol section not found, bad object file format!");
1285 LLVM_DEBUG(dbgs() << "\t\tThis is section symbol\n");
1286 bool isCode = si->isText();
1287 if (auto SectionIDOrErr = findOrEmitSection(Obj, Section: (*si), IsCode: isCode,
1288 LocalSections&: ObjSectionToID))
1289 Value.SectionID = *SectionIDOrErr;
1290 else
1291 return SectionIDOrErr.takeError();
1292 Value.Addend = Addend;
1293 break;
1294 }
1295 case SymbolRef::ST_Data:
1296 case SymbolRef::ST_Function:
1297 case SymbolRef::ST_Other:
1298 case SymbolRef::ST_Unknown: {
1299 Value.SymbolName = TargetName.data();
1300 Value.Addend = Addend;
1301
1302 // Absolute relocations will have a zero symbol ID (STN_UNDEF), which
1303 // will manifest here as a NULL symbol name.
1304 // We can set this as a valid (but empty) symbol name, and rely
1305 // on addRelocationForSymbol to handle this.
1306 if (!Value.SymbolName)
1307 Value.SymbolName = "";
1308 break;
1309 }
1310 default:
1311 llvm_unreachable("Unresolved symbol type!");
1312 break;
1313 }
1314 }
1315
1316 uint64_t Offset = RelI->getOffset();
1317
1318 LLVM_DEBUG(dbgs() << "\t\tSectionID: " << SectionID << " Offset: " << Offset
1319 << "\n");
1320 if ((Arch == Triple::aarch64 || Arch == Triple::aarch64_be)) {
1321 if ((RelType == ELF::R_AARCH64_CALL26 ||
1322 RelType == ELF::R_AARCH64_JUMP26) &&
1323 MemMgr.allowStubAllocation()) {
1324 resolveAArch64Branch(SectionID, Value, RelI, Stubs);
1325 } else if (RelType == ELF::R_AARCH64_ADR_GOT_PAGE) {
1326 // Create new GOT entry or find existing one. If GOT entry is
1327 // to be created, then we also emit ABS64 relocation for it.
1328 uint64_t GOTOffset = findOrAllocGOTEntry(Value, GOTRelType: ELF::R_AARCH64_ABS64);
1329 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: GOTOffset + Addend,
1330 Type: ELF::R_AARCH64_ADR_PREL_PG_HI21);
1331
1332 } else if (RelType == ELF::R_AARCH64_LD64_GOT_LO12_NC) {
1333 uint64_t GOTOffset = findOrAllocGOTEntry(Value, GOTRelType: ELF::R_AARCH64_ABS64);
1334 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: GOTOffset + Addend,
1335 Type: ELF::R_AARCH64_LDST64_ABS_LO12_NC);
1336 } else {
1337 processSimpleRelocation(SectionID, Offset, RelType, Value);
1338 }
1339 } else if (Arch == Triple::arm) {
1340 if (RelType == ELF::R_ARM_PC24 || RelType == ELF::R_ARM_CALL ||
1341 RelType == ELF::R_ARM_JUMP24) {
1342 // This is an ARM branch relocation, need to use a stub function.
1343 LLVM_DEBUG(dbgs() << "\t\tThis is an ARM branch relocation.\n");
1344 SectionEntry &Section = Sections[SectionID];
1345
1346 // Look for an existing stub.
1347 StubMap::const_iterator i = Stubs.find(x: Value);
1348 if (i != Stubs.end()) {
1349 resolveRelocation(Section, Offset,
1350 Value: Section.getLoadAddressWithOffset(OffsetBytes: i->second), Type: RelType,
1351 Addend: 0);
1352 LLVM_DEBUG(dbgs() << " Stub function found\n");
1353 } else {
1354 // Create a new stub function.
1355 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
1356 Stubs[Value] = Section.getStubOffset();
1357 uint8_t *StubTargetAddr = createStubFunction(
1358 Addr: Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset()));
1359 RelocationEntry RE(SectionID, StubTargetAddr - Section.getAddress(),
1360 ELF::R_ARM_ABS32, Value.Addend);
1361 if (Value.SymbolName)
1362 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1363 else
1364 addRelocationForSection(RE, SectionID: Value.SectionID);
1365
1366 resolveRelocation(
1367 Section, Offset,
1368 Value: Section.getLoadAddressWithOffset(OffsetBytes: Section.getStubOffset()), Type: RelType,
1369 Addend: 0);
1370 Section.advanceStubOffset(StubSize: getMaxStubSize());
1371 }
1372 } else {
1373 uint32_t *Placeholder =
1374 reinterpret_cast<uint32_t*>(computePlaceholderAddress(SectionID, Offset));
1375 if (RelType == ELF::R_ARM_PREL31 || RelType == ELF::R_ARM_TARGET1 ||
1376 RelType == ELF::R_ARM_ABS32) {
1377 Value.Addend += *Placeholder;
1378 } else if (RelType == ELF::R_ARM_MOVW_ABS_NC || RelType == ELF::R_ARM_MOVT_ABS) {
1379 // See ELF for ARM documentation
1380 Value.Addend += (int16_t)((*Placeholder & 0xFFF) | (((*Placeholder >> 16) & 0xF) << 12));
1381 }
1382 processSimpleRelocation(SectionID, Offset, RelType, Value);
1383 }
1384 } else if (IsMipsO32ABI) {
1385 uint8_t *Placeholder = reinterpret_cast<uint8_t *>(
1386 computePlaceholderAddress(SectionID, Offset));
1387 uint32_t Opcode = readBytesUnaligned(Src: Placeholder, Size: 4);
1388 if (RelType == ELF::R_MIPS_26) {
1389 // This is an Mips branch relocation, need to use a stub function.
1390 LLVM_DEBUG(dbgs() << "\t\tThis is a Mips branch relocation.");
1391 SectionEntry &Section = Sections[SectionID];
1392
1393 // Extract the addend from the instruction.
1394 // We shift up by two since the Value will be down shifted again
1395 // when applying the relocation.
1396 uint32_t Addend = (Opcode & 0x03ffffff) << 2;
1397
1398 Value.Addend += Addend;
1399
1400 // Look up for existing stub.
1401 StubMap::const_iterator i = Stubs.find(x: Value);
1402 if (i != Stubs.end()) {
1403 RelocationEntry RE(SectionID, Offset, RelType, i->second);
1404 addRelocationForSection(RE, SectionID);
1405 LLVM_DEBUG(dbgs() << " Stub function found\n");
1406 } else {
1407 // Create a new stub function.
1408 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
1409 Stubs[Value] = Section.getStubOffset();
1410
1411 unsigned AbiVariant = Obj.getPlatformFlags();
1412
1413 uint8_t *StubTargetAddr = createStubFunction(
1414 Addr: Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset()), AbiVariant);
1415
1416 // Creating Hi and Lo relocations for the filled stub instructions.
1417 RelocationEntry REHi(SectionID, StubTargetAddr - Section.getAddress(),
1418 ELF::R_MIPS_HI16, Value.Addend);
1419 RelocationEntry RELo(SectionID,
1420 StubTargetAddr - Section.getAddress() + 4,
1421 ELF::R_MIPS_LO16, Value.Addend);
1422
1423 if (Value.SymbolName) {
1424 addRelocationForSymbol(RE: REHi, SymbolName: Value.SymbolName);
1425 addRelocationForSymbol(RE: RELo, SymbolName: Value.SymbolName);
1426 } else {
1427 addRelocationForSection(RE: REHi, SectionID: Value.SectionID);
1428 addRelocationForSection(RE: RELo, SectionID: Value.SectionID);
1429 }
1430
1431 RelocationEntry RE(SectionID, Offset, RelType, Section.getStubOffset());
1432 addRelocationForSection(RE, SectionID);
1433 Section.advanceStubOffset(StubSize: getMaxStubSize());
1434 }
1435 } else if (RelType == ELF::R_MIPS_HI16 || RelType == ELF::R_MIPS_PCHI16) {
1436 int64_t Addend = (Opcode & 0x0000ffff) << 16;
1437 RelocationEntry RE(SectionID, Offset, RelType, Addend);
1438 PendingRelocs.push_back(Elt: std::make_pair(x&: Value, y&: RE));
1439 } else if (RelType == ELF::R_MIPS_LO16 || RelType == ELF::R_MIPS_PCLO16) {
1440 int64_t Addend = Value.Addend + SignExtend32<16>(X: Opcode & 0x0000ffff);
1441 for (auto I = PendingRelocs.begin(); I != PendingRelocs.end();) {
1442 const RelocationValueRef &MatchingValue = I->first;
1443 RelocationEntry &Reloc = I->second;
1444 if (MatchingValue == Value &&
1445 RelType == getMatchingLoRelocation(RelType: Reloc.RelType) &&
1446 SectionID == Reloc.SectionID) {
1447 Reloc.Addend += Addend;
1448 if (Value.SymbolName)
1449 addRelocationForSymbol(RE: Reloc, SymbolName: Value.SymbolName);
1450 else
1451 addRelocationForSection(RE: Reloc, SectionID: Value.SectionID);
1452 I = PendingRelocs.erase(CI: I);
1453 } else
1454 ++I;
1455 }
1456 RelocationEntry RE(SectionID, Offset, RelType, Addend);
1457 if (Value.SymbolName)
1458 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1459 else
1460 addRelocationForSection(RE, SectionID: Value.SectionID);
1461 } else {
1462 if (RelType == ELF::R_MIPS_32)
1463 Value.Addend += Opcode;
1464 else if (RelType == ELF::R_MIPS_PC16)
1465 Value.Addend += SignExtend32<18>(X: (Opcode & 0x0000ffff) << 2);
1466 else if (RelType == ELF::R_MIPS_PC19_S2)
1467 Value.Addend += SignExtend32<21>(X: (Opcode & 0x0007ffff) << 2);
1468 else if (RelType == ELF::R_MIPS_PC21_S2)
1469 Value.Addend += SignExtend32<23>(X: (Opcode & 0x001fffff) << 2);
1470 else if (RelType == ELF::R_MIPS_PC26_S2)
1471 Value.Addend += SignExtend32<28>(X: (Opcode & 0x03ffffff) << 2);
1472 processSimpleRelocation(SectionID, Offset, RelType, Value);
1473 }
1474 } else if (IsMipsN32ABI || IsMipsN64ABI) {
1475 uint32_t r_type = RelType & 0xff;
1476 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend);
1477 if (r_type == ELF::R_MIPS_CALL16 || r_type == ELF::R_MIPS_GOT_PAGE
1478 || r_type == ELF::R_MIPS_GOT_DISP) {
1479 StringMap<uint64_t>::iterator i = GOTSymbolOffsets.find(Key: TargetName);
1480 if (i != GOTSymbolOffsets.end())
1481 RE.SymOffset = i->second;
1482 else {
1483 RE.SymOffset = allocateGOTEntries(no: 1);
1484 GOTSymbolOffsets[TargetName] = RE.SymOffset;
1485 }
1486 if (Value.SymbolName)
1487 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1488 else
1489 addRelocationForSection(RE, SectionID: Value.SectionID);
1490 } else if (RelType == ELF::R_MIPS_26) {
1491 // This is an Mips branch relocation, need to use a stub function.
1492 LLVM_DEBUG(dbgs() << "\t\tThis is a Mips branch relocation.");
1493 SectionEntry &Section = Sections[SectionID];
1494
1495 // Look up for existing stub.
1496 StubMap::const_iterator i = Stubs.find(x: Value);
1497 if (i != Stubs.end()) {
1498 RelocationEntry RE(SectionID, Offset, RelType, i->second);
1499 addRelocationForSection(RE, SectionID);
1500 LLVM_DEBUG(dbgs() << " Stub function found\n");
1501 } else {
1502 // Create a new stub function.
1503 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
1504 Stubs[Value] = Section.getStubOffset();
1505
1506 unsigned AbiVariant = Obj.getPlatformFlags();
1507
1508 uint8_t *StubTargetAddr = createStubFunction(
1509 Addr: Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset()), AbiVariant);
1510
1511 if (IsMipsN32ABI) {
1512 // Creating Hi and Lo relocations for the filled stub instructions.
1513 RelocationEntry REHi(SectionID, StubTargetAddr - Section.getAddress(),
1514 ELF::R_MIPS_HI16, Value.Addend);
1515 RelocationEntry RELo(SectionID,
1516 StubTargetAddr - Section.getAddress() + 4,
1517 ELF::R_MIPS_LO16, Value.Addend);
1518 if (Value.SymbolName) {
1519 addRelocationForSymbol(RE: REHi, SymbolName: Value.SymbolName);
1520 addRelocationForSymbol(RE: RELo, SymbolName: Value.SymbolName);
1521 } else {
1522 addRelocationForSection(RE: REHi, SectionID: Value.SectionID);
1523 addRelocationForSection(RE: RELo, SectionID: Value.SectionID);
1524 }
1525 } else {
1526 // Creating Highest, Higher, Hi and Lo relocations for the filled stub
1527 // instructions.
1528 RelocationEntry REHighest(SectionID,
1529 StubTargetAddr - Section.getAddress(),
1530 ELF::R_MIPS_HIGHEST, Value.Addend);
1531 RelocationEntry REHigher(SectionID,
1532 StubTargetAddr - Section.getAddress() + 4,
1533 ELF::R_MIPS_HIGHER, Value.Addend);
1534 RelocationEntry REHi(SectionID,
1535 StubTargetAddr - Section.getAddress() + 12,
1536 ELF::R_MIPS_HI16, Value.Addend);
1537 RelocationEntry RELo(SectionID,
1538 StubTargetAddr - Section.getAddress() + 20,
1539 ELF::R_MIPS_LO16, Value.Addend);
1540 if (Value.SymbolName) {
1541 addRelocationForSymbol(RE: REHighest, SymbolName: Value.SymbolName);
1542 addRelocationForSymbol(RE: REHigher, SymbolName: Value.SymbolName);
1543 addRelocationForSymbol(RE: REHi, SymbolName: Value.SymbolName);
1544 addRelocationForSymbol(RE: RELo, SymbolName: Value.SymbolName);
1545 } else {
1546 addRelocationForSection(RE: REHighest, SectionID: Value.SectionID);
1547 addRelocationForSection(RE: REHigher, SectionID: Value.SectionID);
1548 addRelocationForSection(RE: REHi, SectionID: Value.SectionID);
1549 addRelocationForSection(RE: RELo, SectionID: Value.SectionID);
1550 }
1551 }
1552 RelocationEntry RE(SectionID, Offset, RelType, Section.getStubOffset());
1553 addRelocationForSection(RE, SectionID);
1554 Section.advanceStubOffset(StubSize: getMaxStubSize());
1555 }
1556 } else {
1557 processSimpleRelocation(SectionID, Offset, RelType, Value);
1558 }
1559
1560 } else if (Arch == Triple::ppc64 || Arch == Triple::ppc64le) {
1561 if (RelType == ELF::R_PPC64_REL24) {
1562 // Determine ABI variant in use for this object.
1563 unsigned AbiVariant = Obj.getPlatformFlags();
1564 AbiVariant &= ELF::EF_PPC64_ABI;
1565 // A PPC branch relocation will need a stub function if the target is
1566 // an external symbol (either Value.SymbolName is set, or SymType is
1567 // Symbol::ST_Unknown) or if the target address is not within the
1568 // signed 24-bits branch address.
1569 SectionEntry &Section = Sections[SectionID];
1570 uint8_t *Target = Section.getAddressWithOffset(OffsetBytes: Offset);
1571 bool RangeOverflow = false;
1572 bool IsExtern = Value.SymbolName || SymType == SymbolRef::ST_Unknown;
1573 if (!IsExtern) {
1574 if (AbiVariant != 2) {
1575 // In the ELFv1 ABI, a function call may point to the .opd entry,
1576 // so the final symbol value is calculated based on the relocation
1577 // values in the .opd section.
1578 if (auto Err = findOPDEntrySection(Obj, LocalSections&: ObjSectionToID, Rel&: Value))
1579 return std::move(Err);
1580 } else {
1581 // In the ELFv2 ABI, a function symbol may provide a local entry
1582 // point, which must be used for direct calls.
1583 if (Value.SectionID == SectionID){
1584 uint8_t SymOther = Symbol->getOther();
1585 Value.Addend += ELF::decodePPC64LocalEntryOffset(Other: SymOther);
1586 }
1587 }
1588 uint8_t *RelocTarget =
1589 Sections[Value.SectionID].getAddressWithOffset(OffsetBytes: Value.Addend);
1590 int64_t delta = static_cast<int64_t>(Target - RelocTarget);
1591 // If it is within 26-bits branch range, just set the branch target
1592 if (SignExtend64<26>(x: delta) != delta) {
1593 RangeOverflow = true;
1594 } else if ((AbiVariant != 2) ||
1595 (AbiVariant == 2 && Value.SectionID == SectionID)) {
1596 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend);
1597 addRelocationForSection(RE, SectionID: Value.SectionID);
1598 }
1599 }
1600 if (IsExtern || (AbiVariant == 2 && Value.SectionID != SectionID) ||
1601 RangeOverflow) {
1602 // It is an external symbol (either Value.SymbolName is set, or
1603 // SymType is SymbolRef::ST_Unknown) or out of range.
1604 StubMap::const_iterator i = Stubs.find(x: Value);
1605 if (i != Stubs.end()) {
1606 // Symbol function stub already created, just relocate to it
1607 resolveRelocation(Section, Offset,
1608 Value: Section.getLoadAddressWithOffset(OffsetBytes: i->second),
1609 Type: RelType, Addend: 0);
1610 LLVM_DEBUG(dbgs() << " Stub function found\n");
1611 } else {
1612 // Create a new stub function.
1613 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
1614 Stubs[Value] = Section.getStubOffset();
1615 uint8_t *StubTargetAddr = createStubFunction(
1616 Addr: Section.getAddressWithOffset(OffsetBytes: Section.getStubOffset()),
1617 AbiVariant);
1618 RelocationEntry RE(SectionID, StubTargetAddr - Section.getAddress(),
1619 ELF::R_PPC64_ADDR64, Value.Addend);
1620
1621 // Generates the 64-bits address loads as exemplified in section
1622 // 4.5.1 in PPC64 ELF ABI. Note that the relocations need to
1623 // apply to the low part of the instructions, so we have to update
1624 // the offset according to the target endianness.
1625 uint64_t StubRelocOffset = StubTargetAddr - Section.getAddress();
1626 if (!IsTargetLittleEndian)
1627 StubRelocOffset += 2;
1628
1629 RelocationEntry REhst(SectionID, StubRelocOffset + 0,
1630 ELF::R_PPC64_ADDR16_HIGHEST, Value.Addend);
1631 RelocationEntry REhr(SectionID, StubRelocOffset + 4,
1632 ELF::R_PPC64_ADDR16_HIGHER, Value.Addend);
1633 RelocationEntry REh(SectionID, StubRelocOffset + 12,
1634 ELF::R_PPC64_ADDR16_HI, Value.Addend);
1635 RelocationEntry REl(SectionID, StubRelocOffset + 16,
1636 ELF::R_PPC64_ADDR16_LO, Value.Addend);
1637
1638 if (Value.SymbolName) {
1639 addRelocationForSymbol(RE: REhst, SymbolName: Value.SymbolName);
1640 addRelocationForSymbol(RE: REhr, SymbolName: Value.SymbolName);
1641 addRelocationForSymbol(RE: REh, SymbolName: Value.SymbolName);
1642 addRelocationForSymbol(RE: REl, SymbolName: Value.SymbolName);
1643 } else {
1644 addRelocationForSection(RE: REhst, SectionID: Value.SectionID);
1645 addRelocationForSection(RE: REhr, SectionID: Value.SectionID);
1646 addRelocationForSection(RE: REh, SectionID: Value.SectionID);
1647 addRelocationForSection(RE: REl, SectionID: Value.SectionID);
1648 }
1649
1650 resolveRelocation(
1651 Section, Offset,
1652 Value: Section.getLoadAddressWithOffset(OffsetBytes: Section.getStubOffset()),
1653 Type: RelType, Addend: 0);
1654 Section.advanceStubOffset(StubSize: getMaxStubSize());
1655 }
1656 if (IsExtern || (AbiVariant == 2 && Value.SectionID != SectionID)) {
1657 // Restore the TOC for external calls
1658 if (AbiVariant == 2)
1659 writeInt32BE(Addr: Target + 4, Value: 0xE8410018); // ld r2,24(r1)
1660 else
1661 writeInt32BE(Addr: Target + 4, Value: 0xE8410028); // ld r2,40(r1)
1662 }
1663 }
1664 } else if (RelType == ELF::R_PPC64_TOC16 ||
1665 RelType == ELF::R_PPC64_TOC16_DS ||
1666 RelType == ELF::R_PPC64_TOC16_LO ||
1667 RelType == ELF::R_PPC64_TOC16_LO_DS ||
1668 RelType == ELF::R_PPC64_TOC16_HI ||
1669 RelType == ELF::R_PPC64_TOC16_HA) {
1670 // These relocations are supposed to subtract the TOC address from
1671 // the final value. This does not fit cleanly into the RuntimeDyld
1672 // scheme, since there may be *two* sections involved in determining
1673 // the relocation value (the section of the symbol referred to by the
1674 // relocation, and the TOC section associated with the current module).
1675 //
1676 // Fortunately, these relocations are currently only ever generated
1677 // referring to symbols that themselves reside in the TOC, which means
1678 // that the two sections are actually the same. Thus they cancel out
1679 // and we can immediately resolve the relocation right now.
1680 switch (RelType) {
1681 case ELF::R_PPC64_TOC16: RelType = ELF::R_PPC64_ADDR16; break;
1682 case ELF::R_PPC64_TOC16_DS: RelType = ELF::R_PPC64_ADDR16_DS; break;
1683 case ELF::R_PPC64_TOC16_LO: RelType = ELF::R_PPC64_ADDR16_LO; break;
1684 case ELF::R_PPC64_TOC16_LO_DS: RelType = ELF::R_PPC64_ADDR16_LO_DS; break;
1685 case ELF::R_PPC64_TOC16_HI: RelType = ELF::R_PPC64_ADDR16_HI; break;
1686 case ELF::R_PPC64_TOC16_HA: RelType = ELF::R_PPC64_ADDR16_HA; break;
1687 default: llvm_unreachable("Wrong relocation type.");
1688 }
1689
1690 RelocationValueRef TOCValue;
1691 if (auto Err = findPPC64TOCSection(Obj, LocalSections&: ObjSectionToID, Rel&: TOCValue))
1692 return std::move(Err);
1693 if (Value.SymbolName || Value.SectionID != TOCValue.SectionID)
1694 llvm_unreachable("Unsupported TOC relocation.");
1695 Value.Addend -= TOCValue.Addend;
1696 resolveRelocation(Section: Sections[SectionID], Offset, Value: Value.Addend, Type: RelType, Addend: 0);
1697 } else {
1698 // There are two ways to refer to the TOC address directly: either
1699 // via a ELF::R_PPC64_TOC relocation (where both symbol and addend are
1700 // ignored), or via any relocation that refers to the magic ".TOC."
1701 // symbols (in which case the addend is respected).
1702 if (RelType == ELF::R_PPC64_TOC) {
1703 RelType = ELF::R_PPC64_ADDR64;
1704 if (auto Err = findPPC64TOCSection(Obj, LocalSections&: ObjSectionToID, Rel&: Value))
1705 return std::move(Err);
1706 } else if (TargetName == ".TOC.") {
1707 if (auto Err = findPPC64TOCSection(Obj, LocalSections&: ObjSectionToID, Rel&: Value))
1708 return std::move(Err);
1709 Value.Addend += Addend;
1710 }
1711
1712 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend);
1713
1714 if (Value.SymbolName)
1715 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1716 else
1717 addRelocationForSection(RE, SectionID: Value.SectionID);
1718 }
1719 } else if (Arch == Triple::systemz &&
1720 (RelType == ELF::R_390_PLT32DBL || RelType == ELF::R_390_GOTENT)) {
1721 // Create function stubs for both PLT and GOT references, regardless of
1722 // whether the GOT reference is to data or code. The stub contains the
1723 // full address of the symbol, as needed by GOT references, and the
1724 // executable part only adds an overhead of 8 bytes.
1725 //
1726 // We could try to conserve space by allocating the code and data
1727 // parts of the stub separately. However, as things stand, we allocate
1728 // a stub for every relocation, so using a GOT in JIT code should be
1729 // no less space efficient than using an explicit constant pool.
1730 LLVM_DEBUG(dbgs() << "\t\tThis is a SystemZ indirect relocation.");
1731 SectionEntry &Section = Sections[SectionID];
1732
1733 // Look for an existing stub.
1734 StubMap::const_iterator i = Stubs.find(x: Value);
1735 uintptr_t StubAddress;
1736 if (i != Stubs.end()) {
1737 StubAddress = uintptr_t(Section.getAddressWithOffset(OffsetBytes: i->second));
1738 LLVM_DEBUG(dbgs() << " Stub function found\n");
1739 } else {
1740 // Create a new stub function.
1741 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
1742
1743 uintptr_t BaseAddress = uintptr_t(Section.getAddress());
1744 StubAddress =
1745 alignTo(Size: BaseAddress + Section.getStubOffset(), A: getStubAlignment());
1746 unsigned StubOffset = StubAddress - BaseAddress;
1747
1748 Stubs[Value] = StubOffset;
1749 createStubFunction(Addr: (uint8_t *)StubAddress);
1750 RelocationEntry RE(SectionID, StubOffset + 8, ELF::R_390_64,
1751 Value.Offset);
1752 if (Value.SymbolName)
1753 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1754 else
1755 addRelocationForSection(RE, SectionID: Value.SectionID);
1756 Section.advanceStubOffset(StubSize: getMaxStubSize());
1757 }
1758
1759 if (RelType == ELF::R_390_GOTENT)
1760 resolveRelocation(Section, Offset, Value: StubAddress + 8, Type: ELF::R_390_PC32DBL,
1761 Addend);
1762 else
1763 resolveRelocation(Section, Offset, Value: StubAddress, Type: RelType, Addend);
1764 } else if (Arch == Triple::x86_64) {
1765 if (RelType == ELF::R_X86_64_PLT32) {
1766 // The way the PLT relocations normally work is that the linker allocates
1767 // the
1768 // PLT and this relocation makes a PC-relative call into the PLT. The PLT
1769 // entry will then jump to an address provided by the GOT. On first call,
1770 // the
1771 // GOT address will point back into PLT code that resolves the symbol. After
1772 // the first call, the GOT entry points to the actual function.
1773 //
1774 // For local functions we're ignoring all of that here and just replacing
1775 // the PLT32 relocation type with PC32, which will translate the relocation
1776 // into a PC-relative call directly to the function. For external symbols we
1777 // can't be sure the function will be within 2^32 bytes of the call site, so
1778 // we need to create a stub, which calls into the GOT. This case is
1779 // equivalent to the usual PLT implementation except that we use the stub
1780 // mechanism in RuntimeDyld (which puts stubs at the end of the section)
1781 // rather than allocating a PLT section.
1782 if (Value.SymbolName && MemMgr.allowStubAllocation()) {
1783 // This is a call to an external function.
1784 // Look for an existing stub.
1785 SectionEntry *Section = &Sections[SectionID];
1786 StubMap::const_iterator i = Stubs.find(x: Value);
1787 uintptr_t StubAddress;
1788 if (i != Stubs.end()) {
1789 StubAddress = uintptr_t(Section->getAddress()) + i->second;
1790 LLVM_DEBUG(dbgs() << " Stub function found\n");
1791 } else {
1792 // Create a new stub function (equivalent to a PLT entry).
1793 LLVM_DEBUG(dbgs() << " Create a new stub function\n");
1794
1795 uintptr_t BaseAddress = uintptr_t(Section->getAddress());
1796 StubAddress = alignTo(Size: BaseAddress + Section->getStubOffset(),
1797 A: getStubAlignment());
1798 unsigned StubOffset = StubAddress - BaseAddress;
1799 Stubs[Value] = StubOffset;
1800 createStubFunction(Addr: (uint8_t *)StubAddress);
1801
1802 // Bump our stub offset counter
1803 Section->advanceStubOffset(StubSize: getMaxStubSize());
1804
1805 // Allocate a GOT Entry
1806 uint64_t GOTOffset = allocateGOTEntries(no: 1);
1807 // This potentially creates a new Section which potentially
1808 // invalidates the Section pointer, so reload it.
1809 Section = &Sections[SectionID];
1810
1811 // The load of the GOT address has an addend of -4
1812 resolveGOTOffsetRelocation(SectionID, Offset: StubOffset + 2, GOTOffset: GOTOffset - 4,
1813 Type: ELF::R_X86_64_PC32);
1814
1815 // Fill in the value of the symbol we're targeting into the GOT
1816 addRelocationForSymbol(
1817 RE: computeGOTOffsetRE(GOTOffset, SymbolOffset: 0, Type: ELF::R_X86_64_64),
1818 SymbolName: Value.SymbolName);
1819 }
1820
1821 // Make the target call a call into the stub table.
1822 resolveRelocation(Section: *Section, Offset, Value: StubAddress, Type: ELF::R_X86_64_PC32,
1823 Addend);
1824 } else {
1825 Value.Addend += support::ulittle32_t::ref(
1826 computePlaceholderAddress(SectionID, Offset));
1827 processSimpleRelocation(SectionID, Offset, RelType: ELF::R_X86_64_PC32, Value);
1828 }
1829 } else if (RelType == ELF::R_X86_64_GOTPCREL ||
1830 RelType == ELF::R_X86_64_GOTPCRELX ||
1831 RelType == ELF::R_X86_64_REX_GOTPCRELX) {
1832 uint64_t GOTOffset = allocateGOTEntries(no: 1);
1833 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: GOTOffset + Addend,
1834 Type: ELF::R_X86_64_PC32);
1835
1836 // Fill in the value of the symbol we're targeting into the GOT
1837 RelocationEntry RE =
1838 computeGOTOffsetRE(GOTOffset, SymbolOffset: Value.Offset, Type: ELF::R_X86_64_64);
1839 if (Value.SymbolName)
1840 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1841 else
1842 addRelocationForSection(RE, SectionID: Value.SectionID);
1843 } else if (RelType == ELF::R_X86_64_GOT64) {
1844 // Fill in a 64-bit GOT offset.
1845 uint64_t GOTOffset = allocateGOTEntries(no: 1);
1846 resolveRelocation(Section: Sections[SectionID], Offset, Value: GOTOffset,
1847 Type: ELF::R_X86_64_64, Addend: 0);
1848
1849 // Fill in the value of the symbol we're targeting into the GOT
1850 RelocationEntry RE =
1851 computeGOTOffsetRE(GOTOffset, SymbolOffset: Value.Offset, Type: ELF::R_X86_64_64);
1852 if (Value.SymbolName)
1853 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1854 else
1855 addRelocationForSection(RE, SectionID: Value.SectionID);
1856 } else if (RelType == ELF::R_X86_64_GOTPC32) {
1857 // Materialize the address of the base of the GOT relative to the PC.
1858 // This doesn't create a GOT entry, but it does mean we need a GOT
1859 // section.
1860 (void)allocateGOTEntries(no: 0);
1861 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: Addend, Type: ELF::R_X86_64_PC32);
1862 } else if (RelType == ELF::R_X86_64_GOTPC64) {
1863 (void)allocateGOTEntries(no: 0);
1864 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: Addend, Type: ELF::R_X86_64_PC64);
1865 } else if (RelType == ELF::R_X86_64_GOTOFF64) {
1866 // GOTOFF relocations ultimately require a section difference relocation.
1867 (void)allocateGOTEntries(no: 0);
1868 processSimpleRelocation(SectionID, Offset, RelType, Value);
1869 } else if (RelType == ELF::R_X86_64_PC32) {
1870 Value.Addend += support::ulittle32_t::ref(computePlaceholderAddress(SectionID, Offset));
1871 processSimpleRelocation(SectionID, Offset, RelType, Value);
1872 } else if (RelType == ELF::R_X86_64_PC64) {
1873 Value.Addend += support::ulittle64_t::ref(computePlaceholderAddress(SectionID, Offset));
1874 processSimpleRelocation(SectionID, Offset, RelType, Value);
1875 } else if (RelType == ELF::R_X86_64_GOTTPOFF) {
1876 processX86_64GOTTPOFFRelocation(SectionID, Offset, Value, Addend);
1877 } else if (RelType == ELF::R_X86_64_TLSGD ||
1878 RelType == ELF::R_X86_64_TLSLD) {
1879 // The next relocation must be the relocation for __tls_get_addr.
1880 ++RelI;
1881 auto &GetAddrRelocation = *RelI;
1882 processX86_64TLSRelocation(SectionID, Offset, RelType, Value, Addend,
1883 GetAddrRelocation);
1884 } else {
1885 processSimpleRelocation(SectionID, Offset, RelType, Value);
1886 }
1887 } else {
1888 if (Arch == Triple::x86) {
1889 Value.Addend += support::ulittle32_t::ref(computePlaceholderAddress(SectionID, Offset));
1890 }
1891 processSimpleRelocation(SectionID, Offset, RelType, Value);
1892 }
1893 return ++RelI;
1894}
1895
1896void RuntimeDyldELF::processX86_64GOTTPOFFRelocation(unsigned SectionID,
1897 uint64_t Offset,
1898 RelocationValueRef Value,
1899 int64_t Addend) {
1900 // Use the approach from "x86-64 Linker Optimizations" from the TLS spec
1901 // to replace the GOTTPOFF relocation with a TPOFF relocation. The spec
1902 // only mentions one optimization even though there are two different
1903 // code sequences for the Initial Exec TLS Model. We match the code to
1904 // find out which one was used.
1905
1906 // A possible TLS code sequence and its replacement
1907 struct CodeSequence {
1908 // The expected code sequence
1909 ArrayRef<uint8_t> ExpectedCodeSequence;
1910 // The negative offset of the GOTTPOFF relocation to the beginning of
1911 // the sequence
1912 uint64_t TLSSequenceOffset;
1913 // The new code sequence
1914 ArrayRef<uint8_t> NewCodeSequence;
1915 // The offset of the new TPOFF relocation
1916 uint64_t TpoffRelocationOffset;
1917 };
1918
1919 std::array<CodeSequence, 2> CodeSequences;
1920
1921 // Initial Exec Code Model Sequence
1922 {
1923 static const std::initializer_list<uint8_t> ExpectedCodeSequenceList = {
1924 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00,
1925 0x00, // mov %fs:0, %rax
1926 0x48, 0x03, 0x05, 0x00, 0x00, 0x00, 0x00 // add x@gotpoff(%rip),
1927 // %rax
1928 };
1929 CodeSequences[0].ExpectedCodeSequence =
1930 ArrayRef<uint8_t>(ExpectedCodeSequenceList);
1931 CodeSequences[0].TLSSequenceOffset = 12;
1932
1933 static const std::initializer_list<uint8_t> NewCodeSequenceList = {
1934 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0, %rax
1935 0x48, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00 // lea x@tpoff(%rax), %rax
1936 };
1937 CodeSequences[0].NewCodeSequence = ArrayRef<uint8_t>(NewCodeSequenceList);
1938 CodeSequences[0].TpoffRelocationOffset = 12;
1939 }
1940
1941 // Initial Exec Code Model Sequence, II
1942 {
1943 static const std::initializer_list<uint8_t> ExpectedCodeSequenceList = {
1944 0x48, 0x8b, 0x05, 0x00, 0x00, 0x00, 0x00, // mov x@gotpoff(%rip), %rax
1945 0x64, 0x48, 0x8b, 0x00, 0x00, 0x00, 0x00 // mov %fs:(%rax), %rax
1946 };
1947 CodeSequences[1].ExpectedCodeSequence =
1948 ArrayRef<uint8_t>(ExpectedCodeSequenceList);
1949 CodeSequences[1].TLSSequenceOffset = 3;
1950
1951 static const std::initializer_list<uint8_t> NewCodeSequenceList = {
1952 0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00, // 6 byte nop
1953 0x64, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:x@tpoff, %rax
1954 };
1955 CodeSequences[1].NewCodeSequence = ArrayRef<uint8_t>(NewCodeSequenceList);
1956 CodeSequences[1].TpoffRelocationOffset = 10;
1957 }
1958
1959 bool Resolved = false;
1960 auto &Section = Sections[SectionID];
1961 for (const auto &C : CodeSequences) {
1962 assert(C.ExpectedCodeSequence.size() == C.NewCodeSequence.size() &&
1963 "Old and new code sequences must have the same size");
1964
1965 if (Offset < C.TLSSequenceOffset ||
1966 (Offset - C.TLSSequenceOffset + C.NewCodeSequence.size()) >
1967 Section.getSize()) {
1968 // This can't be a matching sequence as it doesn't fit in the current
1969 // section
1970 continue;
1971 }
1972
1973 auto TLSSequenceStartOffset = Offset - C.TLSSequenceOffset;
1974 auto *TLSSequence = Section.getAddressWithOffset(OffsetBytes: TLSSequenceStartOffset);
1975 if (ArrayRef<uint8_t>(TLSSequence, C.ExpectedCodeSequence.size()) !=
1976 C.ExpectedCodeSequence) {
1977 continue;
1978 }
1979
1980 memcpy(dest: TLSSequence, src: C.NewCodeSequence.data(), n: C.NewCodeSequence.size());
1981
1982 // The original GOTTPOFF relocation has an addend as it is PC relative,
1983 // so it needs to be corrected. The TPOFF32 relocation is used as an
1984 // absolute value (which is an offset from %fs:0), so remove the addend
1985 // again.
1986 RelocationEntry RE(SectionID,
1987 TLSSequenceStartOffset + C.TpoffRelocationOffset,
1988 ELF::R_X86_64_TPOFF32, Value.Addend - Addend);
1989
1990 if (Value.SymbolName)
1991 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
1992 else
1993 addRelocationForSection(RE, SectionID: Value.SectionID);
1994
1995 Resolved = true;
1996 break;
1997 }
1998
1999 if (!Resolved) {
2000 // The GOTTPOFF relocation was not used in one of the sequences
2001 // described in the spec, so we can't optimize it to a TPOFF
2002 // relocation.
2003 uint64_t GOTOffset = allocateGOTEntries(no: 1);
2004 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset: GOTOffset + Addend,
2005 Type: ELF::R_X86_64_PC32);
2006 RelocationEntry RE =
2007 computeGOTOffsetRE(GOTOffset, SymbolOffset: Value.Offset, Type: ELF::R_X86_64_TPOFF64);
2008 if (Value.SymbolName)
2009 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
2010 else
2011 addRelocationForSection(RE, SectionID: Value.SectionID);
2012 }
2013}
2014
2015void RuntimeDyldELF::processX86_64TLSRelocation(
2016 unsigned SectionID, uint64_t Offset, uint64_t RelType,
2017 RelocationValueRef Value, int64_t Addend,
2018 const RelocationRef &GetAddrRelocation) {
2019 // Since we are statically linking and have no additional DSOs, we can resolve
2020 // the relocation directly without using __tls_get_addr.
2021 // Use the approach from "x86-64 Linker Optimizations" from the TLS spec
2022 // to replace it with the Local Exec relocation variant.
2023
2024 // Find out whether the code was compiled with the large or small memory
2025 // model. For this we look at the next relocation which is the relocation
2026 // for the __tls_get_addr function. If it's a 32 bit relocation, it's the
2027 // small code model, with a 64 bit relocation it's the large code model.
2028 bool IsSmallCodeModel;
2029 // Is the relocation for the __tls_get_addr a PC-relative GOT relocation?
2030 bool IsGOTPCRel = false;
2031
2032 switch (GetAddrRelocation.getType()) {
2033 case ELF::R_X86_64_GOTPCREL:
2034 case ELF::R_X86_64_REX_GOTPCRELX:
2035 case ELF::R_X86_64_GOTPCRELX:
2036 IsGOTPCRel = true;
2037 [[fallthrough]];
2038 case ELF::R_X86_64_PLT32:
2039 IsSmallCodeModel = true;
2040 break;
2041 case ELF::R_X86_64_PLTOFF64:
2042 IsSmallCodeModel = false;
2043 break;
2044 default:
2045 report_fatal_error(
2046 reason: "invalid TLS relocations for General/Local Dynamic TLS Model: "
2047 "expected PLT or GOT relocation for __tls_get_addr function");
2048 }
2049
2050 // The negative offset to the start of the TLS code sequence relative to
2051 // the offset of the TLSGD/TLSLD relocation
2052 uint64_t TLSSequenceOffset;
2053 // The expected start of the code sequence
2054 ArrayRef<uint8_t> ExpectedCodeSequence;
2055 // The new TLS code sequence that will replace the existing code
2056 ArrayRef<uint8_t> NewCodeSequence;
2057
2058 if (RelType == ELF::R_X86_64_TLSGD) {
2059 // The offset of the new TPOFF32 relocation (offset starting from the
2060 // beginning of the whole TLS sequence)
2061 uint64_t TpoffRelocOffset;
2062
2063 if (IsSmallCodeModel) {
2064 if (!IsGOTPCRel) {
2065 static const std::initializer_list<uint8_t> CodeSequence = {
2066 0x66, // data16 (no-op prefix)
2067 0x48, 0x8d, 0x3d, 0x00, 0x00,
2068 0x00, 0x00, // lea <disp32>(%rip), %rdi
2069 0x66, 0x66, // two data16 prefixes
2070 0x48, // rex64 (no-op prefix)
2071 0xe8, 0x00, 0x00, 0x00, 0x00 // call __tls_get_addr@plt
2072 };
2073 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2074 TLSSequenceOffset = 4;
2075 } else {
2076 // This code sequence is not described in the TLS spec but gcc
2077 // generates it sometimes.
2078 static const std::initializer_list<uint8_t> CodeSequence = {
2079 0x66, // data16 (no-op prefix)
2080 0x48, 0x8d, 0x3d, 0x00, 0x00,
2081 0x00, 0x00, // lea <disp32>(%rip), %rdi
2082 0x66, // data16 prefix (no-op prefix)
2083 0x48, // rex64 (no-op prefix)
2084 0xff, 0x15, 0x00, 0x00, 0x00,
2085 0x00 // call *__tls_get_addr@gotpcrel(%rip)
2086 };
2087 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2088 TLSSequenceOffset = 4;
2089 }
2090
2091 // The replacement code for the small code model. It's the same for
2092 // both sequences.
2093 static const std::initializer_list<uint8_t> SmallSequence = {
2094 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00,
2095 0x00, // mov %fs:0, %rax
2096 0x48, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00 // lea x@tpoff(%rax),
2097 // %rax
2098 };
2099 NewCodeSequence = ArrayRef<uint8_t>(SmallSequence);
2100 TpoffRelocOffset = 12;
2101 } else {
2102 static const std::initializer_list<uint8_t> CodeSequence = {
2103 0x48, 0x8d, 0x3d, 0x00, 0x00, 0x00, 0x00, // lea <disp32>(%rip),
2104 // %rdi
2105 0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2106 0x00, // movabs $__tls_get_addr@pltoff, %rax
2107 0x48, 0x01, 0xd8, // add %rbx, %rax
2108 0xff, 0xd0 // call *%rax
2109 };
2110 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2111 TLSSequenceOffset = 3;
2112
2113 // The replacement code for the large code model
2114 static const std::initializer_list<uint8_t> LargeSequence = {
2115 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00,
2116 0x00, // mov %fs:0, %rax
2117 0x48, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00, // lea x@tpoff(%rax),
2118 // %rax
2119 0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00 // nopw 0x0(%rax,%rax,1)
2120 };
2121 NewCodeSequence = ArrayRef<uint8_t>(LargeSequence);
2122 TpoffRelocOffset = 12;
2123 }
2124
2125 // The TLSGD/TLSLD relocations are PC-relative, so they have an addend.
2126 // The new TPOFF32 relocations is used as an absolute offset from
2127 // %fs:0, so remove the TLSGD/TLSLD addend again.
2128 RelocationEntry RE(SectionID, Offset - TLSSequenceOffset + TpoffRelocOffset,
2129 ELF::R_X86_64_TPOFF32, Value.Addend - Addend);
2130 if (Value.SymbolName)
2131 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
2132 else
2133 addRelocationForSection(RE, SectionID: Value.SectionID);
2134 } else if (RelType == ELF::R_X86_64_TLSLD) {
2135 if (IsSmallCodeModel) {
2136 if (!IsGOTPCRel) {
2137 static const std::initializer_list<uint8_t> CodeSequence = {
2138 0x48, 0x8d, 0x3d, 0x00, 0x00, 0x00, // leaq <disp32>(%rip), %rdi
2139 0x00, 0xe8, 0x00, 0x00, 0x00, 0x00 // call __tls_get_addr@plt
2140 };
2141 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2142 TLSSequenceOffset = 3;
2143
2144 // The replacement code for the small code model
2145 static const std::initializer_list<uint8_t> SmallSequence = {
2146 0x66, 0x66, 0x66, // three data16 prefixes (no-op)
2147 0x64, 0x48, 0x8b, 0x04, 0x25,
2148 0x00, 0x00, 0x00, 0x00 // mov %fs:0, %rax
2149 };
2150 NewCodeSequence = ArrayRef<uint8_t>(SmallSequence);
2151 } else {
2152 // This code sequence is not described in the TLS spec but gcc
2153 // generates it sometimes.
2154 static const std::initializer_list<uint8_t> CodeSequence = {
2155 0x48, 0x8d, 0x3d, 0x00,
2156 0x00, 0x00, 0x00, // leaq <disp32>(%rip), %rdi
2157 0xff, 0x15, 0x00, 0x00,
2158 0x00, 0x00 // call
2159 // *__tls_get_addr@gotpcrel(%rip)
2160 };
2161 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2162 TLSSequenceOffset = 3;
2163
2164 // The replacement is code is just like above but it needs to be
2165 // one byte longer.
2166 static const std::initializer_list<uint8_t> SmallSequence = {
2167 0x0f, 0x1f, 0x40, 0x00, // 4 byte nop
2168 0x64, 0x48, 0x8b, 0x04, 0x25,
2169 0x00, 0x00, 0x00, 0x00 // mov %fs:0, %rax
2170 };
2171 NewCodeSequence = ArrayRef<uint8_t>(SmallSequence);
2172 }
2173 } else {
2174 // This is the same sequence as for the TLSGD sequence with the large
2175 // memory model above
2176 static const std::initializer_list<uint8_t> CodeSequence = {
2177 0x48, 0x8d, 0x3d, 0x00, 0x00, 0x00, 0x00, // lea <disp32>(%rip),
2178 // %rdi
2179 0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2180 0x48, // movabs $__tls_get_addr@pltoff, %rax
2181 0x01, 0xd8, // add %rbx, %rax
2182 0xff, 0xd0 // call *%rax
2183 };
2184 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);
2185 TLSSequenceOffset = 3;
2186
2187 // The replacement code for the large code model
2188 static const std::initializer_list<uint8_t> LargeSequence = {
2189 0x66, 0x66, 0x66, // three data16 prefixes (no-op)
2190 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00,
2191 0x00, // 10 byte nop
2192 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00 // mov %fs:0,%rax
2193 };
2194 NewCodeSequence = ArrayRef<uint8_t>(LargeSequence);
2195 }
2196 } else {
2197 llvm_unreachable("both TLS relocations handled above");
2198 }
2199
2200 assert(ExpectedCodeSequence.size() == NewCodeSequence.size() &&
2201 "Old and new code sequences must have the same size");
2202
2203 auto &Section = Sections[SectionID];
2204 if (Offset < TLSSequenceOffset ||
2205 (Offset - TLSSequenceOffset + NewCodeSequence.size()) >
2206 Section.getSize()) {
2207 report_fatal_error(reason: "unexpected end of section in TLS sequence");
2208 }
2209
2210 auto *TLSSequence = Section.getAddressWithOffset(OffsetBytes: Offset - TLSSequenceOffset);
2211 if (ArrayRef<uint8_t>(TLSSequence, ExpectedCodeSequence.size()) !=
2212 ExpectedCodeSequence) {
2213 report_fatal_error(
2214 reason: "invalid TLS sequence for Global/Local Dynamic TLS Model");
2215 }
2216
2217 memcpy(dest: TLSSequence, src: NewCodeSequence.data(), n: NewCodeSequence.size());
2218}
2219
2220size_t RuntimeDyldELF::getGOTEntrySize() {
2221 // We don't use the GOT in all of these cases, but it's essentially free
2222 // to put them all here.
2223 size_t Result = 0;
2224 switch (Arch) {
2225 case Triple::x86_64:
2226 case Triple::aarch64:
2227 case Triple::aarch64_be:
2228 case Triple::ppc64:
2229 case Triple::ppc64le:
2230 case Triple::systemz:
2231 Result = sizeof(uint64_t);
2232 break;
2233 case Triple::x86:
2234 case Triple::arm:
2235 case Triple::thumb:
2236 Result = sizeof(uint32_t);
2237 break;
2238 case Triple::mips:
2239 case Triple::mipsel:
2240 case Triple::mips64:
2241 case Triple::mips64el:
2242 if (IsMipsO32ABI || IsMipsN32ABI)
2243 Result = sizeof(uint32_t);
2244 else if (IsMipsN64ABI)
2245 Result = sizeof(uint64_t);
2246 else
2247 llvm_unreachable("Mips ABI not handled");
2248 break;
2249 default:
2250 llvm_unreachable("Unsupported CPU type!");
2251 }
2252 return Result;
2253}
2254
2255uint64_t RuntimeDyldELF::allocateGOTEntries(unsigned no) {
2256 if (GOTSectionID == 0) {
2257 GOTSectionID = Sections.size();
2258 // Reserve a section id. We'll allocate the section later
2259 // once we know the total size
2260 Sections.push_back(x: SectionEntry(".got", nullptr, 0, 0, 0));
2261 }
2262 uint64_t StartOffset = CurrentGOTIndex * getGOTEntrySize();
2263 CurrentGOTIndex += no;
2264 return StartOffset;
2265}
2266
2267uint64_t RuntimeDyldELF::findOrAllocGOTEntry(const RelocationValueRef &Value,
2268 unsigned GOTRelType) {
2269 auto E = GOTOffsetMap.insert(x: {Value, 0});
2270 if (E.second) {
2271 uint64_t GOTOffset = allocateGOTEntries(no: 1);
2272
2273 // Create relocation for newly created GOT entry
2274 RelocationEntry RE =
2275 computeGOTOffsetRE(GOTOffset, SymbolOffset: Value.Offset, Type: GOTRelType);
2276 if (Value.SymbolName)
2277 addRelocationForSymbol(RE, SymbolName: Value.SymbolName);
2278 else
2279 addRelocationForSection(RE, SectionID: Value.SectionID);
2280
2281 E.first->second = GOTOffset;
2282 }
2283
2284 return E.first->second;
2285}
2286
2287void RuntimeDyldELF::resolveGOTOffsetRelocation(unsigned SectionID,
2288 uint64_t Offset,
2289 uint64_t GOTOffset,
2290 uint32_t Type) {
2291 // Fill in the relative address of the GOT Entry into the stub
2292 RelocationEntry GOTRE(SectionID, Offset, Type, GOTOffset);
2293 addRelocationForSection(RE: GOTRE, SectionID: GOTSectionID);
2294}
2295
2296RelocationEntry RuntimeDyldELF::computeGOTOffsetRE(uint64_t GOTOffset,
2297 uint64_t SymbolOffset,
2298 uint32_t Type) {
2299 return RelocationEntry(GOTSectionID, GOTOffset, Type, SymbolOffset);
2300}
2301
2302void RuntimeDyldELF::processNewSymbol(const SymbolRef &ObjSymbol, SymbolTableEntry& Symbol) {
2303 // This should never return an error as `processNewSymbol` wouldn't have been
2304 // called if getFlags() returned an error before.
2305 auto ObjSymbolFlags = cantFail(ValOrErr: ObjSymbol.getFlags());
2306
2307 if (ObjSymbolFlags & SymbolRef::SF_Indirect) {
2308 if (IFuncStubSectionID == 0) {
2309 // Create a dummy section for the ifunc stubs. It will be actually
2310 // allocated in finalizeLoad() below.
2311 IFuncStubSectionID = Sections.size();
2312 Sections.push_back(
2313 x: SectionEntry(".text.__llvm_IFuncStubs", nullptr, 0, 0, 0));
2314 // First 64B are reserverd for the IFunc resolver
2315 IFuncStubOffset = 64;
2316 }
2317
2318 IFuncStubs.push_back(Elt: IFuncStub{.StubOffset: IFuncStubOffset, .OriginalSymbol: Symbol});
2319 // Modify the symbol so that it points to the ifunc stub instead of to the
2320 // resolver function.
2321 Symbol = SymbolTableEntry(IFuncStubSectionID, IFuncStubOffset,
2322 Symbol.getFlags());
2323 IFuncStubOffset += getMaxIFuncStubSize();
2324 }
2325}
2326
2327Error RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj,
2328 ObjSectionToIDMap &SectionMap) {
2329 if (IsMipsO32ABI)
2330 if (!PendingRelocs.empty())
2331 return make_error<RuntimeDyldError>(Args: "Can't find matching LO16 reloc");
2332
2333 // Create the IFunc stubs if necessary. This must be done before processing
2334 // the GOT entries, as the IFunc stubs may create some.
2335 if (IFuncStubSectionID != 0) {
2336 uint8_t *IFuncStubsAddr = MemMgr.allocateCodeSection(
2337 Size: IFuncStubOffset, Alignment: 1, SectionID: IFuncStubSectionID, SectionName: ".text.__llvm_IFuncStubs");
2338 if (!IFuncStubsAddr)
2339 return make_error<RuntimeDyldError>(
2340 Args: "Unable to allocate memory for IFunc stubs!");
2341 Sections[IFuncStubSectionID] =
2342 SectionEntry(".text.__llvm_IFuncStubs", IFuncStubsAddr, IFuncStubOffset,
2343 IFuncStubOffset, 0);
2344
2345 createIFuncResolver(Addr: IFuncStubsAddr);
2346
2347 LLVM_DEBUG(dbgs() << "Creating IFunc stubs SectionID: "
2348 << IFuncStubSectionID << " Addr: "
2349 << Sections[IFuncStubSectionID].getAddress() << '\n');
2350 for (auto &IFuncStub : IFuncStubs) {
2351 auto &Symbol = IFuncStub.OriginalSymbol;
2352 LLVM_DEBUG(dbgs() << "\tSectionID: " << Symbol.getSectionID()
2353 << " Offset: " << format("%p", Symbol.getOffset())
2354 << " IFuncStubOffset: "
2355 << format("%p\n", IFuncStub.StubOffset));
2356 createIFuncStub(IFuncStubSectionID, IFuncResolverOffset: 0, IFuncStubOffset: IFuncStub.StubOffset,
2357 IFuncSectionID: Symbol.getSectionID(), IFuncOffset: Symbol.getOffset());
2358 }
2359
2360 IFuncStubSectionID = 0;
2361 IFuncStubOffset = 0;
2362 IFuncStubs.clear();
2363 }
2364
2365 // If necessary, allocate the global offset table
2366 if (GOTSectionID != 0) {
2367 // Allocate memory for the section
2368 size_t TotalSize = CurrentGOTIndex * getGOTEntrySize();
2369 uint8_t *Addr = MemMgr.allocateDataSection(Size: TotalSize, Alignment: getGOTEntrySize(),
2370 SectionID: GOTSectionID, SectionName: ".got", IsReadOnly: false);
2371 if (!Addr)
2372 return make_error<RuntimeDyldError>(Args: "Unable to allocate memory for GOT!");
2373
2374 Sections[GOTSectionID] =
2375 SectionEntry(".got", Addr, TotalSize, TotalSize, 0);
2376
2377 // For now, initialize all GOT entries to zero. We'll fill them in as
2378 // needed when GOT-based relocations are applied.
2379 memset(s: Addr, c: 0, n: TotalSize);
2380 if (IsMipsN32ABI || IsMipsN64ABI) {
2381 // To correctly resolve Mips GOT relocations, we need a mapping from
2382 // object's sections to GOTs.
2383 for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end();
2384 SI != SE; ++SI) {
2385 if (SI->relocation_begin() != SI->relocation_end()) {
2386 Expected<section_iterator> RelSecOrErr = SI->getRelocatedSection();
2387 if (!RelSecOrErr)
2388 return make_error<RuntimeDyldError>(
2389 Args: toString(E: RelSecOrErr.takeError()));
2390
2391 section_iterator RelocatedSection = *RelSecOrErr;
2392 ObjSectionToIDMap::iterator i = SectionMap.find(x: *RelocatedSection);
2393 assert(i != SectionMap.end());
2394 SectionToGOTMap[i->second] = GOTSectionID;
2395 }
2396 }
2397 GOTSymbolOffsets.clear();
2398 }
2399 }
2400
2401 // Look for and record the EH frame section.
2402 ObjSectionToIDMap::iterator i, e;
2403 for (i = SectionMap.begin(), e = SectionMap.end(); i != e; ++i) {
2404 const SectionRef &Section = i->first;
2405
2406 StringRef Name;
2407 Expected<StringRef> NameOrErr = Section.getName();
2408 if (NameOrErr)
2409 Name = *NameOrErr;
2410 else
2411 consumeError(Err: NameOrErr.takeError());
2412
2413 if (Name == ".eh_frame") {
2414 UnregisteredEHFrameSections.push_back(Elt: i->second);
2415 break;
2416 }
2417 }
2418
2419 GOTOffsetMap.clear();
2420 GOTSectionID = 0;
2421 CurrentGOTIndex = 0;
2422
2423 return Error::success();
2424}
2425
2426bool RuntimeDyldELF::isCompatibleFile(const object::ObjectFile &Obj) const {
2427 return Obj.isELF();
2428}
2429
2430void RuntimeDyldELF::createIFuncResolver(uint8_t *Addr) const {
2431 if (Arch == Triple::x86_64) {
2432 // The adddres of the GOT1 entry is in %r11, the GOT2 entry is in %r11+8
2433 // (see createIFuncStub() for details)
2434 // The following code first saves all registers that contain the original
2435 // function arguments as those registers are not saved by the resolver
2436 // function. %r11 is saved as well so that the GOT2 entry can be updated
2437 // afterwards. Then it calls the actual IFunc resolver function whose
2438 // address is stored in GOT2. After the resolver function returns, all
2439 // saved registers are restored and the return value is written to GOT1.
2440 // Finally, jump to the now resolved function.
2441 // clang-format off
2442 const uint8_t StubCode[] = {
2443 0x57, // push %rdi
2444 0x56, // push %rsi
2445 0x52, // push %rdx
2446 0x51, // push %rcx
2447 0x41, 0x50, // push %r8
2448 0x41, 0x51, // push %r9
2449 0x41, 0x53, // push %r11
2450 0x41, 0xff, 0x53, 0x08, // call *0x8(%r11)
2451 0x41, 0x5b, // pop %r11
2452 0x41, 0x59, // pop %r9
2453 0x41, 0x58, // pop %r8
2454 0x59, // pop %rcx
2455 0x5a, // pop %rdx
2456 0x5e, // pop %rsi
2457 0x5f, // pop %rdi
2458 0x49, 0x89, 0x03, // mov %rax,(%r11)
2459 0xff, 0xe0 // jmp *%rax
2460 };
2461 // clang-format on
2462 static_assert(sizeof(StubCode) <= 64,
2463 "maximum size of the IFunc resolver is 64B");
2464 memcpy(dest: Addr, src: StubCode, n: sizeof(StubCode));
2465 } else {
2466 report_fatal_error(
2467 reason: "IFunc resolver is not supported for target architecture");
2468 }
2469}
2470
2471void RuntimeDyldELF::createIFuncStub(unsigned IFuncStubSectionID,
2472 uint64_t IFuncResolverOffset,
2473 uint64_t IFuncStubOffset,
2474 unsigned IFuncSectionID,
2475 uint64_t IFuncOffset) {
2476 auto &IFuncStubSection = Sections[IFuncStubSectionID];
2477 auto *Addr = IFuncStubSection.getAddressWithOffset(OffsetBytes: IFuncStubOffset);
2478
2479 if (Arch == Triple::x86_64) {
2480 // The first instruction loads a PC-relative address into %r11 which is a
2481 // GOT entry for this stub. This initially contains the address to the
2482 // IFunc resolver. We can use %r11 here as it's caller saved but not used
2483 // to pass any arguments. In fact, x86_64 ABI even suggests using %r11 for
2484 // code in the PLT. The IFunc resolver will use %r11 to update the GOT
2485 // entry.
2486 //
2487 // The next instruction just jumps to the address contained in the GOT
2488 // entry. As mentioned above, we do this two-step jump by first setting
2489 // %r11 so that the IFunc resolver has access to it.
2490 //
2491 // The IFunc resolver of course also needs to know the actual address of
2492 // the actual IFunc resolver function. This will be stored in a GOT entry
2493 // right next to the first one for this stub. So, the IFunc resolver will
2494 // be able to call it with %r11+8.
2495 //
2496 // In total, two adjacent GOT entries (+relocation) and one additional
2497 // relocation are required:
2498 // GOT1: Address of the IFunc resolver.
2499 // GOT2: Address of the IFunc resolver function.
2500 // IFuncStubOffset+3: 32-bit PC-relative address of GOT1.
2501 uint64_t GOT1 = allocateGOTEntries(no: 2);
2502 uint64_t GOT2 = GOT1 + getGOTEntrySize();
2503
2504 RelocationEntry RE1(GOTSectionID, GOT1, ELF::R_X86_64_64,
2505 IFuncResolverOffset, {});
2506 addRelocationForSection(RE: RE1, SectionID: IFuncStubSectionID);
2507 RelocationEntry RE2(GOTSectionID, GOT2, ELF::R_X86_64_64, IFuncOffset, {});
2508 addRelocationForSection(RE: RE2, SectionID: IFuncSectionID);
2509
2510 const uint8_t StubCode[] = {
2511 0x4c, 0x8d, 0x1d, 0x00, 0x00, 0x00, 0x00, // leaq 0x0(%rip),%r11
2512 0x41, 0xff, 0x23 // jmpq *(%r11)
2513 };
2514 assert(sizeof(StubCode) <= getMaxIFuncStubSize() &&
2515 "IFunc stub size must not exceed getMaxIFuncStubSize()");
2516 memcpy(dest: Addr, src: StubCode, n: sizeof(StubCode));
2517
2518 // The PC-relative value starts 4 bytes from the end of the leaq
2519 // instruction, so the addend is -4.
2520 resolveGOTOffsetRelocation(SectionID: IFuncStubSectionID, Offset: IFuncStubOffset + 3,
2521 GOTOffset: GOT1 - 4, Type: ELF::R_X86_64_PC32);
2522 } else {
2523 report_fatal_error(reason: "IFunc stub is not supported for target architecture");
2524 }
2525}
2526
2527unsigned RuntimeDyldELF::getMaxIFuncStubSize() const {
2528 if (Arch == Triple::x86_64) {
2529 return 10;
2530 }
2531 return 0;
2532}
2533
2534bool RuntimeDyldELF::relocationNeedsGot(const RelocationRef &R) const {
2535 unsigned RelTy = R.getType();
2536 if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be)
2537 return RelTy == ELF::R_AARCH64_ADR_GOT_PAGE ||
2538 RelTy == ELF::R_AARCH64_LD64_GOT_LO12_NC;
2539
2540 if (Arch == Triple::x86_64)
2541 return RelTy == ELF::R_X86_64_GOTPCREL ||
2542 RelTy == ELF::R_X86_64_GOTPCRELX ||
2543 RelTy == ELF::R_X86_64_GOT64 ||
2544 RelTy == ELF::R_X86_64_REX_GOTPCRELX;
2545 return false;
2546}
2547
2548bool RuntimeDyldELF::relocationNeedsStub(const RelocationRef &R) const {
2549 if (Arch != Triple::x86_64)
2550 return true; // Conservative answer
2551
2552 switch (R.getType()) {
2553 default:
2554 return true; // Conservative answer
2555
2556
2557 case ELF::R_X86_64_GOTPCREL:
2558 case ELF::R_X86_64_GOTPCRELX:
2559 case ELF::R_X86_64_REX_GOTPCRELX:
2560 case ELF::R_X86_64_GOTPC64:
2561 case ELF::R_X86_64_GOT64:
2562 case ELF::R_X86_64_GOTOFF64:
2563 case ELF::R_X86_64_PC32:
2564 case ELF::R_X86_64_PC64:
2565 case ELF::R_X86_64_64:
2566 // We know that these reloation types won't need a stub function. This list
2567 // can be extended as needed.
2568 return false;
2569 }
2570}
2571
2572} // namespace llvm
2573