1//===- X86_64.cpp ---------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "OutputSections.h"
10#include "RelocScan.h"
11#include "Relocations.h"
12#include "Symbols.h"
13#include "SyntheticSections.h"
14#include "Target.h"
15#include "TargetImpl.h"
16#include "llvm/BinaryFormat/ELF.h"
17#include "llvm/Support/Endian.h"
18#include "llvm/Support/MathExtras.h"
19
20using namespace llvm;
21using namespace llvm::object;
22using namespace llvm::support::endian;
23using namespace llvm::ELF;
24using namespace lld;
25using namespace lld::elf;
26
27namespace {
28class X86_64 : public TargetInfo {
29public:
30 X86_64(Ctx &);
31 void initTargetSpecificSections() override;
32 RelExpr getRelExpr(RelType type, const Symbol &s,
33 const uint8_t *loc) const override;
34 RelType getDynRel(RelType type) const override;
35 void writeGotPltHeader(uint8_t *buf) const override;
36 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
37 void writeIgotPlt(uint8_t *buf, const Symbol &s) const override;
38 void writePltHeader(uint8_t *buf) const override;
39 void writePlt(uint8_t *buf, const Symbol &sym,
40 uint64_t pltEntryAddr) const override;
41 void relocate(uint8_t *loc, const Relocation &rel,
42 uint64_t val) const override;
43 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;
44 void applyJumpInstrMod(uint8_t *loc, JumpModType type,
45 unsigned size) const override;
46 RelExpr adjustGotPcExpr(RelType type, int64_t addend,
47 const uint8_t *loc) const override;
48 void relocateAlloc(InputSection &sec, uint8_t *buf) const override;
49 bool adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end,
50 uint8_t stOther) const override;
51 bool deleteFallThruJmpInsn(InputSection &is,
52 InputSection *nextIS) const override;
53 bool relaxOnce(int pass) const override;
54 void relaxCFIJumpTables() const override;
55 void applyBranchToBranchOpt() const override;
56 template <class ELFT, class RelTy>
57 void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels);
58 void scanSection(InputSectionBase &sec) override;
59
60private:
61 void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
62 void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
63 void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
64 void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
65};
66} // namespace
67
68// This is vector of NOP instructions of sizes from 1 to 8 bytes. The
69// appropriately sized instructions are used to fill the gaps between sections
70// which are executed during fall through.
71static const std::vector<std::vector<uint8_t>> nopInstructions = {
72 {0x90},
73 {0x66, 0x90},
74 {0x0f, 0x1f, 0x00},
75 {0x0f, 0x1f, 0x40, 0x00},
76 {0x0f, 0x1f, 0x44, 0x00, 0x00},
77 {0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00},
78 {0x0F, 0x1F, 0x80, 0x00, 0x00, 0x00, 0x00},
79 {0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00},
80 {0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}};
81
82X86_64::X86_64(Ctx &ctx) : TargetInfo(ctx) {
83 copyRel = R_X86_64_COPY;
84 gotRel = R_X86_64_GLOB_DAT;
85 pltRel = R_X86_64_JUMP_SLOT;
86 relativeRel = R_X86_64_RELATIVE;
87 iRelativeRel = R_X86_64_IRELATIVE;
88 symbolicRel = ctx.arg.is64 ? R_X86_64_64 : R_X86_64_32;
89 tlsDescRel = R_X86_64_TLSDESC;
90 tlsGotRel = R_X86_64_TPOFF64;
91 tlsModuleIndexRel = R_X86_64_DTPMOD64;
92 tlsOffsetRel = R_X86_64_DTPOFF64;
93 gotBaseSymInGotPlt = true;
94 gotEntrySize = 8;
95 pltHeaderSize = 16;
96 pltEntrySize = 16;
97 ipltEntrySize = 16;
98 trapInstr = {0xcc, 0xcc, 0xcc, 0xcc}; // 0xcc = INT3
99 nopInstrs = nopInstructions;
100
101 // Align to the large page size (known as a superpage or huge page).
102 // FreeBSD automatically promotes large, superpage-aligned allocations.
103 defaultImageBase = 0x200000;
104}
105
106// Opcodes for the different X86_64 jmp instructions.
107enum JmpInsnOpcode : uint32_t {
108 J_JMP_32,
109 J_JNE_32,
110 J_JE_32,
111 J_JG_32,
112 J_JGE_32,
113 J_JB_32,
114 J_JBE_32,
115 J_JL_32,
116 J_JLE_32,
117 J_JA_32,
118 J_JAE_32,
119 J_UNKNOWN,
120};
121
122// Given the first (optional) and second byte of the insn's opcode, this
123// returns the corresponding enum value.
124static JmpInsnOpcode getJmpInsnType(const uint8_t *first,
125 const uint8_t *second) {
126 if (*second == 0xe9)
127 return J_JMP_32;
128
129 if (first == nullptr)
130 return J_UNKNOWN;
131
132 if (*first == 0x0f) {
133 switch (*second) {
134 case 0x84:
135 return J_JE_32;
136 case 0x85:
137 return J_JNE_32;
138 case 0x8f:
139 return J_JG_32;
140 case 0x8d:
141 return J_JGE_32;
142 case 0x82:
143 return J_JB_32;
144 case 0x86:
145 return J_JBE_32;
146 case 0x8c:
147 return J_JL_32;
148 case 0x8e:
149 return J_JLE_32;
150 case 0x87:
151 return J_JA_32;
152 case 0x83:
153 return J_JAE_32;
154 }
155 }
156 return J_UNKNOWN;
157}
158
159// Return the relocation index for input section IS with a specific Offset.
160// Returns the maximum size of the vector if no such relocation is found.
161static unsigned getRelocationWithOffset(const InputSection &is,
162 uint64_t offset) {
163 unsigned size = is.relocs().size();
164 for (unsigned i = size - 1; i + 1 > 0; --i) {
165 if (is.relocs()[i].offset == offset && is.relocs()[i].expr != R_NONE)
166 return i;
167 }
168 return size;
169}
170
171// Returns true if R corresponds to a relocation used for a jump instruction.
172// TODO: Once special relocations for relaxable jump instructions are available,
173// this should be modified to use those relocations.
174static bool isRelocationForJmpInsn(Relocation &R) {
175 return R.type == R_X86_64_PLT32 || R.type == R_X86_64_PC32 ||
176 R.type == R_X86_64_PC8;
177}
178
179// Return true if Relocation R points to the first instruction in the
180// next section.
181// TODO: Delete this once psABI reserves a new relocation type for fall thru
182// jumps.
183static bool isFallThruRelocation(InputSection &is, InputSection *nextIS,
184 Relocation &r) {
185 if (!isRelocationForJmpInsn(R&: r))
186 return false;
187
188 uint64_t addrLoc = is.getOutputSection()->addr + is.outSecOff + r.offset;
189 uint64_t targetOffset = is.getRelocTargetVA(is.getCtx(), r, p: addrLoc);
190
191 // If this jmp is a fall thru, the target offset is the beginning of the
192 // next section.
193 uint64_t nextSectionOffset =
194 nextIS->getOutputSection()->addr + nextIS->outSecOff;
195 return (addrLoc + 4 + targetOffset) == nextSectionOffset;
196}
197
198// Return the jmp instruction opcode that is the inverse of the given
199// opcode. For example, JE inverted is JNE.
200static JmpInsnOpcode invertJmpOpcode(const JmpInsnOpcode opcode) {
201 switch (opcode) {
202 case J_JE_32:
203 return J_JNE_32;
204 case J_JNE_32:
205 return J_JE_32;
206 case J_JG_32:
207 return J_JLE_32;
208 case J_JGE_32:
209 return J_JL_32;
210 case J_JB_32:
211 return J_JAE_32;
212 case J_JBE_32:
213 return J_JA_32;
214 case J_JL_32:
215 return J_JGE_32;
216 case J_JLE_32:
217 return J_JG_32;
218 case J_JA_32:
219 return J_JBE_32;
220 case J_JAE_32:
221 return J_JB_32;
222 default:
223 return J_UNKNOWN;
224 }
225}
226
227// Deletes direct jump instruction in input sections that jumps to the
228// following section as it is not required. If there are two consecutive jump
229// instructions, it checks if they can be flipped and one can be deleted.
230// For example:
231// .section .text
232// a.BB.foo:
233// ...
234// 10: jne aa.BB.foo
235// 16: jmp bar
236// aa.BB.foo:
237// ...
238//
239// can be converted to:
240// a.BB.foo:
241// ...
242// 10: je bar #jne flipped to je and the jmp is deleted.
243// aa.BB.foo:
244// ...
245bool X86_64::deleteFallThruJmpInsn(InputSection &is,
246 InputSection *nextIS) const {
247 const unsigned sizeOfDirectJmpInsn = 5;
248
249 if (nextIS == nullptr)
250 return false;
251
252 if (is.getSize() < sizeOfDirectJmpInsn)
253 return false;
254
255 // If this jmp insn can be removed, it is the last insn and the
256 // relocation is 4 bytes before the end.
257 unsigned rIndex = getRelocationWithOffset(is, offset: is.getSize() - 4);
258 if (rIndex == is.relocs().size())
259 return false;
260
261 Relocation &r = is.relocs()[rIndex];
262
263 // Check if the relocation corresponds to a direct jmp.
264 const uint8_t *secContents = is.content().data();
265 // If it is not a direct jmp instruction, there is nothing to do here.
266 if (*(secContents + r.offset - 1) != 0xe9)
267 return false;
268
269 if (isFallThruRelocation(is, nextIS, r)) {
270 // This is a fall thru and can be deleted.
271 r.expr = R_NONE;
272 r.offset = 0;
273 is.drop_back(num: sizeOfDirectJmpInsn);
274 is.nopFiller = true;
275 return true;
276 }
277
278 // Now, check if flip and delete is possible.
279 const unsigned sizeOfJmpCCInsn = 6;
280 // To flip, there must be at least one JmpCC and one direct jmp.
281 if (is.getSize() < sizeOfDirectJmpInsn + sizeOfJmpCCInsn)
282 return false;
283
284 unsigned rbIndex =
285 getRelocationWithOffset(is, offset: (is.getSize() - sizeOfDirectJmpInsn - 4));
286 if (rbIndex == is.relocs().size())
287 return false;
288
289 Relocation &rB = is.relocs()[rbIndex];
290
291 const uint8_t *jmpInsnB = secContents + rB.offset - 1;
292 JmpInsnOpcode jmpOpcodeB = getJmpInsnType(first: jmpInsnB - 1, second: jmpInsnB);
293 if (jmpOpcodeB == J_UNKNOWN)
294 return false;
295
296 if (!isFallThruRelocation(is, nextIS, r&: rB))
297 return false;
298
299 // jmpCC jumps to the fall thru block, the branch can be flipped and the
300 // jmp can be deleted.
301 JmpInsnOpcode jInvert = invertJmpOpcode(opcode: jmpOpcodeB);
302 if (jInvert == J_UNKNOWN)
303 return false;
304 is.jumpInstrMod = make<JumpInstrMod>();
305 *is.jumpInstrMod = {.offset: rB.offset - 1, .original: jInvert, .size: 4};
306 // Move R's values to rB except the offset.
307 rB = {.expr: r.expr, .type: r.type, .offset: rB.offset, .addend: r.addend, .sym: r.sym};
308 // Cancel R
309 r.expr = R_NONE;
310 r.offset = 0;
311 is.drop_back(num: sizeOfDirectJmpInsn);
312 is.nopFiller = true;
313 return true;
314}
315
316void X86_64::relaxCFIJumpTables() const {
317 // Relax CFI jump tables.
318 // - Split jump table into pieces and place target functions inside the jump
319 // table if small enough.
320 // - Move jump table before last called function and delete last branch
321 // instruction.
322 DenseMap<InputSection *, SmallVector<InputSection *, 0>> sectionReplacements;
323 SmallVector<InputSection *, 0> storage;
324 for (OutputSection *osec : ctx.outputSections) {
325 if (!(osec->flags & SHF_EXECINSTR))
326 continue;
327 for (InputSection *sec : getInputSections(os: *osec, storage)) {
328 if (sec->type != SHT_LLVM_CFI_JUMP_TABLE || sec->entsize == 0 ||
329 sec->size % sec->entsize != 0)
330 continue;
331
332 // We're going to replace the jump table with this list of sections. This
333 // list will be made up of slices of the original section and function
334 // bodies that were moved into the jump table.
335 SmallVector<InputSection *, 0> replacements;
336
337 // r is the only relocation in a jump table entry. Figure out whether it
338 // is a branch pointing to the start of a statically known section that
339 // hasn't already been moved while processing a different jump table
340 // section, and if so return it.
341 auto getMovableSection = [&](Relocation &r) -> InputSection * {
342 if (r.type != R_X86_64_PC32 && r.type != R_X86_64_PLT32)
343 return nullptr;
344 auto *sym = dyn_cast<Defined>(Val: r.sym);
345 if (!sym || sym->isPreemptible || sym->isGnuIFunc() ||
346 sym->value + r.addend != -4ull) // Usual addend for branch targets.
347 return nullptr;
348 auto *target = dyn_cast_or_null<InputSection>(Val: sym->section);
349 if (!target || sectionReplacements.count(Val: target))
350 return nullptr;
351 return target;
352 };
353
354 // Figure out the movable section for the last entry. We do this first
355 // because the last entry controls which output section the jump table is
356 // placed into, which affects move eligibility for other sections.
357 auto *lastSec = [&]() -> InputSection * {
358 // If the jump table section is more aligned than the entry size, skip
359 // this because there's no guarantee that we'll be able to emit a
360 // padding section that places the last entry at a correctly aligned
361 // address.
362 if (sec->addralign > sec->entsize)
363 return nullptr;
364
365 auto rels = sec->relocs();
366 if (rels.empty() || rels.back().offset < sec->size - sec->entsize)
367 return nullptr;
368 if (rels.size() >= 2 &&
369 rels[rels.size() - 2].offset >= sec->size - sec->entsize)
370 return nullptr;
371 return getMovableSection(rels.back());
372 }();
373 OutputSection *targetOutputSec;
374 if (lastSec) {
375 // If the last section is more aligned than the jump table, we need
376 // to emit a padding section before the jump table to ensure that the
377 // last section ends up at the correct alignment.
378 if (lastSec->addralign > sec->addralign) {
379 // We need to add enough padding to make this equal to zero.
380 size_t mod = (sec->size - sec->entsize) % lastSec->addralign;
381 if (mod != 0) {
382 auto *pad = make<PaddingSection>(args&: ctx, args: lastSec->addralign - mod,
383 args: lastSec->getParent());
384 pad->addralign = lastSec->addralign;
385 replacements.push_back(Elt: pad);
386 } else {
387 sec->addralign = lastSec->addralign;
388 }
389 }
390
391 // We've already decided to move the output section so make sure that we
392 // don't try to move it again.
393 sectionReplacements[lastSec] = {};
394 targetOutputSec = lastSec->getParent();
395 } else {
396 targetOutputSec = sec->getParent();
397 }
398
399 // First, push the original jump table section. This is only so that it
400 // can act as a relocation target. Later on, we will set the size of the
401 // jump table section to 0 so that the slices and moved function bodies
402 // become the actual relocation targets.
403 replacements.push_back(Elt: sec);
404
405 // Add the slice [begin, end) of the original section to the replacement
406 // list. [rbegin, rend) is the slice of the relocation list that covers
407 // [begin, end).
408 auto addSectionSlice = [&](size_t begin, size_t end, Relocation *rbegin,
409 Relocation *rend) {
410 auto *slice = make<InputSection>(
411 args&: sec->file, args&: sec->name, args&: sec->type, args&: sec->flags, args&: sec->entsize,
412 args&: sec->entsize,
413 args: sec->contentMaybeDecompress().slice(N: begin, M: end - begin));
414 for (const Relocation &r : ArrayRef<Relocation>(rbegin, rend)) {
415 slice->relocations.push_back(
416 Elt: Relocation{.expr: r.expr, .type: r.type, .offset: r.offset - begin, .addend: r.addend, .sym: r.sym});
417 }
418 replacements.push_back(Elt: slice);
419 };
420
421 // Walk the jump table entries other than the last one looking for
422 // sections that are small enough to be moved into the jump table and in
423 // the same section as the jump table's destination.
424 size_t begin = 0, cur = 0;
425 Relocation *rbegin = sec->relocs().begin(), *rcur = rbegin;
426 while (cur != sec->size - sec->entsize) {
427 size_t next = cur + sec->entsize;
428 Relocation *rnext = rcur;
429 while (rnext != sec->relocs().end() && rnext->offset < next)
430 ++rnext;
431 if (rcur + 1 == rnext) {
432 if (InputSection *target = getMovableSection(*rcur);
433 target && target->size != 0 && target->size <= sec->entsize &&
434 target->addralign <= sec->entsize &&
435 target->getParent() == targetOutputSec) {
436 // Okay, we found a small enough section. Move it into the jump
437 // table. First add a slice for the unmodified jump table entries
438 // before this one. This slice may be of zero size if two
439 // consecutive functions are moved to the jump table, and is
440 // used to correctly align the target function.
441 addSectionSlice(begin, cur, rbegin, rcur);
442 // Add the target to our replacement list, and set the target's
443 // replacement list to the empty list. This removes it from its
444 // original position and adds it here, as well as causing
445 // future getMovableSection() queries to return nullptr.
446 replacements.push_back(Elt: target);
447 sectionReplacements[target] = {};
448 begin = next;
449 rbegin = rnext;
450 }
451 }
452 cur = next;
453 rcur = rnext;
454 }
455
456 // Finally, process the last entry. If it is movable, move the entire
457 // jump table behind it and delete the last entry (so that the last
458 // function's body acts as the last jump table entry), otherwise leave the
459 // jump table where it is and keep the last entry.
460 if (lastSec) {
461 addSectionSlice(begin, cur, rbegin, rcur);
462 replacements.push_back(Elt: lastSec);
463 sectionReplacements[sec] = {};
464 for (auto *s : replacements)
465 s->parent = lastSec->parent;
466 sectionReplacements[lastSec] = std::move(replacements);
467 } else {
468 addSectionSlice(begin, sec->size, rbegin, sec->relocs().end());
469 for (auto *s : replacements)
470 s->parent = sec->parent;
471 sectionReplacements[sec] = std::move(replacements);
472 }
473
474 // Everything from the original section has been recreated, so delete the
475 // original contents.
476 sec->relocations.clear();
477 sec->size = 0;
478 }
479 }
480
481 if (sectionReplacements.empty())
482 return;
483
484 // Now that we have the complete mapping of replacements, go through the input
485 // section lists and apply the replacements.
486 for (OutputSection *osec : ctx.outputSections) {
487 if (!(osec->flags & SHF_EXECINSTR))
488 continue;
489 for (SectionCommand *cmd : osec->commands) {
490 auto *isd = dyn_cast<InputSectionDescription>(Val: cmd);
491 if (!isd)
492 continue;
493 SmallVector<InputSection *, 0> newSections;
494 for (auto *sec : isd->sections) {
495 auto i = sectionReplacements.find(Val: sec);
496 if (i == sectionReplacements.end())
497 newSections.push_back(Elt: sec);
498 else
499 newSections.append(in_start: i->second.begin(), in_end: i->second.end());
500 }
501 isd->sections = std::move(newSections);
502 }
503 }
504}
505
506bool X86_64::relaxOnce(int pass) const {
507 uint64_t minVA = UINT64_MAX, maxVA = 0;
508 for (OutputSection *osec : ctx.outputSections) {
509 if (!(osec->flags & SHF_ALLOC))
510 continue;
511 minVA = std::min(a: minVA, b: osec->addr);
512 maxVA = std::max(a: maxVA, b: osec->addr + osec->size);
513 }
514 // If the max VA is under 2^31, GOTPCRELX relocations cannot overflow. In
515 // -pie/-shared, the condition can be relaxed to test the max VA difference as
516 // there is no R_RELAX_GOT_PC_NOPIC.
517 if (isUInt<31>(x: maxVA) || (isUInt<31>(x: maxVA - minVA) && ctx.arg.isPic))
518 return false;
519
520 SmallVector<InputSection *, 0> storage;
521 bool changed = false;
522 for (OutputSection *osec : ctx.outputSections) {
523 if (!(osec->flags & SHF_EXECINSTR))
524 continue;
525 for (InputSection *sec : getInputSections(os: *osec, storage)) {
526 for (Relocation &rel : sec->relocs()) {
527 if (rel.expr != R_RELAX_GOT_PC && rel.expr != R_RELAX_GOT_PC_NOPIC)
528 continue;
529 assert(rel.addend == -4);
530
531 Relocation rel1 = rel;
532 rel1.addend = rel.expr == R_RELAX_GOT_PC_NOPIC ? 0 : -4;
533 uint64_t v = sec->getRelocTargetVA(ctx, r: rel1,
534 p: sec->getOutputSection()->addr +
535 sec->outSecOff + rel.offset);
536 if (isInt<32>(x: v))
537 continue;
538 if (rel.sym->auxIdx == 0) {
539 rel.sym->allocateAux(ctx);
540 addGotEntry(ctx, sym&: *rel.sym);
541 changed = true;
542 }
543 rel.expr = R_GOT_PC;
544 }
545 }
546 }
547 return changed;
548}
549
550void X86_64::initTargetSpecificSections() {
551 if (ctx.arg.andFeatures & GNU_PROPERTY_X86_FEATURE_1_IBT) {
552 ctx.in.ibtPlt = std::make_unique<IBTPltSection>(args&: ctx);
553 ctx.inputSections.push_back(Elt: ctx.in.ibtPlt.get());
554 }
555}
556
557// Only needed to support relocations used by relocateNonAlloc and relocateEh.
558RelExpr X86_64::getRelExpr(RelType type, const Symbol &s,
559 const uint8_t *loc) const {
560 switch (type) {
561 case R_X86_64_8:
562 case R_X86_64_16:
563 case R_X86_64_32:
564 case R_X86_64_32S:
565 case R_X86_64_64:
566 return R_ABS;
567 case R_X86_64_SIZE32:
568 case R_X86_64_SIZE64:
569 return R_SIZE;
570 case R_X86_64_DTPOFF32:
571 case R_X86_64_DTPOFF64:
572 return R_DTPREL;
573 case R_X86_64_PC8:
574 case R_X86_64_PC16:
575 case R_X86_64_PC32:
576 case R_X86_64_PC64:
577 return R_PC;
578 case R_X86_64_GOTOFF64:
579 return R_GOTPLTREL;
580 case R_X86_64_GOTPC32:
581 case R_X86_64_GOTPC64:
582 return R_GOTPLTONLY_PC;
583 case R_X86_64_NONE:
584 return R_NONE;
585 default:
586 Err(ctx) << getErrorLoc(ctx, loc) << "unknown relocation (" << type.v
587 << ") against symbol " << &s;
588 return R_NONE;
589 }
590}
591
592void X86_64::writeGotPltHeader(uint8_t *buf) const {
593 // The first entry holds the link-time address of _DYNAMIC. It is documented
594 // in the psABI and glibc before Aug 2021 used the entry to compute run-time
595 // load address of the shared object (note that this is relevant for linking
596 // ld.so, not any other program).
597 write64le(P: buf, V: ctx.in.dynamic->getVA());
598}
599
600void X86_64::writeGotPlt(uint8_t *buf, const Symbol &s) const {
601 // See comments in X86::writeGotPlt.
602 write64le(P: buf, V: s.getPltVA(ctx) + 6);
603}
604
605void X86_64::writeIgotPlt(uint8_t *buf, const Symbol &s) const {
606 // An x86 entry is the address of the ifunc resolver function (for -z rel).
607 if (ctx.arg.writeAddends)
608 write64le(P: buf, V: s.getVA(ctx));
609}
610
611void X86_64::writePltHeader(uint8_t *buf) const {
612 const uint8_t pltData[] = {
613 0xff, 0x35, 0, 0, 0, 0, // pushq GOTPLT+8(%rip)
614 0xff, 0x25, 0, 0, 0, 0, // jmp *GOTPLT+16(%rip)
615 0x0f, 0x1f, 0x40, 0x00, // nop
616 };
617 memcpy(dest: buf, src: pltData, n: sizeof(pltData));
618 uint64_t gotPlt = ctx.in.gotPlt->getVA();
619 uint64_t plt = ctx.in.ibtPlt ? ctx.in.ibtPlt->getVA() : ctx.in.plt->getVA();
620 write32le(P: buf + 2, V: gotPlt - plt + 2); // GOTPLT+8
621 write32le(P: buf + 8, V: gotPlt - plt + 4); // GOTPLT+16
622}
623
624void X86_64::writePlt(uint8_t *buf, const Symbol &sym,
625 uint64_t pltEntryAddr) const {
626 const uint8_t inst[] = {
627 0xff, 0x25, 0, 0, 0, 0, // jmpq *got(%rip)
628 0x68, 0, 0, 0, 0, // pushq <relocation index>
629 0xe9, 0, 0, 0, 0, // jmpq plt[0]
630 };
631 memcpy(dest: buf, src: inst, n: sizeof(inst));
632
633 write32le(P: buf + 2, V: sym.getGotPltVA(ctx) - pltEntryAddr - 6);
634 write32le(P: buf + 7, V: sym.getPltIdx(ctx));
635 write32le(P: buf + 12, V: ctx.in.plt->getVA() - pltEntryAddr - 16);
636}
637
638RelType X86_64::getDynRel(RelType type) const {
639 if (type == symbolicRel || type == R_X86_64_SIZE32 || type == R_X86_64_SIZE64)
640 return type;
641 return R_X86_64_NONE;
642}
643
644template <class ELFT, class RelTy>
645void X86_64::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
646 RelocScan rs(ctx, &sec);
647 sec.relocations.reserve(N: rels.size());
648
649 for (auto it = rels.begin(); it != rels.end(); ++it) {
650 const RelTy &rel = *it;
651 uint32_t symIdx = rel.getSymbol(false);
652 Symbol &sym = sec.getFile<ELFT>()->getSymbol(symIdx);
653 uint64_t offset = rel.r_offset;
654 RelType type = rel.getType(false);
655 if (sym.isUndefined() && symIdx != 0 &&
656 rs.maybeReportUndefined(sym&: cast<Undefined>(Val&: sym), offset))
657 continue;
658 int64_t addend = rs.getAddend<ELFT>(rel, type);
659 RelExpr expr;
660 // Relocation types that only need a RelExpr set `expr` and break out of
661 // the switch to reach rs.process(). Types that need special handling
662 // (fast-path helpers, TLS) call a handler and use `continue`.
663 switch (type) {
664 case R_X86_64_NONE:
665 continue;
666
667 // Absolute relocations:
668 case R_X86_64_8:
669 case R_X86_64_16:
670 case R_X86_64_32:
671 case R_X86_64_32S:
672 case R_X86_64_64:
673 expr = R_ABS;
674 break;
675
676 // PC-relative relocations:
677 case R_X86_64_PC8:
678 case R_X86_64_PC16:
679 case R_X86_64_PC32:
680 case R_X86_64_PC64:
681 rs.processR_PC(type, offset, addend, sym);
682 continue;
683
684 // GOT-generating relocations:
685 case R_X86_64_GOTPC32:
686 case R_X86_64_GOTPC64:
687 ctx.in.gotPlt->hasGotPltOffRel.store(i: true, m: std::memory_order_relaxed);
688 expr = R_GOTPLTONLY_PC;
689 break;
690 case R_X86_64_GOTOFF64:
691 ctx.in.gotPlt->hasGotPltOffRel.store(i: true, m: std::memory_order_relaxed);
692 expr = R_GOTPLTREL;
693 break;
694 case R_X86_64_GOT32:
695 case R_X86_64_GOT64:
696 ctx.in.gotPlt->hasGotPltOffRel.store(i: true, m: std::memory_order_relaxed);
697 expr = R_GOTPLT;
698 break;
699 case R_X86_64_PLTOFF64:
700 ctx.in.gotPlt->hasGotPltOffRel.store(i: true, m: std::memory_order_relaxed);
701 expr = R_PLT_GOTPLT;
702 break;
703 case R_X86_64_GOTPCREL:
704 case R_X86_64_GOTPCRELX:
705 case R_X86_64_REX_GOTPCRELX:
706 case R_X86_64_CODE_4_GOTPCRELX:
707 expr = R_GOT_PC;
708 break;
709
710 // PLT-generating relocation:
711 case R_X86_64_PLT32:
712 rs.processR_PLT_PC(type, offset, addend, sym);
713 continue;
714
715 // TLS relocations:
716 case R_X86_64_TPOFF32:
717 case R_X86_64_TPOFF64:
718 if (rs.checkTlsLe(offset, sym, type))
719 continue;
720 expr = R_TPREL;
721 break;
722 case R_X86_64_GOTTPOFF:
723 case R_X86_64_CODE_4_GOTTPOFF:
724 case R_X86_64_CODE_6_GOTTPOFF:
725 rs.handleTlsIe(ieExpr: R_GOT_PC, type, offset, addend, sym);
726 continue;
727 case R_X86_64_TLSGD:
728 if (rs.handleTlsGd(sharedExpr: R_TLSGD_PC, ieExpr: R_GOT_PC, leExpr: R_TPREL, type, offset, addend,
729 sym))
730 ++it;
731 continue;
732 case R_X86_64_TLSLD:
733 if (rs.handleTlsLd(sharedExpr: R_TLSLD_PC, type, offset, addend, sym))
734 ++it;
735 continue;
736 case R_X86_64_DTPOFF32:
737 case R_X86_64_DTPOFF64:
738 sec.addReloc(
739 r: {.expr: ctx.arg.shared ? R_DTPREL : R_TPREL, .type: type, .offset: offset, .addend: addend, .sym: &sym});
740 continue;
741 case R_X86_64_TLSDESC_CALL:
742 // For executables, TLSDESC is optimized to IE or LE. Use R_TPREL as the
743 // rewrites for this relocation are identical.
744 if (!ctx.arg.shared)
745 sec.addReloc(r: {.expr: R_TPREL, .type: type, .offset: offset, .addend: addend, .sym: &sym});
746 continue;
747 case R_X86_64_GOTPC32_TLSDESC:
748 case R_X86_64_CODE_4_GOTPC32_TLSDESC:
749 rs.handleTlsDesc(sharedExpr: R_TLSDESC_PC, ieExpr: R_GOT_PC, type, offset, addend, sym);
750 continue;
751
752 // Misc relocations:
753 case R_X86_64_SIZE32:
754 case R_X86_64_SIZE64:
755 expr = R_SIZE;
756 break;
757
758 default:
759 Err(ctx) << getErrorLoc(ctx, loc: sec.content().data() + offset)
760 << "unknown relocation (" << type.v << ") against symbol "
761 << &sym;
762 continue;
763 }
764 rs.process(expr, type, offset, sym, addend);
765 }
766
767 if (ctx.arg.branchToBranch)
768 llvm::stable_sort(sec.relocs(),
769 [](auto &l, auto &r) { return l.offset < r.offset; });
770}
771
772void X86_64::scanSection(InputSectionBase &sec) {
773 if (ctx.arg.is64)
774 elf::scanSection1<X86_64, ELF64LE>(target&: *this, sec);
775 else // ilp32
776 elf::scanSection1<X86_64, ELF32LE>(target&: *this, sec);
777}
778
779void X86_64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
780 uint64_t val) const {
781 if (rel.type == R_X86_64_TLSGD) {
782 // Convert
783 // .byte 0x66
784 // leaq x@tlsgd(%rip), %rdi
785 // .word 0x6666
786 // rex64
787 // call __tls_get_addr@plt
788 // to the following two instructions.
789 const uint8_t inst[] = {
790 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00,
791 0x00, 0x00, // mov %fs:0x0,%rax
792 0x48, 0x8d, 0x80, 0, 0, 0, 0, // lea x@tpoff,%rax
793 };
794 memcpy(dest: loc - 4, src: inst, n: sizeof(inst));
795
796 // The original code used a pc relative relocation and so we have to
797 // compensate for the -4 in had in the addend.
798 write32le(P: loc + 8, V: val + 4);
799 } else if (rel.type == R_X86_64_GOTPC32_TLSDESC ||
800 rel.type == R_X86_64_CODE_4_GOTPC32_TLSDESC) {
801 // Convert leaq x@tlsdesc(%rip), %REG to movq $x@tpoff, %REG.
802 if ((loc[-3] & 0xfb) != 0x48 || loc[-2] != 0x8d ||
803 (loc[-1] & 0xc7) != 0x05) {
804 Err(ctx) << getErrorLoc(ctx, loc: (rel.type == R_X86_64_GOTPC32_TLSDESC)
805 ? loc - 3
806 : loc - 4)
807 << "R_X86_64_GOTPC32_TLSDESC/R_X86_64_CODE_4_GOTPC32_TLSDESC "
808 "must be used in leaq x@tlsdesc(%rip), %REG";
809 return;
810 }
811 if (rel.type == R_X86_64_GOTPC32_TLSDESC) {
812 loc[-3] = 0x48 | ((loc[-3] >> 2) & 1);
813 } else {
814 loc[-3] = (loc[-3] & ~0x44) | ((loc[-3] & 0x44) >> 2);
815 }
816 loc[-2] = 0xc7;
817 loc[-1] = 0xc0 | ((loc[-1] >> 3) & 7);
818
819 write32le(P: loc, V: val + 4);
820 } else {
821 // Convert call *x@tlsdesc(%REG) to xchg ax, ax.
822 assert(rel.type == R_X86_64_TLSDESC_CALL);
823 loc[0] = 0x66;
824 loc[1] = 0x90;
825 }
826}
827
828void X86_64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
829 uint64_t val) const {
830 if (rel.type == R_X86_64_TLSGD) {
831 // Convert
832 // .byte 0x66
833 // leaq x@tlsgd(%rip), %rdi
834 // .word 0x6666
835 // rex64
836 // call __tls_get_addr@plt
837 // to the following two instructions.
838 const uint8_t inst[] = {
839 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00,
840 0x00, 0x00, // mov %fs:0x0,%rax
841 0x48, 0x03, 0x05, 0, 0, 0, 0, // addq x@gottpoff(%rip),%rax
842 };
843 memcpy(dest: loc - 4, src: inst, n: sizeof(inst));
844
845 // Both code sequences are PC relatives, but since we are moving the
846 // constant forward by 8 bytes we have to subtract the value by 8.
847 write32le(P: loc + 8, V: val - 8);
848 } else if (rel.type == R_X86_64_GOTPC32_TLSDESC ||
849 rel.type == R_X86_64_CODE_4_GOTPC32_TLSDESC) {
850 // Convert leaq x@tlsdesc(%rip), %REG to movq x@gottpoff(%rip), %REG.
851 if ((loc[-3] & 0xfb) != 0x48 || loc[-2] != 0x8d ||
852 (loc[-1] & 0xc7) != 0x05) {
853 Err(ctx) << getErrorLoc(ctx, loc: (rel.type == R_X86_64_GOTPC32_TLSDESC)
854 ? loc - 3
855 : loc - 4)
856 << "R_X86_64_GOTPC32_TLSDESC/R_X86_64_CODE_4_GOTPC32_TLSDESC "
857 "must be used in leaq x@tlsdesc(%rip), %REG";
858 return;
859 }
860 loc[-2] = 0x8b;
861 write32le(P: loc, V: val);
862 }
863}
864
865// In some conditions,
866// R_X86_64_GOTTPOFF/R_X86_64_CODE_4_GOTTPOFF/R_X86_64_CODE_6_GOTTPOFF
867// relocation can be optimized to R_X86_64_TPOFF32 so that it does not use GOT.
868void X86_64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
869 uint64_t val) const {
870 uint8_t *inst = loc - 3;
871 uint8_t reg = loc[-1] >> 3;
872 uint8_t *regSlot = loc - 1;
873
874 if (rel.type == R_X86_64_GOTTPOFF) {
875 // Note that ADD with RSP or R12 is converted to ADD instead of LEA
876 // because LEA with these registers needs 4 bytes to encode and thus
877 // wouldn't fit the space.
878
879 if (memcmp(s1: inst, s2: "\x48\x03\x25", n: 3) == 0) {
880 // "addq foo@gottpoff(%rip),%rsp" -> "addq $foo,%rsp"
881 memcpy(dest: inst, src: "\x48\x81\xc4", n: 3);
882 } else if (memcmp(s1: inst, s2: "\x4c\x03\x25", n: 3) == 0) {
883 // "addq foo@gottpoff(%rip),%r12" -> "addq $foo,%r12"
884 memcpy(dest: inst, src: "\x49\x81\xc4", n: 3);
885 } else if (memcmp(s1: inst, s2: "\x4c\x03", n: 2) == 0) {
886 // "addq foo@gottpoff(%rip),%r[8-15]" -> "leaq foo(%r[8-15]),%r[8-15]"
887 memcpy(dest: inst, src: "\x4d\x8d", n: 2);
888 *regSlot = 0x80 | (reg << 3) | reg;
889 } else if (memcmp(s1: inst, s2: "\x48\x03", n: 2) == 0) {
890 // "addq foo@gottpoff(%rip),%reg -> "leaq foo(%reg),%reg"
891 memcpy(dest: inst, src: "\x48\x8d", n: 2);
892 *regSlot = 0x80 | (reg << 3) | reg;
893 } else if (memcmp(s1: inst, s2: "\x4c\x8b", n: 2) == 0) {
894 // "movq foo@gottpoff(%rip),%r[8-15]" -> "movq $foo,%r[8-15]"
895 memcpy(dest: inst, src: "\x49\xc7", n: 2);
896 *regSlot = 0xc0 | reg;
897 } else if (memcmp(s1: inst, s2: "\x48\x8b", n: 2) == 0) {
898 // "movq foo@gottpoff(%rip),%reg" -> "movq $foo,%reg"
899 memcpy(dest: inst, src: "\x48\xc7", n: 2);
900 *regSlot = 0xc0 | reg;
901 } else {
902 Err(ctx)
903 << getErrorLoc(ctx, loc: loc - 3)
904 << "R_X86_64_GOTTPOFF must be used in MOVQ or ADDQ instructions only";
905 }
906 } else if (rel.type == R_X86_64_CODE_4_GOTTPOFF) {
907 if (loc[-4] != 0xd5) {
908 Err(ctx) << getErrorLoc(ctx, loc: loc - 4)
909 << "invalid prefix with R_X86_64_CODE_4_GOTTPOFF!";
910 return;
911 }
912 const uint8_t rex = loc[-3];
913 loc[-3] = (rex & ~0x44) | (rex & 0x44) >> 2;
914 *regSlot = 0xc0 | reg;
915
916 if (loc[-2] == 0x8b) {
917 // "movq foo@gottpoff(%rip),%r[16-31]" -> "movq $foo,%r[16-31]"
918 loc[-2] = 0xc7;
919 } else if (loc[-2] == 0x03) {
920 // "addq foo@gottpoff(%rip),%r[16-31]" -> "addq $foo,%r[16-31]"
921 loc[-2] = 0x81;
922 } else {
923 Err(ctx) << getErrorLoc(ctx, loc: loc - 4)
924 << "R_X86_64_CODE_4_GOTTPOFF must be used in MOVQ or ADDQ "
925 "instructions only";
926 }
927 } else if (rel.type == R_X86_64_CODE_6_GOTTPOFF) {
928 if (loc[-6] != 0x62) {
929 Err(ctx) << getErrorLoc(ctx, loc: loc - 6)
930 << "invalid prefix with R_X86_64_CODE_6_GOTTPOFF!";
931 return;
932 }
933 // Check bits are satisfied:
934 // loc[-5]: X==1 (inverted polarity), (loc[-5] & 0x7) == 0x4
935 // loc[-4]: W==1, X2==1 (inverted polarity), pp==0b00(NP)
936 // loc[-3]: NF==1 or ND==1
937 // loc[-2]: opcode==0x1 or opcode==0x3
938 // loc[-1]: Mod==0b00, RM==0b101
939 if (((loc[-5] & 0x47) == 0x44) && ((loc[-4] & 0x87) == 0x84) &&
940 ((loc[-3] & 0x14) != 0) && (loc[-2] == 0x1 || loc[-2] == 0x3) &&
941 ((loc[-1] & 0xc7) == 0x5)) {
942 // "addq %reg1, foo@GOTTPOFF(%rip), %reg2" -> "addq $foo, %reg1, %reg2"
943 // "addq foo@GOTTPOFF(%rip), %reg1, %reg2" -> "addq $foo, %reg1, %reg2"
944 // "{nf} addq %reg1, foo@GOTTPOFF(%rip), %reg2"
945 // -> "{nf} addq $foo, %reg1, %reg2"
946 // "{nf} addq name@GOTTPOFF(%rip), %reg1, %reg2"
947 // -> "{nf} addq $foo, %reg1, %reg2"
948 // "{nf} addq name@GOTTPOFF(%rip), %reg" -> "{nf} addq $foo, %reg"
949 loc[-2] = 0x81;
950 // Move R bits to B bits in EVEX payloads and ModRM byte.
951 const uint8_t evexPayload0 = loc[-5];
952 if ((evexPayload0 & (1 << 7)) == 0)
953 loc[-5] = (evexPayload0 | (1 << 7)) & ~(1 << 5);
954 if ((evexPayload0 & (1 << 4)) == 0)
955 loc[-5] = evexPayload0 | (1 << 4) | (1 << 3);
956 *regSlot = 0xc0 | reg;
957 } else {
958 Err(ctx) << getErrorLoc(ctx, loc: loc - 6)
959 << "R_X86_64_CODE_6_GOTTPOFF must be used in ADDQ instructions "
960 "with NDD/NF/NDD+NF only";
961 }
962 } else {
963 llvm_unreachable("Unsupported relocation type!");
964 }
965
966 // The original code used a PC relative relocation.
967 // Need to compensate for the -4 it had in the addend.
968 write32le(P: loc, V: val + 4);
969}
970
971void X86_64::relaxTlsLdToLe(uint8_t *loc, const Relocation &rel,
972 uint64_t val) const {
973 const uint8_t inst[] = {
974 0x66, 0x66, // .word 0x6666
975 0x66, // .byte 0x66
976 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0,%rax
977 };
978
979 if (loc[4] == 0xe8) {
980 // Convert
981 // leaq bar@tlsld(%rip), %rdi # 48 8d 3d <Loc>
982 // callq __tls_get_addr@PLT # e8 <disp32>
983 // leaq bar@dtpoff(%rax), %rcx
984 // to
985 // .word 0x6666
986 // .byte 0x66
987 // mov %fs:0,%rax
988 // leaq bar@tpoff(%rax), %rcx
989 memcpy(dest: loc - 3, src: inst, n: sizeof(inst));
990 return;
991 }
992
993 if (loc[4] == 0xff && loc[5] == 0x15) {
994 // Convert
995 // leaq x@tlsld(%rip),%rdi # 48 8d 3d <Loc>
996 // call *__tls_get_addr@GOTPCREL(%rip) # ff 15 <disp32>
997 // to
998 // .long 0x66666666
999 // movq %fs:0,%rax
1000 // See "Table 11.9: LD -> LE Code Transition (LP64)" in
1001 // https://raw.githubusercontent.com/wiki/hjl-tools/x86-psABI/x86-64-psABI-1.0.pdf
1002 loc[-3] = 0x66;
1003 memcpy(dest: loc - 2, src: inst, n: sizeof(inst));
1004 return;
1005 }
1006
1007 ErrAlways(ctx)
1008 << getErrorLoc(ctx, loc: loc - 3)
1009 << "expected R_X86_64_PLT32 or R_X86_64_GOTPCRELX after R_X86_64_TLSLD";
1010}
1011
1012// A JumpInstrMod at a specific offset indicates that the jump instruction
1013// opcode at that offset must be modified. This is specifically used to relax
1014// jump instructions with basic block sections. This function looks at the
1015// JumpMod and effects the change.
1016void X86_64::applyJumpInstrMod(uint8_t *loc, JumpModType type,
1017 unsigned size) const {
1018 switch (type) {
1019 case J_JMP_32:
1020 if (size == 4)
1021 *loc = 0xe9;
1022 else
1023 *loc = 0xeb;
1024 break;
1025 case J_JE_32:
1026 if (size == 4) {
1027 loc[-1] = 0x0f;
1028 *loc = 0x84;
1029 } else
1030 *loc = 0x74;
1031 break;
1032 case J_JNE_32:
1033 if (size == 4) {
1034 loc[-1] = 0x0f;
1035 *loc = 0x85;
1036 } else
1037 *loc = 0x75;
1038 break;
1039 case J_JG_32:
1040 if (size == 4) {
1041 loc[-1] = 0x0f;
1042 *loc = 0x8f;
1043 } else
1044 *loc = 0x7f;
1045 break;
1046 case J_JGE_32:
1047 if (size == 4) {
1048 loc[-1] = 0x0f;
1049 *loc = 0x8d;
1050 } else
1051 *loc = 0x7d;
1052 break;
1053 case J_JB_32:
1054 if (size == 4) {
1055 loc[-1] = 0x0f;
1056 *loc = 0x82;
1057 } else
1058 *loc = 0x72;
1059 break;
1060 case J_JBE_32:
1061 if (size == 4) {
1062 loc[-1] = 0x0f;
1063 *loc = 0x86;
1064 } else
1065 *loc = 0x76;
1066 break;
1067 case J_JL_32:
1068 if (size == 4) {
1069 loc[-1] = 0x0f;
1070 *loc = 0x8c;
1071 } else
1072 *loc = 0x7c;
1073 break;
1074 case J_JLE_32:
1075 if (size == 4) {
1076 loc[-1] = 0x0f;
1077 *loc = 0x8e;
1078 } else
1079 *loc = 0x7e;
1080 break;
1081 case J_JA_32:
1082 if (size == 4) {
1083 loc[-1] = 0x0f;
1084 *loc = 0x87;
1085 } else
1086 *loc = 0x77;
1087 break;
1088 case J_JAE_32:
1089 if (size == 4) {
1090 loc[-1] = 0x0f;
1091 *loc = 0x83;
1092 } else
1093 *loc = 0x73;
1094 break;
1095 case J_UNKNOWN:
1096 llvm_unreachable("Unknown Jump Relocation");
1097 }
1098}
1099
1100int64_t X86_64::getImplicitAddend(const uint8_t *buf, RelType type) const {
1101 switch (type) {
1102 case R_X86_64_8:
1103 case R_X86_64_PC8:
1104 return SignExtend64<8>(x: *buf);
1105 case R_X86_64_16:
1106 case R_X86_64_PC16:
1107 return SignExtend64<16>(x: read16le(P: buf));
1108 case R_X86_64_32:
1109 case R_X86_64_32S:
1110 case R_X86_64_TPOFF32:
1111 case R_X86_64_GOT32:
1112 case R_X86_64_GOTPC32:
1113 case R_X86_64_GOTPC32_TLSDESC:
1114 case R_X86_64_GOTPCREL:
1115 case R_X86_64_GOTPCRELX:
1116 case R_X86_64_REX_GOTPCRELX:
1117 case R_X86_64_CODE_4_GOTPCRELX:
1118 case R_X86_64_PC32:
1119 case R_X86_64_GOTTPOFF:
1120 case R_X86_64_CODE_4_GOTTPOFF:
1121 case R_X86_64_CODE_6_GOTTPOFF:
1122 case R_X86_64_PLT32:
1123 case R_X86_64_TLSGD:
1124 case R_X86_64_TLSLD:
1125 case R_X86_64_DTPOFF32:
1126 case R_X86_64_SIZE32:
1127 return SignExtend64<32>(x: read32le(P: buf));
1128 case R_X86_64_64:
1129 case R_X86_64_TPOFF64:
1130 case R_X86_64_DTPOFF64:
1131 case R_X86_64_DTPMOD64:
1132 case R_X86_64_PC64:
1133 case R_X86_64_SIZE64:
1134 case R_X86_64_GLOB_DAT:
1135 case R_X86_64_GOT64:
1136 case R_X86_64_GOTOFF64:
1137 case R_X86_64_GOTPC64:
1138 case R_X86_64_PLTOFF64:
1139 case R_X86_64_IRELATIVE:
1140 case R_X86_64_RELATIVE:
1141 return read64le(P: buf);
1142 case R_X86_64_TLSDESC:
1143 return read64le(P: buf + 8);
1144 case R_X86_64_JUMP_SLOT:
1145 case R_X86_64_NONE:
1146 // These relocations are defined as not having an implicit addend.
1147 return 0;
1148 default:
1149 InternalErr(ctx, buf) << "cannot read addend for relocation " << type;
1150 return 0;
1151 }
1152}
1153
1154static void relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val);
1155
1156void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
1157 switch (rel.type) {
1158 case R_X86_64_8:
1159 checkIntUInt(ctx, loc, v: val, n: 8, rel);
1160 *loc = val;
1161 break;
1162 case R_X86_64_PC8:
1163 checkInt(ctx, loc, v: val, n: 8, rel);
1164 *loc = val;
1165 break;
1166 case R_X86_64_16:
1167 checkIntUInt(ctx, loc, v: val, n: 16, rel);
1168 write16le(P: loc, V: val);
1169 break;
1170 case R_X86_64_PC16:
1171 checkInt(ctx, loc, v: val, n: 16, rel);
1172 write16le(P: loc, V: val);
1173 break;
1174 case R_X86_64_32:
1175 checkUInt(ctx, loc, v: val, n: 32, rel);
1176 write32le(P: loc, V: val);
1177 break;
1178 case R_X86_64_32S:
1179 case R_X86_64_GOT32:
1180 case R_X86_64_GOTPC32:
1181 case R_X86_64_GOTPCREL:
1182 case R_X86_64_PC32:
1183 case R_X86_64_PLT32:
1184 case R_X86_64_DTPOFF32:
1185 case R_X86_64_SIZE32:
1186 checkInt(ctx, loc, v: val, n: 32, rel);
1187 write32le(P: loc, V: val);
1188 break;
1189 case R_X86_64_64:
1190 case R_X86_64_TPOFF64:
1191 case R_X86_64_DTPOFF64:
1192 case R_X86_64_PC64:
1193 case R_X86_64_SIZE64:
1194 case R_X86_64_GOT64:
1195 case R_X86_64_GOTOFF64:
1196 case R_X86_64_GOTPC64:
1197 case R_X86_64_PLTOFF64:
1198 write64le(P: loc, V: val);
1199 break;
1200 case R_X86_64_GOTPCRELX:
1201 case R_X86_64_REX_GOTPCRELX:
1202 case R_X86_64_CODE_4_GOTPCRELX:
1203 if (rel.expr != R_GOT_PC) {
1204 relaxGot(loc, rel, val);
1205 } else {
1206 checkInt(ctx, loc, v: val, n: 32, rel);
1207 write32le(P: loc, V: val);
1208 }
1209 break;
1210 case R_X86_64_GOTPC32_TLSDESC:
1211 case R_X86_64_CODE_4_GOTPC32_TLSDESC:
1212 case R_X86_64_TLSDESC_CALL:
1213 case R_X86_64_TLSGD:
1214 if (rel.expr == R_TPREL) {
1215 relaxTlsGdToLe(loc, rel, val);
1216 } else if (rel.expr == R_GOT_PC) {
1217 relaxTlsGdToIe(loc, rel, val);
1218 } else {
1219 checkInt(ctx, loc, v: val, n: 32, rel);
1220 write32le(P: loc, V: val);
1221 }
1222 break;
1223 case R_X86_64_TLSLD:
1224 if (rel.expr == R_TPREL) {
1225 relaxTlsLdToLe(loc, rel, val);
1226 } else {
1227 checkInt(ctx, loc, v: val, n: 32, rel);
1228 write32le(P: loc, V: val);
1229 }
1230 break;
1231 case R_X86_64_GOTTPOFF:
1232 case R_X86_64_CODE_4_GOTTPOFF:
1233 case R_X86_64_CODE_6_GOTTPOFF:
1234 if (rel.expr == R_TPREL) {
1235 relaxTlsIeToLe(loc, rel, val);
1236 } else {
1237 checkInt(ctx, loc, v: val, n: 32, rel);
1238 write32le(P: loc, V: val);
1239 }
1240 break;
1241 case R_X86_64_TPOFF32:
1242 checkInt(ctx, loc, v: val, n: 32, rel);
1243 write32le(P: loc, V: val);
1244 break;
1245
1246 case R_X86_64_TLSDESC:
1247 // The addend is stored in the second 64-bit word.
1248 write64le(P: loc + 8, V: val);
1249 break;
1250 default:
1251 llvm_unreachable("unknown relocation");
1252 }
1253}
1254
1255RelExpr X86_64::adjustGotPcExpr(RelType type, int64_t addend,
1256 const uint8_t *loc) const {
1257 // Only R_X86_64_[REX_]|[CODE_4_]GOTPCRELX can be relaxed. GNU as may emit
1258 // GOTPCRELX with addend != -4. Such an instruction does not load the full GOT
1259 // entry, so we cannot relax the relocation. E.g. movl x@GOTPCREL+4(%rip),
1260 // %rax (addend=0) loads the high 32 bits of the GOT entry.
1261 if (!ctx.arg.relax || addend != -4 ||
1262 (type != R_X86_64_GOTPCRELX && type != R_X86_64_REX_GOTPCRELX &&
1263 type != R_X86_64_CODE_4_GOTPCRELX))
1264 return R_GOT_PC;
1265 const uint8_t op = loc[-2];
1266 const uint8_t modRm = loc[-1];
1267
1268 // FIXME: When PIC is disabled and foo is defined locally in the
1269 // lower 32 bit address space, memory operand in mov can be converted into
1270 // immediate operand. Otherwise, mov must be changed to lea. We support only
1271 // latter relaxation at this moment.
1272 if (op == 0x8b)
1273 return R_RELAX_GOT_PC;
1274
1275 // Relax call and jmp.
1276 if (op == 0xff && (modRm == 0x15 || modRm == 0x25))
1277 return R_RELAX_GOT_PC;
1278
1279 // We don't support test/binop instructions without a REX/REX2 prefix.
1280 if (type == R_X86_64_GOTPCRELX)
1281 return R_GOT_PC;
1282
1283 // Relaxation of test, adc, add, and, cmp, or, sbb, sub, xor.
1284 // If PIC then no relaxation is available.
1285 return ctx.arg.isPic ? R_GOT_PC : R_RELAX_GOT_PC_NOPIC;
1286}
1287
1288// A subset of relaxations can only be applied for no-PIC. This method
1289// handles such relaxations. Instructions encoding information was taken from:
1290// "Intel 64 and IA-32 Architectures Software Developer's Manual V2"
1291// (http://www.intel.com/content/dam/www/public/us/en/documents/manuals/
1292// 64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf)
1293static void relaxGotNoPic(uint8_t *loc, uint64_t val, uint8_t op, uint8_t modRm,
1294 bool isRex2) {
1295 const uint8_t rex = loc[-3];
1296 // Convert "test %reg, foo@GOTPCREL(%rip)" to "test $foo, %reg".
1297 if (op == 0x85) {
1298 // See "TEST-Logical Compare" (4-428 Vol. 2B),
1299 // TEST r/m64, r64 uses "full" ModR / M byte (no opcode extension).
1300
1301 // ModR/M byte has form XX YYY ZZZ, where
1302 // YYY is MODRM.reg(register 2), ZZZ is MODRM.rm(register 1).
1303 // XX has different meanings:
1304 // 00: The operand's memory address is in reg1.
1305 // 01: The operand's memory address is reg1 + a byte-sized displacement.
1306 // 10: The operand's memory address is reg1 + a word-sized displacement.
1307 // 11: The operand is reg1 itself.
1308 // If an instruction requires only one operand, the unused reg2 field
1309 // holds extra opcode bits rather than a register code
1310 // 0xC0 == 11 000 000 binary.
1311 // 0x38 == 00 111 000 binary.
1312 // We transfer reg2 to reg1 here as operand.
1313 // See "2.1.3 ModR/M and SIB Bytes" (Vol. 2A 2-3).
1314 loc[-1] = 0xc0 | (modRm & 0x38) >> 3; // ModR/M byte.
1315
1316 // Change opcode from TEST r/m64, r64 to TEST r/m64, imm32
1317 // See "TEST-Logical Compare" (4-428 Vol. 2B).
1318 loc[-2] = 0xf7;
1319
1320 // Move R bit to the B bit in REX/REX2 byte.
1321 // REX byte is encoded as 0100WRXB, where
1322 // 0100 is 4bit fixed pattern.
1323 // REX.W When 1, a 64-bit operand size is used. Otherwise, when 0, the
1324 // default operand size is used (which is 32-bit for most but not all
1325 // instructions).
1326 // REX.R This 1-bit value is an extension to the MODRM.reg field.
1327 // REX.X This 1-bit value is an extension to the SIB.index field.
1328 // REX.B This 1-bit value is an extension to the MODRM.rm field or the
1329 // SIB.base field.
1330 // See "2.2.1.2 More on REX Prefix Fields " (2-8 Vol. 2A).
1331 //
1332 // REX2 prefix is encoded as 0xd5|M|R2|X2|B2|WRXB, where
1333 // 0xd5 is 1byte fixed pattern.
1334 // REX2's [W,R,X,B] have the same meanings as REX's.
1335 // REX2.M encodes the map id.
1336 // R2/X2/B2 provides the fifth and most siginicant bits of the R/X/B
1337 // register identifiers, each of which can now address all 32 GPRs.
1338 if (isRex2)
1339 loc[-3] = (rex & ~0x44) | (rex & 0x44) >> 2;
1340 else
1341 loc[-3] = (rex & ~0x4) | (rex & 0x4) >> 2;
1342 write32le(P: loc, V: val);
1343 return;
1344 }
1345
1346 // If we are here then we need to relax the adc, add, and, cmp, or, sbb, sub
1347 // or xor operations.
1348
1349 // Convert "binop foo@GOTPCREL(%rip), %reg" to "binop $foo, %reg".
1350 // Logic is close to one for test instruction above, but we also
1351 // write opcode extension here, see below for details.
1352 loc[-1] = 0xc0 | (modRm & 0x38) >> 3 | (op & 0x3c); // ModR/M byte.
1353
1354 // Primary opcode is 0x81, opcode extension is one of:
1355 // 000b = ADD, 001b is OR, 010b is ADC, 011b is SBB,
1356 // 100b is AND, 101b is SUB, 110b is XOR, 111b is CMP.
1357 // This value was wrote to MODRM.reg in a line above.
1358 // See "3.2 INSTRUCTIONS (A-M)" (Vol. 2A 3-15),
1359 // "INSTRUCTION SET REFERENCE, N-Z" (Vol. 2B 4-1) for
1360 // descriptions about each operation.
1361 loc[-2] = 0x81;
1362 if (isRex2)
1363 loc[-3] = (rex & ~0x44) | (rex & 0x44) >> 2;
1364 else
1365 loc[-3] = (rex & ~0x4) | (rex & 0x4) >> 2;
1366 write32le(P: loc, V: val);
1367}
1368
1369static void relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) {
1370 assert(isInt<32>(val) &&
1371 "GOTPCRELX should not have been relaxed if it overflows");
1372 const uint8_t op = loc[-2];
1373 const uint8_t modRm = loc[-1];
1374
1375 // Convert "mov foo@GOTPCREL(%rip),%reg" to "lea foo(%rip),%reg".
1376 if (op == 0x8b) {
1377 loc[-2] = 0x8d;
1378 write32le(P: loc, V: val);
1379 return;
1380 }
1381
1382 if (op != 0xff) {
1383 // We are relaxing a rip relative to an absolute, so compensate
1384 // for the old -4 addend.
1385 assert(!rel.sym->file->ctx.arg.isPic);
1386 relaxGotNoPic(loc, val: val + 4, op, modRm,
1387 isRex2: rel.type == R_X86_64_CODE_4_GOTPCRELX);
1388 return;
1389 }
1390
1391 // Convert call/jmp instructions.
1392 if (modRm == 0x15) {
1393 // ABI says we can convert "call *foo@GOTPCREL(%rip)" to "nop; call foo".
1394 // Instead we convert to "addr32 call foo" where addr32 is an instruction
1395 // prefix. That makes result expression to be a single instruction.
1396 loc[-2] = 0x67; // addr32 prefix
1397 loc[-1] = 0xe8; // call
1398 write32le(P: loc, V: val);
1399 return;
1400 }
1401
1402 // Convert "jmp *foo@GOTPCREL(%rip)" to "jmp foo; nop".
1403 // jmp doesn't return, so it is fine to use nop here, it is just a stub.
1404 assert(modRm == 0x25);
1405 loc[-2] = 0xe9; // jmp
1406 loc[3] = 0x90; // nop
1407 write32le(P: loc - 1, V: val + 1);
1408}
1409
1410// A split-stack prologue starts by checking the amount of stack remaining
1411// in one of two ways:
1412// A) Comparing of the stack pointer to a field in the tcb.
1413// B) Or a load of a stack pointer offset with an lea to r10 or r11.
1414bool X86_64::adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end,
1415 uint8_t stOther) const {
1416 if (!ctx.arg.is64) {
1417 ErrAlways(ctx) << "target doesn't support split stacks";
1418 return false;
1419 }
1420
1421 if (loc + 8 >= end)
1422 return false;
1423
1424 // Replace "cmp %fs:0x70,%rsp" and subsequent branch
1425 // with "stc, nopl 0x0(%rax,%rax,1)"
1426 if (memcmp(s1: loc, s2: "\x64\x48\x3b\x24\x25", n: 5) == 0) {
1427 memcpy(dest: loc, src: "\xf9\x0f\x1f\x84\x00\x00\x00\x00", n: 8);
1428 return true;
1429 }
1430
1431 // Adjust "lea X(%rsp),%rYY" to lea "(X - 0x4000)(%rsp),%rYY" where rYY could
1432 // be r10 or r11. The lea instruction feeds a subsequent compare which checks
1433 // if there is X available stack space. Making X larger effectively reserves
1434 // that much additional space. The stack grows downward so subtract the value.
1435 if (memcmp(s1: loc, s2: "\x4c\x8d\x94\x24", n: 4) == 0 ||
1436 memcmp(s1: loc, s2: "\x4c\x8d\x9c\x24", n: 4) == 0) {
1437 // The offset bytes are encoded four bytes after the start of the
1438 // instruction.
1439 write32le(P: loc + 4, V: read32le(P: loc + 4) - 0x4000);
1440 return true;
1441 }
1442 return false;
1443}
1444
1445void X86_64::relocateAlloc(InputSection &sec, uint8_t *buf) const {
1446 uint64_t secAddr = sec.getOutputSection()->addr + sec.outSecOff;
1447 for (const Relocation &rel : sec.relocs()) {
1448 if (rel.expr == R_NONE) // See deleteFallThruJmpInsn
1449 continue;
1450 uint8_t *loc = buf + rel.offset;
1451 const uint64_t val = sec.getRelocTargetVA(ctx, r: rel, p: secAddr + rel.offset);
1452 relocate(loc, rel, val);
1453 }
1454 if (sec.jumpInstrMod) {
1455 applyJumpInstrMod(loc: buf + sec.jumpInstrMod->offset,
1456 type: sec.jumpInstrMod->original, size: sec.jumpInstrMod->size);
1457 }
1458}
1459
1460static std::optional<uint64_t> getControlTransferAddend(InputSection &is,
1461 Relocation &r) {
1462 // Identify a control transfer relocation for the branch-to-branch
1463 // optimization. A "control transfer relocation" usually means a CALL or JMP
1464 // target but it also includes relative vtable relocations for example.
1465 //
1466 // We require the relocation type to be PLT32. With a relocation type of PLT32
1467 // the value may be assumed to be used for branching directly to the symbol
1468 // and the addend is only used to produce the relocated value (hence the
1469 // effective addend is always 0). This is because if a PLT is needed the
1470 // addend will be added to the address of the PLT, and it doesn't make sense
1471 // to branch into the middle of a PLT. For example, relative vtable
1472 // relocations use PLT32 and 0 or a positive value as the addend but still are
1473 // used to branch to the symbol.
1474 //
1475 // STT_SECTION symbols are a special case on x86 because the LLVM assembler
1476 // uses them for branches to local symbols which are assembled as referring to
1477 // the section symbol with the addend equal to the symbol value - 4.
1478 if (r.type == R_X86_64_PLT32) {
1479 if (r.sym->isSection())
1480 return r.addend + 4;
1481 return 0;
1482 }
1483 return std::nullopt;
1484}
1485
1486static std::pair<Relocation *, uint64_t>
1487getBranchInfoAtTarget(InputSection &is, uint64_t offset) {
1488 auto content = is.contentMaybeDecompress();
1489 if (content.size() > offset && content[offset] == 0xe9) { // JMP immediate
1490 auto *i = llvm::partition_point(
1491 Range&: is.relocations, P: [&](Relocation &r) { return r.offset < offset + 1; });
1492 // Unlike with getControlTransferAddend() it is valid to accept a PC32
1493 // relocation here because we know that this is actually a JMP and not some
1494 // other reference, so the interpretation is that we add 4 to the addend and
1495 // use that as the effective addend.
1496 if (i != is.relocations.end() && i->offset == offset + 1 &&
1497 (i->type == R_X86_64_PC32 || i->type == R_X86_64_PLT32)) {
1498 return {i, i->addend + 4};
1499 }
1500 }
1501 return {nullptr, 0};
1502}
1503
1504static void redirectControlTransferRelocations(Relocation &r1,
1505 const Relocation &r2) {
1506 // The isSection() check handles the STT_SECTION case described above.
1507 // In that case the original addend is irrelevant because it referred to an
1508 // offset within the original target section so we overwrite it.
1509 //
1510 // The +4 is here to compensate for r2.addend which will likely be -4,
1511 // but may also be addend-4 in case of a PC32 branch to symbol+addend.
1512 if (r1.sym->isSection())
1513 r1.addend = r2.addend;
1514 else
1515 r1.addend += r2.addend + 4;
1516 r1.expr = r2.expr;
1517 r1.sym = r2.sym;
1518}
1519
1520void X86_64::applyBranchToBranchOpt() const {
1521 applyBranchToBranchOptImpl(ctx, getControlTransferAddend,
1522 getBranchInfoAtTarget,
1523 redirectControlTransferRelocations);
1524}
1525
1526// If Intel Indirect Branch Tracking is enabled, we have to emit special PLT
1527// entries containing endbr64 instructions. A PLT entry will be split into two
1528// parts, one in .plt.sec (writePlt), and the other in .plt (writeIBTPlt).
1529namespace {
1530class IntelIBT : public X86_64 {
1531public:
1532 IntelIBT(Ctx &ctx) : X86_64(ctx) { pltHeaderSize = 0; };
1533 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
1534 void writePlt(uint8_t *buf, const Symbol &sym,
1535 uint64_t pltEntryAddr) const override;
1536 void writeIBTPlt(uint8_t *buf, size_t numEntries) const override;
1537
1538 static const unsigned IBTPltHeaderSize = 16;
1539};
1540} // namespace
1541
1542void IntelIBT::writeGotPlt(uint8_t *buf, const Symbol &s) const {
1543 uint64_t va = ctx.in.ibtPlt->getVA() + IBTPltHeaderSize +
1544 s.getPltIdx(ctx) * pltEntrySize;
1545 write64le(P: buf, V: va);
1546}
1547
1548void IntelIBT::writePlt(uint8_t *buf, const Symbol &sym,
1549 uint64_t pltEntryAddr) const {
1550 const uint8_t Inst[] = {
1551 0xf3, 0x0f, 0x1e, 0xfa, // endbr64
1552 0xff, 0x25, 0, 0, 0, 0, // jmpq *got(%rip)
1553 0x66, 0x0f, 0x1f, 0x44, 0, 0, // nop
1554 };
1555 memcpy(dest: buf, src: Inst, n: sizeof(Inst));
1556 write32le(P: buf + 6, V: sym.getGotPltVA(ctx) - pltEntryAddr - 10);
1557}
1558
1559void IntelIBT::writeIBTPlt(uint8_t *buf, size_t numEntries) const {
1560 writePltHeader(buf);
1561 buf += IBTPltHeaderSize;
1562
1563 const uint8_t inst[] = {
1564 0xf3, 0x0f, 0x1e, 0xfa, // endbr64
1565 0x68, 0, 0, 0, 0, // pushq <relocation index>
1566 0xe9, 0, 0, 0, 0, // jmpq plt[0]
1567 0x66, 0x90, // nop
1568 };
1569
1570 for (size_t i = 0; i < numEntries; ++i) {
1571 memcpy(dest: buf, src: inst, n: sizeof(inst));
1572 write32le(P: buf + 5, V: i);
1573 write32le(P: buf + 10, V: -pltHeaderSize - sizeof(inst) * i - 30);
1574 buf += sizeof(inst);
1575 }
1576}
1577
1578// These nonstandard PLT entries are to migtigate Spectre v2 security
1579// vulnerability. In order to mitigate Spectre v2, we want to avoid indirect
1580// branch instructions such as `jmp *GOTPLT(%rip)`. So, in the following PLT
1581// entries, we use a CALL followed by MOV and RET to do the same thing as an
1582// indirect jump. That instruction sequence is so-called "retpoline".
1583//
1584// We have two types of retpoline PLTs as a size optimization. If `-z now`
1585// is specified, all dynamic symbols are resolved at load-time. Thus, when
1586// that option is given, we can omit code for symbol lazy resolution.
1587namespace {
1588class Retpoline : public X86_64 {
1589public:
1590 Retpoline(Ctx &);
1591 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
1592 void writePltHeader(uint8_t *buf) const override;
1593 void writePlt(uint8_t *buf, const Symbol &sym,
1594 uint64_t pltEntryAddr) const override;
1595};
1596
1597class RetpolineZNow : public X86_64 {
1598public:
1599 RetpolineZNow(Ctx &);
1600 void writeGotPlt(uint8_t *buf, const Symbol &s) const override {}
1601 void writePltHeader(uint8_t *buf) const override;
1602 void writePlt(uint8_t *buf, const Symbol &sym,
1603 uint64_t pltEntryAddr) const override;
1604};
1605} // namespace
1606
1607Retpoline::Retpoline(Ctx &ctx) : X86_64(ctx) {
1608 pltHeaderSize = 48;
1609 pltEntrySize = 32;
1610 ipltEntrySize = 32;
1611}
1612
1613void Retpoline::writeGotPlt(uint8_t *buf, const Symbol &s) const {
1614 write64le(P: buf, V: s.getPltVA(ctx) + 17);
1615}
1616
1617void Retpoline::writePltHeader(uint8_t *buf) const {
1618 const uint8_t insn[] = {
1619 0xff, 0x35, 0, 0, 0, 0, // 0: pushq GOTPLT+8(%rip)
1620 0x4c, 0x8b, 0x1d, 0, 0, 0, 0, // 6: mov GOTPLT+16(%rip), %r11
1621 0xe8, 0x0e, 0x00, 0x00, 0x00, // d: callq next
1622 0xf3, 0x90, // 12: loop: pause
1623 0x0f, 0xae, 0xe8, // 14: lfence
1624 0xeb, 0xf9, // 17: jmp loop
1625 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 19: int3; .align 16
1626 0x4c, 0x89, 0x1c, 0x24, // 20: next: mov %r11, (%rsp)
1627 0xc3, // 24: ret
1628 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 25: int3; padding
1629 0xcc, 0xcc, 0xcc, 0xcc, // 2c: int3; padding
1630 };
1631 memcpy(dest: buf, src: insn, n: sizeof(insn));
1632
1633 uint64_t gotPlt = ctx.in.gotPlt->getVA();
1634 uint64_t plt = ctx.in.plt->getVA();
1635 write32le(P: buf + 2, V: gotPlt - plt - 6 + 8);
1636 write32le(P: buf + 9, V: gotPlt - plt - 13 + 16);
1637}
1638
1639void Retpoline::writePlt(uint8_t *buf, const Symbol &sym,
1640 uint64_t pltEntryAddr) const {
1641 const uint8_t insn[] = {
1642 0x4c, 0x8b, 0x1d, 0, 0, 0, 0, // 0: mov foo@GOTPLT(%rip), %r11
1643 0xe8, 0, 0, 0, 0, // 7: callq plt+0x20
1644 0xe9, 0, 0, 0, 0, // c: jmp plt+0x12
1645 0x68, 0, 0, 0, 0, // 11: pushq <relocation index>
1646 0xe9, 0, 0, 0, 0, // 16: jmp plt+0
1647 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 1b: int3; padding
1648 };
1649 memcpy(dest: buf, src: insn, n: sizeof(insn));
1650
1651 uint64_t off = pltEntryAddr - ctx.in.plt->getVA();
1652
1653 write32le(P: buf + 3, V: sym.getGotPltVA(ctx) - pltEntryAddr - 7);
1654 write32le(P: buf + 8, V: -off - 12 + 32);
1655 write32le(P: buf + 13, V: -off - 17 + 18);
1656 write32le(P: buf + 18, V: sym.getPltIdx(ctx));
1657 write32le(P: buf + 23, V: -off - 27);
1658}
1659
1660RetpolineZNow::RetpolineZNow(Ctx &ctx) : X86_64(ctx) {
1661 pltHeaderSize = 32;
1662 pltEntrySize = 16;
1663 ipltEntrySize = 16;
1664}
1665
1666void RetpolineZNow::writePltHeader(uint8_t *buf) const {
1667 const uint8_t insn[] = {
1668 0xe8, 0x0b, 0x00, 0x00, 0x00, // 0: call next
1669 0xf3, 0x90, // 5: loop: pause
1670 0x0f, 0xae, 0xe8, // 7: lfence
1671 0xeb, 0xf9, // a: jmp loop
1672 0xcc, 0xcc, 0xcc, 0xcc, // c: int3; .align 16
1673 0x4c, 0x89, 0x1c, 0x24, // 10: next: mov %r11, (%rsp)
1674 0xc3, // 14: ret
1675 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 15: int3; padding
1676 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 1a: int3; padding
1677 0xcc, // 1f: int3; padding
1678 };
1679 memcpy(dest: buf, src: insn, n: sizeof(insn));
1680}
1681
1682void RetpolineZNow::writePlt(uint8_t *buf, const Symbol &sym,
1683 uint64_t pltEntryAddr) const {
1684 const uint8_t insn[] = {
1685 0x4c, 0x8b, 0x1d, 0, 0, 0, 0, // mov foo@GOTPLT(%rip), %r11
1686 0xe9, 0, 0, 0, 0, // jmp plt+0
1687 0xcc, 0xcc, 0xcc, 0xcc, // int3; padding
1688 };
1689 memcpy(dest: buf, src: insn, n: sizeof(insn));
1690
1691 write32le(P: buf + 3, V: sym.getGotPltVA(ctx) - pltEntryAddr - 7);
1692 write32le(P: buf + 8, V: ctx.in.plt->getVA() - pltEntryAddr - 12);
1693}
1694
1695void elf::setX86_64TargetInfo(Ctx &ctx) {
1696 if (ctx.arg.zRetpolineplt) {
1697 if (ctx.arg.zNow)
1698 ctx.target.reset(p: new RetpolineZNow(ctx));
1699 else
1700 ctx.target.reset(p: new Retpoline(ctx));
1701 return;
1702 }
1703
1704 if (ctx.arg.andFeatures & GNU_PROPERTY_X86_FEATURE_1_IBT)
1705 ctx.target.reset(p: new IntelIBT(ctx));
1706 else
1707 ctx.target.reset(p: new X86_64(ctx));
1708}
1709