1//===- X86_64.cpp ---------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "OutputSections.h"
10#include "RelocScan.h"
11#include "Relocations.h"
12#include "Symbols.h"
13#include "SyntheticSections.h"
14#include "Target.h"
15#include "TargetImpl.h"
16#include "llvm/BinaryFormat/ELF.h"
17#include "llvm/Support/Endian.h"
18#include "llvm/Support/MathExtras.h"
19
20using namespace llvm;
21using namespace llvm::object;
22using namespace llvm::support::endian;
23using namespace llvm::ELF;
24using namespace lld;
25using namespace lld::elf;
26
27namespace {
28class X86_64 : public TargetInfo {
29public:
30 X86_64(Ctx &);
31 void initTargetSpecificSections() override;
32 RelExpr getRelExpr(RelType type, const Symbol &s,
33 const uint8_t *loc) const override;
34 RelType getDynRel(RelType type) const override;
35 void writeGotPltHeader(uint8_t *buf) const override;
36 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
37 void writeIgotPlt(uint8_t *buf, const Symbol &s) const override;
38 void writePltHeader(uint8_t *buf) const override;
39 void writePlt(uint8_t *buf, const Symbol &sym,
40 uint64_t pltEntryAddr) const override;
41 void relocate(uint8_t *loc, const Relocation &rel,
42 uint64_t val) const override;
43 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;
44 void applyJumpInstrMod(uint8_t *loc, JumpModType type,
45 unsigned size) const override;
46 RelExpr adjustGotPcExpr(RelType type, int64_t addend,
47 const uint8_t *loc) const override;
48 void relocateAlloc(InputSection &sec, uint8_t *buf) const override;
49 bool adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end,
50 uint8_t stOther) const override;
51 bool deleteFallThruJmpInsn(InputSection &is,
52 InputSection *nextIS) const override;
53 bool relaxOnce(int pass) const override;
54 void applyBranchToBranchOpt() const override;
55 template <class ELFT, class RelTy>
56 void scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels);
57 void scanSection(InputSectionBase &sec) override;
58
59private:
60 void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
61 void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
62 void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
63 void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
64};
65} // namespace
66
67// This is vector of NOP instructions of sizes from 1 to 8 bytes. The
68// appropriately sized instructions are used to fill the gaps between sections
69// which are executed during fall through.
70static const std::vector<std::vector<uint8_t>> nopInstructions = {
71 {0x90},
72 {0x66, 0x90},
73 {0x0f, 0x1f, 0x00},
74 {0x0f, 0x1f, 0x40, 0x00},
75 {0x0f, 0x1f, 0x44, 0x00, 0x00},
76 {0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00},
77 {0x0F, 0x1F, 0x80, 0x00, 0x00, 0x00, 0x00},
78 {0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00},
79 {0x66, 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00}};
80
81X86_64::X86_64(Ctx &ctx) : TargetInfo(ctx) {
82 copyRel = R_X86_64_COPY;
83 gotRel = R_X86_64_GLOB_DAT;
84 pltRel = R_X86_64_JUMP_SLOT;
85 relativeRel = R_X86_64_RELATIVE;
86 iRelativeRel = R_X86_64_IRELATIVE;
87 symbolicRel = ctx.arg.is64 ? R_X86_64_64 : R_X86_64_32;
88 tlsDescRel = R_X86_64_TLSDESC;
89 tlsGotRel = R_X86_64_TPOFF64;
90 tlsModuleIndexRel = R_X86_64_DTPMOD64;
91 tlsOffsetRel = R_X86_64_DTPOFF64;
92 gotBaseSymInGotPlt = true;
93 gotEntrySize = 8;
94 pltHeaderSize = 16;
95 pltEntrySize = 16;
96 ipltEntrySize = 16;
97 trapInstr = {0xcc, 0xcc, 0xcc, 0xcc}; // 0xcc = INT3
98 nopInstrs = nopInstructions;
99
100 // Align to the large page size (known as a superpage or huge page).
101 // FreeBSD automatically promotes large, superpage-aligned allocations.
102 defaultImageBase = 0x200000;
103}
104
105// Opcodes for the different X86_64 jmp instructions.
106enum JmpInsnOpcode : uint32_t {
107 J_JMP_32,
108 J_JNE_32,
109 J_JE_32,
110 J_JG_32,
111 J_JGE_32,
112 J_JB_32,
113 J_JBE_32,
114 J_JL_32,
115 J_JLE_32,
116 J_JA_32,
117 J_JAE_32,
118 J_UNKNOWN,
119};
120
121// Given the first (optional) and second byte of the insn's opcode, this
122// returns the corresponding enum value.
123static JmpInsnOpcode getJmpInsnType(const uint8_t *first,
124 const uint8_t *second) {
125 if (*second == 0xe9)
126 return J_JMP_32;
127
128 if (first == nullptr)
129 return J_UNKNOWN;
130
131 if (*first == 0x0f) {
132 switch (*second) {
133 case 0x84:
134 return J_JE_32;
135 case 0x85:
136 return J_JNE_32;
137 case 0x8f:
138 return J_JG_32;
139 case 0x8d:
140 return J_JGE_32;
141 case 0x82:
142 return J_JB_32;
143 case 0x86:
144 return J_JBE_32;
145 case 0x8c:
146 return J_JL_32;
147 case 0x8e:
148 return J_JLE_32;
149 case 0x87:
150 return J_JA_32;
151 case 0x83:
152 return J_JAE_32;
153 }
154 }
155 return J_UNKNOWN;
156}
157
158// Return the relocation index for input section IS with a specific Offset.
159// Returns the maximum size of the vector if no such relocation is found.
160static unsigned getRelocationWithOffset(const InputSection &is,
161 uint64_t offset) {
162 unsigned size = is.relocs().size();
163 for (unsigned i = size - 1; i + 1 > 0; --i) {
164 if (is.relocs()[i].offset == offset && is.relocs()[i].expr != R_NONE)
165 return i;
166 }
167 return size;
168}
169
170// Returns true if R corresponds to a relocation used for a jump instruction.
171// TODO: Once special relocations for relaxable jump instructions are available,
172// this should be modified to use those relocations.
173static bool isRelocationForJmpInsn(Relocation &R) {
174 return R.type == R_X86_64_PLT32 || R.type == R_X86_64_PC32 ||
175 R.type == R_X86_64_PC8;
176}
177
178// Return true if Relocation R points to the first instruction in the
179// next section.
180// TODO: Delete this once psABI reserves a new relocation type for fall thru
181// jumps.
182static bool isFallThruRelocation(InputSection &is, InputSection *nextIS,
183 Relocation &r) {
184 if (!isRelocationForJmpInsn(R&: r))
185 return false;
186
187 uint64_t addrLoc = is.getOutputSection()->addr + is.outSecOff + r.offset;
188 uint64_t targetOffset = is.getRelocTargetVA(is.getCtx(), r, p: addrLoc);
189
190 // If this jmp is a fall thru, the target offset is the beginning of the
191 // next section.
192 uint64_t nextSectionOffset =
193 nextIS->getOutputSection()->addr + nextIS->outSecOff;
194 return (addrLoc + 4 + targetOffset) == nextSectionOffset;
195}
196
197// Return the jmp instruction opcode that is the inverse of the given
198// opcode. For example, JE inverted is JNE.
199static JmpInsnOpcode invertJmpOpcode(const JmpInsnOpcode opcode) {
200 switch (opcode) {
201 case J_JE_32:
202 return J_JNE_32;
203 case J_JNE_32:
204 return J_JE_32;
205 case J_JG_32:
206 return J_JLE_32;
207 case J_JGE_32:
208 return J_JL_32;
209 case J_JB_32:
210 return J_JAE_32;
211 case J_JBE_32:
212 return J_JA_32;
213 case J_JL_32:
214 return J_JGE_32;
215 case J_JLE_32:
216 return J_JG_32;
217 case J_JA_32:
218 return J_JBE_32;
219 case J_JAE_32:
220 return J_JB_32;
221 default:
222 return J_UNKNOWN;
223 }
224}
225
226// Deletes direct jump instruction in input sections that jumps to the
227// following section as it is not required. If there are two consecutive jump
228// instructions, it checks if they can be flipped and one can be deleted.
229// For example:
230// .section .text
231// a.BB.foo:
232// ...
233// 10: jne aa.BB.foo
234// 16: jmp bar
235// aa.BB.foo:
236// ...
237//
238// can be converted to:
239// a.BB.foo:
240// ...
241// 10: je bar #jne flipped to je and the jmp is deleted.
242// aa.BB.foo:
243// ...
244bool X86_64::deleteFallThruJmpInsn(InputSection &is,
245 InputSection *nextIS) const {
246 const unsigned sizeOfDirectJmpInsn = 5;
247
248 if (nextIS == nullptr)
249 return false;
250
251 if (is.getSize() < sizeOfDirectJmpInsn)
252 return false;
253
254 // If this jmp insn can be removed, it is the last insn and the
255 // relocation is 4 bytes before the end.
256 unsigned rIndex = getRelocationWithOffset(is, offset: is.getSize() - 4);
257 if (rIndex == is.relocs().size())
258 return false;
259
260 Relocation &r = is.relocs()[rIndex];
261
262 // Check if the relocation corresponds to a direct jmp.
263 const uint8_t *secContents = is.content().data();
264 // If it is not a direct jmp instruction, there is nothing to do here.
265 if (*(secContents + r.offset - 1) != 0xe9)
266 return false;
267
268 if (isFallThruRelocation(is, nextIS, r)) {
269 // This is a fall thru and can be deleted.
270 r.expr = R_NONE;
271 r.offset = 0;
272 is.drop_back(num: sizeOfDirectJmpInsn);
273 is.nopFiller = true;
274 return true;
275 }
276
277 // Now, check if flip and delete is possible.
278 const unsigned sizeOfJmpCCInsn = 6;
279 // To flip, there must be at least one JmpCC and one direct jmp.
280 if (is.getSize() < sizeOfDirectJmpInsn + sizeOfJmpCCInsn)
281 return false;
282
283 unsigned rbIndex =
284 getRelocationWithOffset(is, offset: (is.getSize() - sizeOfDirectJmpInsn - 4));
285 if (rbIndex == is.relocs().size())
286 return false;
287
288 Relocation &rB = is.relocs()[rbIndex];
289
290 const uint8_t *jmpInsnB = secContents + rB.offset - 1;
291 JmpInsnOpcode jmpOpcodeB = getJmpInsnType(first: jmpInsnB - 1, second: jmpInsnB);
292 if (jmpOpcodeB == J_UNKNOWN)
293 return false;
294
295 if (!isFallThruRelocation(is, nextIS, r&: rB))
296 return false;
297
298 // jmpCC jumps to the fall thru block, the branch can be flipped and the
299 // jmp can be deleted.
300 JmpInsnOpcode jInvert = invertJmpOpcode(opcode: jmpOpcodeB);
301 if (jInvert == J_UNKNOWN)
302 return false;
303 is.jumpInstrMod = make<JumpInstrMod>();
304 *is.jumpInstrMod = {.offset: rB.offset - 1, .original: jInvert, .size: 4};
305 // Move R's values to rB except the offset.
306 rB = {.expr: r.expr, .type: r.type, .offset: rB.offset, .addend: r.addend, .sym: r.sym};
307 // Cancel R
308 r.expr = R_NONE;
309 r.offset = 0;
310 is.drop_back(num: sizeOfDirectJmpInsn);
311 is.nopFiller = true;
312 return true;
313}
314
315bool X86_64::relaxOnce(int pass) const {
316 uint64_t minVA = UINT64_MAX, maxVA = 0;
317 for (OutputSection *osec : ctx.outputSections) {
318 if (!(osec->flags & SHF_ALLOC))
319 continue;
320 minVA = std::min(a: minVA, b: osec->addr);
321 maxVA = std::max(a: maxVA, b: osec->addr + osec->size);
322 }
323 // If the max VA is under 2^31, GOTPCRELX relocations cannot overfow. In
324 // -pie/-shared, the condition can be relaxed to test the max VA difference as
325 // there is no R_RELAX_GOT_PC_NOPIC.
326 if (isUInt<31>(x: maxVA) || (isUInt<31>(x: maxVA - minVA) && ctx.arg.isPic))
327 return false;
328
329 SmallVector<InputSection *, 0> storage;
330 bool changed = false;
331 for (OutputSection *osec : ctx.outputSections) {
332 if (!(osec->flags & SHF_EXECINSTR))
333 continue;
334 for (InputSection *sec : getInputSections(os: *osec, storage)) {
335 for (Relocation &rel : sec->relocs()) {
336 if (rel.expr != R_RELAX_GOT_PC && rel.expr != R_RELAX_GOT_PC_NOPIC)
337 continue;
338 assert(rel.addend == -4);
339
340 Relocation rel1 = rel;
341 rel1.addend = rel.expr == R_RELAX_GOT_PC_NOPIC ? 0 : -4;
342 uint64_t v = sec->getRelocTargetVA(ctx, r: rel1,
343 p: sec->getOutputSection()->addr +
344 sec->outSecOff + rel.offset);
345 if (isInt<32>(x: v))
346 continue;
347 if (rel.sym->auxIdx == 0) {
348 rel.sym->allocateAux(ctx);
349 addGotEntry(ctx, sym&: *rel.sym);
350 changed = true;
351 }
352 rel.expr = R_GOT_PC;
353 }
354 }
355 }
356 return changed;
357}
358
359void X86_64::initTargetSpecificSections() {
360 if (ctx.arg.andFeatures & GNU_PROPERTY_X86_FEATURE_1_IBT) {
361 ctx.in.ibtPlt = std::make_unique<IBTPltSection>(args&: ctx);
362 ctx.inputSections.push_back(Elt: ctx.in.ibtPlt.get());
363 }
364}
365
366// Only needed to support relocations used by relocateNonAlloc and relocateEh.
367RelExpr X86_64::getRelExpr(RelType type, const Symbol &s,
368 const uint8_t *loc) const {
369 switch (type) {
370 case R_X86_64_8:
371 case R_X86_64_16:
372 case R_X86_64_32:
373 case R_X86_64_32S:
374 case R_X86_64_64:
375 return R_ABS;
376 case R_X86_64_SIZE32:
377 case R_X86_64_SIZE64:
378 return R_SIZE;
379 case R_X86_64_DTPOFF32:
380 case R_X86_64_DTPOFF64:
381 return R_DTPREL;
382 case R_X86_64_PC8:
383 case R_X86_64_PC16:
384 case R_X86_64_PC32:
385 case R_X86_64_PC64:
386 return R_PC;
387 case R_X86_64_GOTOFF64:
388 return R_GOTPLTREL;
389 case R_X86_64_GOTPC32:
390 case R_X86_64_GOTPC64:
391 return R_GOTPLTONLY_PC;
392 case R_X86_64_NONE:
393 return R_NONE;
394 default:
395 Err(ctx) << getErrorLoc(ctx, loc) << "unknown relocation (" << type.v
396 << ") against symbol " << &s;
397 return R_NONE;
398 }
399}
400
401void X86_64::writeGotPltHeader(uint8_t *buf) const {
402 // The first entry holds the link-time address of _DYNAMIC. It is documented
403 // in the psABI and glibc before Aug 2021 used the entry to compute run-time
404 // load address of the shared object (note that this is relevant for linking
405 // ld.so, not any other program).
406 write64le(P: buf, V: ctx.mainPart->dynamic->getVA());
407}
408
409void X86_64::writeGotPlt(uint8_t *buf, const Symbol &s) const {
410 // See comments in X86::writeGotPlt.
411 write64le(P: buf, V: s.getPltVA(ctx) + 6);
412}
413
414void X86_64::writeIgotPlt(uint8_t *buf, const Symbol &s) const {
415 // An x86 entry is the address of the ifunc resolver function (for -z rel).
416 if (ctx.arg.writeAddends)
417 write64le(P: buf, V: s.getVA(ctx));
418}
419
420void X86_64::writePltHeader(uint8_t *buf) const {
421 const uint8_t pltData[] = {
422 0xff, 0x35, 0, 0, 0, 0, // pushq GOTPLT+8(%rip)
423 0xff, 0x25, 0, 0, 0, 0, // jmp *GOTPLT+16(%rip)
424 0x0f, 0x1f, 0x40, 0x00, // nop
425 };
426 memcpy(dest: buf, src: pltData, n: sizeof(pltData));
427 uint64_t gotPlt = ctx.in.gotPlt->getVA();
428 uint64_t plt = ctx.in.ibtPlt ? ctx.in.ibtPlt->getVA() : ctx.in.plt->getVA();
429 write32le(P: buf + 2, V: gotPlt - plt + 2); // GOTPLT+8
430 write32le(P: buf + 8, V: gotPlt - plt + 4); // GOTPLT+16
431}
432
433void X86_64::writePlt(uint8_t *buf, const Symbol &sym,
434 uint64_t pltEntryAddr) const {
435 const uint8_t inst[] = {
436 0xff, 0x25, 0, 0, 0, 0, // jmpq *got(%rip)
437 0x68, 0, 0, 0, 0, // pushq <relocation index>
438 0xe9, 0, 0, 0, 0, // jmpq plt[0]
439 };
440 memcpy(dest: buf, src: inst, n: sizeof(inst));
441
442 write32le(P: buf + 2, V: sym.getGotPltVA(ctx) - pltEntryAddr - 6);
443 write32le(P: buf + 7, V: sym.getPltIdx(ctx));
444 write32le(P: buf + 12, V: ctx.in.plt->getVA() - pltEntryAddr - 16);
445}
446
447RelType X86_64::getDynRel(RelType type) const {
448 if (type == symbolicRel || type == R_X86_64_SIZE32 || type == R_X86_64_SIZE64)
449 return type;
450 return R_X86_64_NONE;
451}
452
453template <class ELFT, class RelTy>
454void X86_64::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
455 RelocScan rs(ctx, &sec);
456 sec.relocations.reserve(N: rels.size());
457
458 for (auto it = rels.begin(); it != rels.end(); ++it) {
459 const RelTy &rel = *it;
460 uint32_t symIdx = rel.getSymbol(false);
461 Symbol &sym = sec.getFile<ELFT>()->getSymbol(symIdx);
462 uint64_t offset = rel.r_offset;
463 RelType type = rel.getType(false);
464 if (sym.isUndefined() && symIdx != 0 &&
465 rs.maybeReportUndefined(sym&: cast<Undefined>(Val&: sym), offset))
466 continue;
467 int64_t addend = rs.getAddend<ELFT>(rel, type);
468 RelExpr expr;
469 // Relocation types that only need a RelExpr set `expr` and break out of
470 // the switch to reach rs.process(). Types that need special handling
471 // (fast-path helpers, TLS) call a handler and use `continue`.
472 switch (type) {
473 case R_X86_64_NONE:
474 continue;
475
476 // Absolute relocations:
477 case R_X86_64_8:
478 case R_X86_64_16:
479 case R_X86_64_32:
480 case R_X86_64_32S:
481 case R_X86_64_64:
482 expr = R_ABS;
483 break;
484
485 // PC-relative relocations:
486 case R_X86_64_PC8:
487 case R_X86_64_PC16:
488 case R_X86_64_PC32:
489 case R_X86_64_PC64:
490 rs.processR_PC(type, offset, addend, sym);
491 continue;
492
493 // GOT-generating relocations:
494 case R_X86_64_GOTPC32:
495 case R_X86_64_GOTPC64:
496 ctx.in.gotPlt->hasGotPltOffRel.store(i: true, m: std::memory_order_relaxed);
497 expr = R_GOTPLTONLY_PC;
498 break;
499 case R_X86_64_GOTOFF64:
500 ctx.in.gotPlt->hasGotPltOffRel.store(i: true, m: std::memory_order_relaxed);
501 expr = R_GOTPLTREL;
502 break;
503 case R_X86_64_GOT32:
504 case R_X86_64_GOT64:
505 ctx.in.gotPlt->hasGotPltOffRel.store(i: true, m: std::memory_order_relaxed);
506 expr = R_GOTPLT;
507 break;
508 case R_X86_64_PLTOFF64:
509 ctx.in.gotPlt->hasGotPltOffRel.store(i: true, m: std::memory_order_relaxed);
510 expr = R_PLT_GOTPLT;
511 break;
512 case R_X86_64_GOTPCREL:
513 case R_X86_64_GOTPCRELX:
514 case R_X86_64_REX_GOTPCRELX:
515 case R_X86_64_CODE_4_GOTPCRELX:
516 expr = R_GOT_PC;
517 break;
518
519 // PLT-generating relocation:
520 case R_X86_64_PLT32:
521 rs.processR_PLT_PC(type, offset, addend, sym);
522 continue;
523
524 // TLS relocations:
525 case R_X86_64_TPOFF32:
526 case R_X86_64_TPOFF64:
527 if (rs.checkTlsLe(offset, sym, type))
528 continue;
529 expr = R_TPREL;
530 break;
531 case R_X86_64_GOTTPOFF:
532 case R_X86_64_CODE_4_GOTTPOFF:
533 case R_X86_64_CODE_6_GOTTPOFF:
534 rs.handleTlsIe(ieExpr: R_GOT_PC, type, offset, addend, sym);
535 continue;
536 case R_X86_64_TLSGD:
537 if (rs.handleTlsGd(sharedExpr: R_TLSGD_PC, ieExpr: R_GOT_PC, leExpr: R_TPREL, type, offset, addend,
538 sym))
539 ++it;
540 continue;
541 case R_X86_64_TLSLD:
542 if (rs.handleTlsLd(sharedExpr: R_TLSLD_PC, type, offset, addend, sym))
543 ++it;
544 continue;
545 case R_X86_64_DTPOFF32:
546 case R_X86_64_DTPOFF64:
547 sec.addReloc(
548 r: {.expr: ctx.arg.shared ? R_DTPREL : R_TPREL, .type: type, .offset: offset, .addend: addend, .sym: &sym});
549 continue;
550 case R_X86_64_TLSDESC_CALL:
551 // For executables, TLSDESC is optimized to IE or LE. Use R_TPREL as the
552 // rewrites for this relocation are identical.
553 if (!ctx.arg.shared)
554 sec.addReloc(r: {.expr: R_TPREL, .type: type, .offset: offset, .addend: addend, .sym: &sym});
555 continue;
556 case R_X86_64_GOTPC32_TLSDESC:
557 case R_X86_64_CODE_4_GOTPC32_TLSDESC:
558 rs.handleTlsDesc(sharedExpr: R_TLSDESC_PC, ieExpr: R_GOT_PC, type, offset, addend, sym);
559 continue;
560
561 // Misc relocations:
562 case R_X86_64_SIZE32:
563 case R_X86_64_SIZE64:
564 expr = R_SIZE;
565 break;
566
567 default:
568 Err(ctx) << getErrorLoc(ctx, loc: sec.content().data() + offset)
569 << "unknown relocation (" << type.v << ") against symbol "
570 << &sym;
571 continue;
572 }
573 rs.process(expr, type, offset, sym, addend);
574 }
575
576 if (ctx.arg.branchToBranch)
577 llvm::stable_sort(sec.relocs(),
578 [](auto &l, auto &r) { return l.offset < r.offset; });
579}
580
581void X86_64::scanSection(InputSectionBase &sec) {
582 if (ctx.arg.is64)
583 elf::scanSection1<X86_64, ELF64LE>(target&: *this, sec);
584 else // ilp32
585 elf::scanSection1<X86_64, ELF32LE>(target&: *this, sec);
586}
587
588void X86_64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
589 uint64_t val) const {
590 if (rel.type == R_X86_64_TLSGD) {
591 // Convert
592 // .byte 0x66
593 // leaq x@tlsgd(%rip), %rdi
594 // .word 0x6666
595 // rex64
596 // call __tls_get_addr@plt
597 // to the following two instructions.
598 const uint8_t inst[] = {
599 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00,
600 0x00, 0x00, // mov %fs:0x0,%rax
601 0x48, 0x8d, 0x80, 0, 0, 0, 0, // lea x@tpoff,%rax
602 };
603 memcpy(dest: loc - 4, src: inst, n: sizeof(inst));
604
605 // The original code used a pc relative relocation and so we have to
606 // compensate for the -4 in had in the addend.
607 write32le(P: loc + 8, V: val + 4);
608 } else if (rel.type == R_X86_64_GOTPC32_TLSDESC ||
609 rel.type == R_X86_64_CODE_4_GOTPC32_TLSDESC) {
610 // Convert leaq x@tlsdesc(%rip), %REG to movq $x@tpoff, %REG.
611 if ((loc[-3] & 0xfb) != 0x48 || loc[-2] != 0x8d ||
612 (loc[-1] & 0xc7) != 0x05) {
613 Err(ctx) << getErrorLoc(ctx, loc: (rel.type == R_X86_64_GOTPC32_TLSDESC)
614 ? loc - 3
615 : loc - 4)
616 << "R_X86_64_GOTPC32_TLSDESC/R_X86_64_CODE_4_GOTPC32_TLSDESC "
617 "must be used in leaq x@tlsdesc(%rip), %REG";
618 return;
619 }
620 if (rel.type == R_X86_64_GOTPC32_TLSDESC) {
621 loc[-3] = 0x48 | ((loc[-3] >> 2) & 1);
622 } else {
623 loc[-3] = (loc[-3] & ~0x44) | ((loc[-3] & 0x44) >> 2);
624 }
625 loc[-2] = 0xc7;
626 loc[-1] = 0xc0 | ((loc[-1] >> 3) & 7);
627
628 write32le(P: loc, V: val + 4);
629 } else {
630 // Convert call *x@tlsdesc(%REG) to xchg ax, ax.
631 assert(rel.type == R_X86_64_TLSDESC_CALL);
632 loc[0] = 0x66;
633 loc[1] = 0x90;
634 }
635}
636
637void X86_64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
638 uint64_t val) const {
639 if (rel.type == R_X86_64_TLSGD) {
640 // Convert
641 // .byte 0x66
642 // leaq x@tlsgd(%rip), %rdi
643 // .word 0x6666
644 // rex64
645 // call __tls_get_addr@plt
646 // to the following two instructions.
647 const uint8_t inst[] = {
648 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00,
649 0x00, 0x00, // mov %fs:0x0,%rax
650 0x48, 0x03, 0x05, 0, 0, 0, 0, // addq x@gottpoff(%rip),%rax
651 };
652 memcpy(dest: loc - 4, src: inst, n: sizeof(inst));
653
654 // Both code sequences are PC relatives, but since we are moving the
655 // constant forward by 8 bytes we have to subtract the value by 8.
656 write32le(P: loc + 8, V: val - 8);
657 } else if (rel.type == R_X86_64_GOTPC32_TLSDESC ||
658 rel.type == R_X86_64_CODE_4_GOTPC32_TLSDESC) {
659 // Convert leaq x@tlsdesc(%rip), %REG to movq x@gottpoff(%rip), %REG.
660 if ((loc[-3] & 0xfb) != 0x48 || loc[-2] != 0x8d ||
661 (loc[-1] & 0xc7) != 0x05) {
662 Err(ctx) << getErrorLoc(ctx, loc: (rel.type == R_X86_64_GOTPC32_TLSDESC)
663 ? loc - 3
664 : loc - 4)
665 << "R_X86_64_GOTPC32_TLSDESC/R_X86_64_CODE_4_GOTPC32_TLSDESC "
666 "must be used in leaq x@tlsdesc(%rip), %REG";
667 return;
668 }
669 loc[-2] = 0x8b;
670 write32le(P: loc, V: val);
671 }
672}
673
674// In some conditions,
675// R_X86_64_GOTTPOFF/R_X86_64_CODE_4_GOTTPOFF/R_X86_64_CODE_6_GOTTPOFF
676// relocation can be optimized to R_X86_64_TPOFF32 so that it does not use GOT.
677void X86_64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
678 uint64_t val) const {
679 uint8_t *inst = loc - 3;
680 uint8_t reg = loc[-1] >> 3;
681 uint8_t *regSlot = loc - 1;
682
683 if (rel.type == R_X86_64_GOTTPOFF) {
684 // Note that ADD with RSP or R12 is converted to ADD instead of LEA
685 // because LEA with these registers needs 4 bytes to encode and thus
686 // wouldn't fit the space.
687
688 if (memcmp(s1: inst, s2: "\x48\x03\x25", n: 3) == 0) {
689 // "addq foo@gottpoff(%rip),%rsp" -> "addq $foo,%rsp"
690 memcpy(dest: inst, src: "\x48\x81\xc4", n: 3);
691 } else if (memcmp(s1: inst, s2: "\x4c\x03\x25", n: 3) == 0) {
692 // "addq foo@gottpoff(%rip),%r12" -> "addq $foo,%r12"
693 memcpy(dest: inst, src: "\x49\x81\xc4", n: 3);
694 } else if (memcmp(s1: inst, s2: "\x4c\x03", n: 2) == 0) {
695 // "addq foo@gottpoff(%rip),%r[8-15]" -> "leaq foo(%r[8-15]),%r[8-15]"
696 memcpy(dest: inst, src: "\x4d\x8d", n: 2);
697 *regSlot = 0x80 | (reg << 3) | reg;
698 } else if (memcmp(s1: inst, s2: "\x48\x03", n: 2) == 0) {
699 // "addq foo@gottpoff(%rip),%reg -> "leaq foo(%reg),%reg"
700 memcpy(dest: inst, src: "\x48\x8d", n: 2);
701 *regSlot = 0x80 | (reg << 3) | reg;
702 } else if (memcmp(s1: inst, s2: "\x4c\x8b", n: 2) == 0) {
703 // "movq foo@gottpoff(%rip),%r[8-15]" -> "movq $foo,%r[8-15]"
704 memcpy(dest: inst, src: "\x49\xc7", n: 2);
705 *regSlot = 0xc0 | reg;
706 } else if (memcmp(s1: inst, s2: "\x48\x8b", n: 2) == 0) {
707 // "movq foo@gottpoff(%rip),%reg" -> "movq $foo,%reg"
708 memcpy(dest: inst, src: "\x48\xc7", n: 2);
709 *regSlot = 0xc0 | reg;
710 } else {
711 Err(ctx)
712 << getErrorLoc(ctx, loc: loc - 3)
713 << "R_X86_64_GOTTPOFF must be used in MOVQ or ADDQ instructions only";
714 }
715 } else if (rel.type == R_X86_64_CODE_4_GOTTPOFF) {
716 if (loc[-4] != 0xd5) {
717 Err(ctx) << getErrorLoc(ctx, loc: loc - 4)
718 << "invalid prefix with R_X86_64_CODE_4_GOTTPOFF!";
719 return;
720 }
721 const uint8_t rex = loc[-3];
722 loc[-3] = (rex & ~0x44) | (rex & 0x44) >> 2;
723 *regSlot = 0xc0 | reg;
724
725 if (loc[-2] == 0x8b) {
726 // "movq foo@gottpoff(%rip),%r[16-31]" -> "movq $foo,%r[16-31]"
727 loc[-2] = 0xc7;
728 } else if (loc[-2] == 0x03) {
729 // "addq foo@gottpoff(%rip),%r[16-31]" -> "addq $foo,%r[16-31]"
730 loc[-2] = 0x81;
731 } else {
732 Err(ctx) << getErrorLoc(ctx, loc: loc - 4)
733 << "R_X86_64_CODE_4_GOTTPOFF must be used in MOVQ or ADDQ "
734 "instructions only";
735 }
736 } else if (rel.type == R_X86_64_CODE_6_GOTTPOFF) {
737 if (loc[-6] != 0x62) {
738 Err(ctx) << getErrorLoc(ctx, loc: loc - 6)
739 << "invalid prefix with R_X86_64_CODE_6_GOTTPOFF!";
740 return;
741 }
742 // Check bits are satisfied:
743 // loc[-5]: X==1 (inverted polarity), (loc[-5] & 0x7) == 0x4
744 // loc[-4]: W==1, X2==1 (inverted polarity), pp==0b00(NP)
745 // loc[-3]: NF==1 or ND==1
746 // loc[-2]: opcode==0x1 or opcode==0x3
747 // loc[-1]: Mod==0b00, RM==0b101
748 if (((loc[-5] & 0x47) == 0x44) && ((loc[-4] & 0x87) == 0x84) &&
749 ((loc[-3] & 0x14) != 0) && (loc[-2] == 0x1 || loc[-2] == 0x3) &&
750 ((loc[-1] & 0xc7) == 0x5)) {
751 // "addq %reg1, foo@GOTTPOFF(%rip), %reg2" -> "addq $foo, %reg1, %reg2"
752 // "addq foo@GOTTPOFF(%rip), %reg1, %reg2" -> "addq $foo, %reg1, %reg2"
753 // "{nf} addq %reg1, foo@GOTTPOFF(%rip), %reg2"
754 // -> "{nf} addq $foo, %reg1, %reg2"
755 // "{nf} addq name@GOTTPOFF(%rip), %reg1, %reg2"
756 // -> "{nf} addq $foo, %reg1, %reg2"
757 // "{nf} addq name@GOTTPOFF(%rip), %reg" -> "{nf} addq $foo, %reg"
758 loc[-2] = 0x81;
759 // Move R bits to B bits in EVEX payloads and ModRM byte.
760 const uint8_t evexPayload0 = loc[-5];
761 if ((evexPayload0 & (1 << 7)) == 0)
762 loc[-5] = (evexPayload0 | (1 << 7)) & ~(1 << 5);
763 if ((evexPayload0 & (1 << 4)) == 0)
764 loc[-5] = evexPayload0 | (1 << 4) | (1 << 3);
765 *regSlot = 0xc0 | reg;
766 } else {
767 Err(ctx) << getErrorLoc(ctx, loc: loc - 6)
768 << "R_X86_64_CODE_6_GOTTPOFF must be used in ADDQ instructions "
769 "with NDD/NF/NDD+NF only";
770 }
771 } else {
772 llvm_unreachable("Unsupported relocation type!");
773 }
774
775 // The original code used a PC relative relocation.
776 // Need to compensate for the -4 it had in the addend.
777 write32le(P: loc, V: val + 4);
778}
779
780void X86_64::relaxTlsLdToLe(uint8_t *loc, const Relocation &rel,
781 uint64_t val) const {
782 const uint8_t inst[] = {
783 0x66, 0x66, // .word 0x6666
784 0x66, // .byte 0x66
785 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0,%rax
786 };
787
788 if (loc[4] == 0xe8) {
789 // Convert
790 // leaq bar@tlsld(%rip), %rdi # 48 8d 3d <Loc>
791 // callq __tls_get_addr@PLT # e8 <disp32>
792 // leaq bar@dtpoff(%rax), %rcx
793 // to
794 // .word 0x6666
795 // .byte 0x66
796 // mov %fs:0,%rax
797 // leaq bar@tpoff(%rax), %rcx
798 memcpy(dest: loc - 3, src: inst, n: sizeof(inst));
799 return;
800 }
801
802 if (loc[4] == 0xff && loc[5] == 0x15) {
803 // Convert
804 // leaq x@tlsld(%rip),%rdi # 48 8d 3d <Loc>
805 // call *__tls_get_addr@GOTPCREL(%rip) # ff 15 <disp32>
806 // to
807 // .long 0x66666666
808 // movq %fs:0,%rax
809 // See "Table 11.9: LD -> LE Code Transition (LP64)" in
810 // https://raw.githubusercontent.com/wiki/hjl-tools/x86-psABI/x86-64-psABI-1.0.pdf
811 loc[-3] = 0x66;
812 memcpy(dest: loc - 2, src: inst, n: sizeof(inst));
813 return;
814 }
815
816 ErrAlways(ctx)
817 << getErrorLoc(ctx, loc: loc - 3)
818 << "expected R_X86_64_PLT32 or R_X86_64_GOTPCRELX after R_X86_64_TLSLD";
819}
820
821// A JumpInstrMod at a specific offset indicates that the jump instruction
822// opcode at that offset must be modified. This is specifically used to relax
823// jump instructions with basic block sections. This function looks at the
824// JumpMod and effects the change.
825void X86_64::applyJumpInstrMod(uint8_t *loc, JumpModType type,
826 unsigned size) const {
827 switch (type) {
828 case J_JMP_32:
829 if (size == 4)
830 *loc = 0xe9;
831 else
832 *loc = 0xeb;
833 break;
834 case J_JE_32:
835 if (size == 4) {
836 loc[-1] = 0x0f;
837 *loc = 0x84;
838 } else
839 *loc = 0x74;
840 break;
841 case J_JNE_32:
842 if (size == 4) {
843 loc[-1] = 0x0f;
844 *loc = 0x85;
845 } else
846 *loc = 0x75;
847 break;
848 case J_JG_32:
849 if (size == 4) {
850 loc[-1] = 0x0f;
851 *loc = 0x8f;
852 } else
853 *loc = 0x7f;
854 break;
855 case J_JGE_32:
856 if (size == 4) {
857 loc[-1] = 0x0f;
858 *loc = 0x8d;
859 } else
860 *loc = 0x7d;
861 break;
862 case J_JB_32:
863 if (size == 4) {
864 loc[-1] = 0x0f;
865 *loc = 0x82;
866 } else
867 *loc = 0x72;
868 break;
869 case J_JBE_32:
870 if (size == 4) {
871 loc[-1] = 0x0f;
872 *loc = 0x86;
873 } else
874 *loc = 0x76;
875 break;
876 case J_JL_32:
877 if (size == 4) {
878 loc[-1] = 0x0f;
879 *loc = 0x8c;
880 } else
881 *loc = 0x7c;
882 break;
883 case J_JLE_32:
884 if (size == 4) {
885 loc[-1] = 0x0f;
886 *loc = 0x8e;
887 } else
888 *loc = 0x7e;
889 break;
890 case J_JA_32:
891 if (size == 4) {
892 loc[-1] = 0x0f;
893 *loc = 0x87;
894 } else
895 *loc = 0x77;
896 break;
897 case J_JAE_32:
898 if (size == 4) {
899 loc[-1] = 0x0f;
900 *loc = 0x83;
901 } else
902 *loc = 0x73;
903 break;
904 case J_UNKNOWN:
905 llvm_unreachable("Unknown Jump Relocation");
906 }
907}
908
909int64_t X86_64::getImplicitAddend(const uint8_t *buf, RelType type) const {
910 switch (type) {
911 case R_X86_64_8:
912 case R_X86_64_PC8:
913 return SignExtend64<8>(x: *buf);
914 case R_X86_64_16:
915 case R_X86_64_PC16:
916 return SignExtend64<16>(x: read16le(P: buf));
917 case R_X86_64_32:
918 case R_X86_64_32S:
919 case R_X86_64_TPOFF32:
920 case R_X86_64_GOT32:
921 case R_X86_64_GOTPC32:
922 case R_X86_64_GOTPC32_TLSDESC:
923 case R_X86_64_GOTPCREL:
924 case R_X86_64_GOTPCRELX:
925 case R_X86_64_REX_GOTPCRELX:
926 case R_X86_64_CODE_4_GOTPCRELX:
927 case R_X86_64_PC32:
928 case R_X86_64_GOTTPOFF:
929 case R_X86_64_CODE_4_GOTTPOFF:
930 case R_X86_64_CODE_6_GOTTPOFF:
931 case R_X86_64_PLT32:
932 case R_X86_64_TLSGD:
933 case R_X86_64_TLSLD:
934 case R_X86_64_DTPOFF32:
935 case R_X86_64_SIZE32:
936 return SignExtend64<32>(x: read32le(P: buf));
937 case R_X86_64_64:
938 case R_X86_64_TPOFF64:
939 case R_X86_64_DTPOFF64:
940 case R_X86_64_DTPMOD64:
941 case R_X86_64_PC64:
942 case R_X86_64_SIZE64:
943 case R_X86_64_GLOB_DAT:
944 case R_X86_64_GOT64:
945 case R_X86_64_GOTOFF64:
946 case R_X86_64_GOTPC64:
947 case R_X86_64_PLTOFF64:
948 case R_X86_64_IRELATIVE:
949 case R_X86_64_RELATIVE:
950 return read64le(P: buf);
951 case R_X86_64_TLSDESC:
952 return read64le(P: buf + 8);
953 case R_X86_64_JUMP_SLOT:
954 case R_X86_64_NONE:
955 // These relocations are defined as not having an implicit addend.
956 return 0;
957 default:
958 InternalErr(ctx, buf) << "cannot read addend for relocation " << type;
959 return 0;
960 }
961}
962
963static void relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val);
964
965void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
966 switch (rel.type) {
967 case R_X86_64_8:
968 checkIntUInt(ctx, loc, v: val, n: 8, rel);
969 *loc = val;
970 break;
971 case R_X86_64_PC8:
972 checkInt(ctx, loc, v: val, n: 8, rel);
973 *loc = val;
974 break;
975 case R_X86_64_16:
976 checkIntUInt(ctx, loc, v: val, n: 16, rel);
977 write16le(P: loc, V: val);
978 break;
979 case R_X86_64_PC16:
980 checkInt(ctx, loc, v: val, n: 16, rel);
981 write16le(P: loc, V: val);
982 break;
983 case R_X86_64_32:
984 checkUInt(ctx, loc, v: val, n: 32, rel);
985 write32le(P: loc, V: val);
986 break;
987 case R_X86_64_32S:
988 case R_X86_64_GOT32:
989 case R_X86_64_GOTPC32:
990 case R_X86_64_GOTPCREL:
991 case R_X86_64_PC32:
992 case R_X86_64_PLT32:
993 case R_X86_64_DTPOFF32:
994 case R_X86_64_SIZE32:
995 checkInt(ctx, loc, v: val, n: 32, rel);
996 write32le(P: loc, V: val);
997 break;
998 case R_X86_64_64:
999 case R_X86_64_TPOFF64:
1000 case R_X86_64_DTPOFF64:
1001 case R_X86_64_PC64:
1002 case R_X86_64_SIZE64:
1003 case R_X86_64_GOT64:
1004 case R_X86_64_GOTOFF64:
1005 case R_X86_64_GOTPC64:
1006 case R_X86_64_PLTOFF64:
1007 write64le(P: loc, V: val);
1008 break;
1009 case R_X86_64_GOTPCRELX:
1010 case R_X86_64_REX_GOTPCRELX:
1011 case R_X86_64_CODE_4_GOTPCRELX:
1012 if (rel.expr != R_GOT_PC) {
1013 relaxGot(loc, rel, val);
1014 } else {
1015 checkInt(ctx, loc, v: val, n: 32, rel);
1016 write32le(P: loc, V: val);
1017 }
1018 break;
1019 case R_X86_64_GOTPC32_TLSDESC:
1020 case R_X86_64_CODE_4_GOTPC32_TLSDESC:
1021 case R_X86_64_TLSDESC_CALL:
1022 case R_X86_64_TLSGD:
1023 if (rel.expr == R_TPREL) {
1024 relaxTlsGdToLe(loc, rel, val);
1025 } else if (rel.expr == R_GOT_PC) {
1026 relaxTlsGdToIe(loc, rel, val);
1027 } else {
1028 checkInt(ctx, loc, v: val, n: 32, rel);
1029 write32le(P: loc, V: val);
1030 }
1031 break;
1032 case R_X86_64_TLSLD:
1033 if (rel.expr == R_TPREL) {
1034 relaxTlsLdToLe(loc, rel, val);
1035 } else {
1036 checkInt(ctx, loc, v: val, n: 32, rel);
1037 write32le(P: loc, V: val);
1038 }
1039 break;
1040 case R_X86_64_GOTTPOFF:
1041 case R_X86_64_CODE_4_GOTTPOFF:
1042 case R_X86_64_CODE_6_GOTTPOFF:
1043 if (rel.expr == R_TPREL) {
1044 relaxTlsIeToLe(loc, rel, val);
1045 } else {
1046 checkInt(ctx, loc, v: val, n: 32, rel);
1047 write32le(P: loc, V: val);
1048 }
1049 break;
1050 case R_X86_64_TPOFF32:
1051 checkInt(ctx, loc, v: val, n: 32, rel);
1052 write32le(P: loc, V: val);
1053 break;
1054
1055 case R_X86_64_TLSDESC:
1056 // The addend is stored in the second 64-bit word.
1057 write64le(P: loc + 8, V: val);
1058 break;
1059 default:
1060 llvm_unreachable("unknown relocation");
1061 }
1062}
1063
1064RelExpr X86_64::adjustGotPcExpr(RelType type, int64_t addend,
1065 const uint8_t *loc) const {
1066 // Only R_X86_64_[REX_]|[CODE_4_]GOTPCRELX can be relaxed. GNU as may emit
1067 // GOTPCRELX with addend != -4. Such an instruction does not load the full GOT
1068 // entry, so we cannot relax the relocation. E.g. movl x@GOTPCREL+4(%rip),
1069 // %rax (addend=0) loads the high 32 bits of the GOT entry.
1070 if (!ctx.arg.relax || addend != -4 ||
1071 (type != R_X86_64_GOTPCRELX && type != R_X86_64_REX_GOTPCRELX &&
1072 type != R_X86_64_CODE_4_GOTPCRELX))
1073 return R_GOT_PC;
1074 const uint8_t op = loc[-2];
1075 const uint8_t modRm = loc[-1];
1076
1077 // FIXME: When PIC is disabled and foo is defined locally in the
1078 // lower 32 bit address space, memory operand in mov can be converted into
1079 // immediate operand. Otherwise, mov must be changed to lea. We support only
1080 // latter relaxation at this moment.
1081 if (op == 0x8b)
1082 return R_RELAX_GOT_PC;
1083
1084 // Relax call and jmp.
1085 if (op == 0xff && (modRm == 0x15 || modRm == 0x25))
1086 return R_RELAX_GOT_PC;
1087
1088 // We don't support test/binop instructions without a REX/REX2 prefix.
1089 if (type == R_X86_64_GOTPCRELX)
1090 return R_GOT_PC;
1091
1092 // Relaxation of test, adc, add, and, cmp, or, sbb, sub, xor.
1093 // If PIC then no relaxation is available.
1094 return ctx.arg.isPic ? R_GOT_PC : R_RELAX_GOT_PC_NOPIC;
1095}
1096
1097// A subset of relaxations can only be applied for no-PIC. This method
1098// handles such relaxations. Instructions encoding information was taken from:
1099// "Intel 64 and IA-32 Architectures Software Developer's Manual V2"
1100// (http://www.intel.com/content/dam/www/public/us/en/documents/manuals/
1101// 64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf)
1102static void relaxGotNoPic(uint8_t *loc, uint64_t val, uint8_t op, uint8_t modRm,
1103 bool isRex2) {
1104 const uint8_t rex = loc[-3];
1105 // Convert "test %reg, foo@GOTPCREL(%rip)" to "test $foo, %reg".
1106 if (op == 0x85) {
1107 // See "TEST-Logical Compare" (4-428 Vol. 2B),
1108 // TEST r/m64, r64 uses "full" ModR / M byte (no opcode extension).
1109
1110 // ModR/M byte has form XX YYY ZZZ, where
1111 // YYY is MODRM.reg(register 2), ZZZ is MODRM.rm(register 1).
1112 // XX has different meanings:
1113 // 00: The operand's memory address is in reg1.
1114 // 01: The operand's memory address is reg1 + a byte-sized displacement.
1115 // 10: The operand's memory address is reg1 + a word-sized displacement.
1116 // 11: The operand is reg1 itself.
1117 // If an instruction requires only one operand, the unused reg2 field
1118 // holds extra opcode bits rather than a register code
1119 // 0xC0 == 11 000 000 binary.
1120 // 0x38 == 00 111 000 binary.
1121 // We transfer reg2 to reg1 here as operand.
1122 // See "2.1.3 ModR/M and SIB Bytes" (Vol. 2A 2-3).
1123 loc[-1] = 0xc0 | (modRm & 0x38) >> 3; // ModR/M byte.
1124
1125 // Change opcode from TEST r/m64, r64 to TEST r/m64, imm32
1126 // See "TEST-Logical Compare" (4-428 Vol. 2B).
1127 loc[-2] = 0xf7;
1128
1129 // Move R bit to the B bit in REX/REX2 byte.
1130 // REX byte is encoded as 0100WRXB, where
1131 // 0100 is 4bit fixed pattern.
1132 // REX.W When 1, a 64-bit operand size is used. Otherwise, when 0, the
1133 // default operand size is used (which is 32-bit for most but not all
1134 // instructions).
1135 // REX.R This 1-bit value is an extension to the MODRM.reg field.
1136 // REX.X This 1-bit value is an extension to the SIB.index field.
1137 // REX.B This 1-bit value is an extension to the MODRM.rm field or the
1138 // SIB.base field.
1139 // See "2.2.1.2 More on REX Prefix Fields " (2-8 Vol. 2A).
1140 //
1141 // REX2 prefix is encoded as 0xd5|M|R2|X2|B2|WRXB, where
1142 // 0xd5 is 1byte fixed pattern.
1143 // REX2's [W,R,X,B] have the same meanings as REX's.
1144 // REX2.M encodes the map id.
1145 // R2/X2/B2 provides the fifth and most siginicant bits of the R/X/B
1146 // register identifiers, each of which can now address all 32 GPRs.
1147 if (isRex2)
1148 loc[-3] = (rex & ~0x44) | (rex & 0x44) >> 2;
1149 else
1150 loc[-3] = (rex & ~0x4) | (rex & 0x4) >> 2;
1151 write32le(P: loc, V: val);
1152 return;
1153 }
1154
1155 // If we are here then we need to relax the adc, add, and, cmp, or, sbb, sub
1156 // or xor operations.
1157
1158 // Convert "binop foo@GOTPCREL(%rip), %reg" to "binop $foo, %reg".
1159 // Logic is close to one for test instruction above, but we also
1160 // write opcode extension here, see below for details.
1161 loc[-1] = 0xc0 | (modRm & 0x38) >> 3 | (op & 0x3c); // ModR/M byte.
1162
1163 // Primary opcode is 0x81, opcode extension is one of:
1164 // 000b = ADD, 001b is OR, 010b is ADC, 011b is SBB,
1165 // 100b is AND, 101b is SUB, 110b is XOR, 111b is CMP.
1166 // This value was wrote to MODRM.reg in a line above.
1167 // See "3.2 INSTRUCTIONS (A-M)" (Vol. 2A 3-15),
1168 // "INSTRUCTION SET REFERENCE, N-Z" (Vol. 2B 4-1) for
1169 // descriptions about each operation.
1170 loc[-2] = 0x81;
1171 if (isRex2)
1172 loc[-3] = (rex & ~0x44) | (rex & 0x44) >> 2;
1173 else
1174 loc[-3] = (rex & ~0x4) | (rex & 0x4) >> 2;
1175 write32le(P: loc, V: val);
1176}
1177
1178static void relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) {
1179 assert(isInt<32>(val) &&
1180 "GOTPCRELX should not have been relaxed if it overflows");
1181 const uint8_t op = loc[-2];
1182 const uint8_t modRm = loc[-1];
1183
1184 // Convert "mov foo@GOTPCREL(%rip),%reg" to "lea foo(%rip),%reg".
1185 if (op == 0x8b) {
1186 loc[-2] = 0x8d;
1187 write32le(P: loc, V: val);
1188 return;
1189 }
1190
1191 if (op != 0xff) {
1192 // We are relaxing a rip relative to an absolute, so compensate
1193 // for the old -4 addend.
1194 assert(!rel.sym->file->ctx.arg.isPic);
1195 relaxGotNoPic(loc, val: val + 4, op, modRm,
1196 isRex2: rel.type == R_X86_64_CODE_4_GOTPCRELX);
1197 return;
1198 }
1199
1200 // Convert call/jmp instructions.
1201 if (modRm == 0x15) {
1202 // ABI says we can convert "call *foo@GOTPCREL(%rip)" to "nop; call foo".
1203 // Instead we convert to "addr32 call foo" where addr32 is an instruction
1204 // prefix. That makes result expression to be a single instruction.
1205 loc[-2] = 0x67; // addr32 prefix
1206 loc[-1] = 0xe8; // call
1207 write32le(P: loc, V: val);
1208 return;
1209 }
1210
1211 // Convert "jmp *foo@GOTPCREL(%rip)" to "jmp foo; nop".
1212 // jmp doesn't return, so it is fine to use nop here, it is just a stub.
1213 assert(modRm == 0x25);
1214 loc[-2] = 0xe9; // jmp
1215 loc[3] = 0x90; // nop
1216 write32le(P: loc - 1, V: val + 1);
1217}
1218
1219// A split-stack prologue starts by checking the amount of stack remaining
1220// in one of two ways:
1221// A) Comparing of the stack pointer to a field in the tcb.
1222// B) Or a load of a stack pointer offset with an lea to r10 or r11.
1223bool X86_64::adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end,
1224 uint8_t stOther) const {
1225 if (!ctx.arg.is64) {
1226 ErrAlways(ctx) << "target doesn't support split stacks";
1227 return false;
1228 }
1229
1230 if (loc + 8 >= end)
1231 return false;
1232
1233 // Replace "cmp %fs:0x70,%rsp" and subsequent branch
1234 // with "stc, nopl 0x0(%rax,%rax,1)"
1235 if (memcmp(s1: loc, s2: "\x64\x48\x3b\x24\x25", n: 5) == 0) {
1236 memcpy(dest: loc, src: "\xf9\x0f\x1f\x84\x00\x00\x00\x00", n: 8);
1237 return true;
1238 }
1239
1240 // Adjust "lea X(%rsp),%rYY" to lea "(X - 0x4000)(%rsp),%rYY" where rYY could
1241 // be r10 or r11. The lea instruction feeds a subsequent compare which checks
1242 // if there is X available stack space. Making X larger effectively reserves
1243 // that much additional space. The stack grows downward so subtract the value.
1244 if (memcmp(s1: loc, s2: "\x4c\x8d\x94\x24", n: 4) == 0 ||
1245 memcmp(s1: loc, s2: "\x4c\x8d\x9c\x24", n: 4) == 0) {
1246 // The offset bytes are encoded four bytes after the start of the
1247 // instruction.
1248 write32le(P: loc + 4, V: read32le(P: loc + 4) - 0x4000);
1249 return true;
1250 }
1251 return false;
1252}
1253
1254void X86_64::relocateAlloc(InputSection &sec, uint8_t *buf) const {
1255 uint64_t secAddr = sec.getOutputSection()->addr + sec.outSecOff;
1256 for (const Relocation &rel : sec.relocs()) {
1257 if (rel.expr == R_NONE) // See deleteFallThruJmpInsn
1258 continue;
1259 uint8_t *loc = buf + rel.offset;
1260 const uint64_t val = sec.getRelocTargetVA(ctx, r: rel, p: secAddr + rel.offset);
1261 relocate(loc, rel, val);
1262 }
1263 if (sec.jumpInstrMod) {
1264 applyJumpInstrMod(loc: buf + sec.jumpInstrMod->offset,
1265 type: sec.jumpInstrMod->original, size: sec.jumpInstrMod->size);
1266 }
1267}
1268
1269static std::optional<uint64_t> getControlTransferAddend(InputSection &is,
1270 Relocation &r) {
1271 // Identify a control transfer relocation for the branch-to-branch
1272 // optimization. A "control transfer relocation" usually means a CALL or JMP
1273 // target but it also includes relative vtable relocations for example.
1274 //
1275 // We require the relocation type to be PLT32. With a relocation type of PLT32
1276 // the value may be assumed to be used for branching directly to the symbol
1277 // and the addend is only used to produce the relocated value (hence the
1278 // effective addend is always 0). This is because if a PLT is needed the
1279 // addend will be added to the address of the PLT, and it doesn't make sense
1280 // to branch into the middle of a PLT. For example, relative vtable
1281 // relocations use PLT32 and 0 or a positive value as the addend but still are
1282 // used to branch to the symbol.
1283 //
1284 // STT_SECTION symbols are a special case on x86 because the LLVM assembler
1285 // uses them for branches to local symbols which are assembled as referring to
1286 // the section symbol with the addend equal to the symbol value - 4.
1287 if (r.type == R_X86_64_PLT32) {
1288 if (r.sym->isSection())
1289 return r.addend + 4;
1290 return 0;
1291 }
1292 return std::nullopt;
1293}
1294
1295static std::pair<Relocation *, uint64_t>
1296getBranchInfoAtTarget(InputSection &is, uint64_t offset) {
1297 auto content = is.contentMaybeDecompress();
1298 if (content.size() > offset && content[offset] == 0xe9) { // JMP immediate
1299 auto *i = llvm::partition_point(
1300 Range&: is.relocations, P: [&](Relocation &r) { return r.offset < offset + 1; });
1301 // Unlike with getControlTransferAddend() it is valid to accept a PC32
1302 // relocation here because we know that this is actually a JMP and not some
1303 // other reference, so the interpretation is that we add 4 to the addend and
1304 // use that as the effective addend.
1305 if (i != is.relocations.end() && i->offset == offset + 1 &&
1306 (i->type == R_X86_64_PC32 || i->type == R_X86_64_PLT32)) {
1307 return {i, i->addend + 4};
1308 }
1309 }
1310 return {nullptr, 0};
1311}
1312
1313static void redirectControlTransferRelocations(Relocation &r1,
1314 const Relocation &r2) {
1315 // The isSection() check handles the STT_SECTION case described above.
1316 // In that case the original addend is irrelevant because it referred to an
1317 // offset within the original target section so we overwrite it.
1318 //
1319 // The +4 is here to compensate for r2.addend which will likely be -4,
1320 // but may also be addend-4 in case of a PC32 branch to symbol+addend.
1321 if (r1.sym->isSection())
1322 r1.addend = r2.addend;
1323 else
1324 r1.addend += r2.addend + 4;
1325 r1.expr = r2.expr;
1326 r1.sym = r2.sym;
1327}
1328
1329void X86_64::applyBranchToBranchOpt() const {
1330 applyBranchToBranchOptImpl(ctx, getControlTransferAddend,
1331 getBranchInfoAtTarget,
1332 redirectControlTransferRelocations);
1333}
1334
1335// If Intel Indirect Branch Tracking is enabled, we have to emit special PLT
1336// entries containing endbr64 instructions. A PLT entry will be split into two
1337// parts, one in .plt.sec (writePlt), and the other in .plt (writeIBTPlt).
1338namespace {
1339class IntelIBT : public X86_64 {
1340public:
1341 IntelIBT(Ctx &ctx) : X86_64(ctx) { pltHeaderSize = 0; };
1342 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
1343 void writePlt(uint8_t *buf, const Symbol &sym,
1344 uint64_t pltEntryAddr) const override;
1345 void writeIBTPlt(uint8_t *buf, size_t numEntries) const override;
1346
1347 static const unsigned IBTPltHeaderSize = 16;
1348};
1349} // namespace
1350
1351void IntelIBT::writeGotPlt(uint8_t *buf, const Symbol &s) const {
1352 uint64_t va = ctx.in.ibtPlt->getVA() + IBTPltHeaderSize +
1353 s.getPltIdx(ctx) * pltEntrySize;
1354 write64le(P: buf, V: va);
1355}
1356
1357void IntelIBT::writePlt(uint8_t *buf, const Symbol &sym,
1358 uint64_t pltEntryAddr) const {
1359 const uint8_t Inst[] = {
1360 0xf3, 0x0f, 0x1e, 0xfa, // endbr64
1361 0xff, 0x25, 0, 0, 0, 0, // jmpq *got(%rip)
1362 0x66, 0x0f, 0x1f, 0x44, 0, 0, // nop
1363 };
1364 memcpy(dest: buf, src: Inst, n: sizeof(Inst));
1365 write32le(P: buf + 6, V: sym.getGotPltVA(ctx) - pltEntryAddr - 10);
1366}
1367
1368void IntelIBT::writeIBTPlt(uint8_t *buf, size_t numEntries) const {
1369 writePltHeader(buf);
1370 buf += IBTPltHeaderSize;
1371
1372 const uint8_t inst[] = {
1373 0xf3, 0x0f, 0x1e, 0xfa, // endbr64
1374 0x68, 0, 0, 0, 0, // pushq <relocation index>
1375 0xe9, 0, 0, 0, 0, // jmpq plt[0]
1376 0x66, 0x90, // nop
1377 };
1378
1379 for (size_t i = 0; i < numEntries; ++i) {
1380 memcpy(dest: buf, src: inst, n: sizeof(inst));
1381 write32le(P: buf + 5, V: i);
1382 write32le(P: buf + 10, V: -pltHeaderSize - sizeof(inst) * i - 30);
1383 buf += sizeof(inst);
1384 }
1385}
1386
1387// These nonstandard PLT entries are to migtigate Spectre v2 security
1388// vulnerability. In order to mitigate Spectre v2, we want to avoid indirect
1389// branch instructions such as `jmp *GOTPLT(%rip)`. So, in the following PLT
1390// entries, we use a CALL followed by MOV and RET to do the same thing as an
1391// indirect jump. That instruction sequence is so-called "retpoline".
1392//
1393// We have two types of retpoline PLTs as a size optimization. If `-z now`
1394// is specified, all dynamic symbols are resolved at load-time. Thus, when
1395// that option is given, we can omit code for symbol lazy resolution.
1396namespace {
1397class Retpoline : public X86_64 {
1398public:
1399 Retpoline(Ctx &);
1400 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
1401 void writePltHeader(uint8_t *buf) const override;
1402 void writePlt(uint8_t *buf, const Symbol &sym,
1403 uint64_t pltEntryAddr) const override;
1404};
1405
1406class RetpolineZNow : public X86_64 {
1407public:
1408 RetpolineZNow(Ctx &);
1409 void writeGotPlt(uint8_t *buf, const Symbol &s) const override {}
1410 void writePltHeader(uint8_t *buf) const override;
1411 void writePlt(uint8_t *buf, const Symbol &sym,
1412 uint64_t pltEntryAddr) const override;
1413};
1414} // namespace
1415
1416Retpoline::Retpoline(Ctx &ctx) : X86_64(ctx) {
1417 pltHeaderSize = 48;
1418 pltEntrySize = 32;
1419 ipltEntrySize = 32;
1420}
1421
1422void Retpoline::writeGotPlt(uint8_t *buf, const Symbol &s) const {
1423 write64le(P: buf, V: s.getPltVA(ctx) + 17);
1424}
1425
1426void Retpoline::writePltHeader(uint8_t *buf) const {
1427 const uint8_t insn[] = {
1428 0xff, 0x35, 0, 0, 0, 0, // 0: pushq GOTPLT+8(%rip)
1429 0x4c, 0x8b, 0x1d, 0, 0, 0, 0, // 6: mov GOTPLT+16(%rip), %r11
1430 0xe8, 0x0e, 0x00, 0x00, 0x00, // d: callq next
1431 0xf3, 0x90, // 12: loop: pause
1432 0x0f, 0xae, 0xe8, // 14: lfence
1433 0xeb, 0xf9, // 17: jmp loop
1434 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 19: int3; .align 16
1435 0x4c, 0x89, 0x1c, 0x24, // 20: next: mov %r11, (%rsp)
1436 0xc3, // 24: ret
1437 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 25: int3; padding
1438 0xcc, 0xcc, 0xcc, 0xcc, // 2c: int3; padding
1439 };
1440 memcpy(dest: buf, src: insn, n: sizeof(insn));
1441
1442 uint64_t gotPlt = ctx.in.gotPlt->getVA();
1443 uint64_t plt = ctx.in.plt->getVA();
1444 write32le(P: buf + 2, V: gotPlt - plt - 6 + 8);
1445 write32le(P: buf + 9, V: gotPlt - plt - 13 + 16);
1446}
1447
1448void Retpoline::writePlt(uint8_t *buf, const Symbol &sym,
1449 uint64_t pltEntryAddr) const {
1450 const uint8_t insn[] = {
1451 0x4c, 0x8b, 0x1d, 0, 0, 0, 0, // 0: mov foo@GOTPLT(%rip), %r11
1452 0xe8, 0, 0, 0, 0, // 7: callq plt+0x20
1453 0xe9, 0, 0, 0, 0, // c: jmp plt+0x12
1454 0x68, 0, 0, 0, 0, // 11: pushq <relocation index>
1455 0xe9, 0, 0, 0, 0, // 16: jmp plt+0
1456 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 1b: int3; padding
1457 };
1458 memcpy(dest: buf, src: insn, n: sizeof(insn));
1459
1460 uint64_t off = pltEntryAddr - ctx.in.plt->getVA();
1461
1462 write32le(P: buf + 3, V: sym.getGotPltVA(ctx) - pltEntryAddr - 7);
1463 write32le(P: buf + 8, V: -off - 12 + 32);
1464 write32le(P: buf + 13, V: -off - 17 + 18);
1465 write32le(P: buf + 18, V: sym.getPltIdx(ctx));
1466 write32le(P: buf + 23, V: -off - 27);
1467}
1468
1469RetpolineZNow::RetpolineZNow(Ctx &ctx) : X86_64(ctx) {
1470 pltHeaderSize = 32;
1471 pltEntrySize = 16;
1472 ipltEntrySize = 16;
1473}
1474
1475void RetpolineZNow::writePltHeader(uint8_t *buf) const {
1476 const uint8_t insn[] = {
1477 0xe8, 0x0b, 0x00, 0x00, 0x00, // 0: call next
1478 0xf3, 0x90, // 5: loop: pause
1479 0x0f, 0xae, 0xe8, // 7: lfence
1480 0xeb, 0xf9, // a: jmp loop
1481 0xcc, 0xcc, 0xcc, 0xcc, // c: int3; .align 16
1482 0x4c, 0x89, 0x1c, 0x24, // 10: next: mov %r11, (%rsp)
1483 0xc3, // 14: ret
1484 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 15: int3; padding
1485 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, // 1a: int3; padding
1486 0xcc, // 1f: int3; padding
1487 };
1488 memcpy(dest: buf, src: insn, n: sizeof(insn));
1489}
1490
1491void RetpolineZNow::writePlt(uint8_t *buf, const Symbol &sym,
1492 uint64_t pltEntryAddr) const {
1493 const uint8_t insn[] = {
1494 0x4c, 0x8b, 0x1d, 0, 0, 0, 0, // mov foo@GOTPLT(%rip), %r11
1495 0xe9, 0, 0, 0, 0, // jmp plt+0
1496 0xcc, 0xcc, 0xcc, 0xcc, // int3; padding
1497 };
1498 memcpy(dest: buf, src: insn, n: sizeof(insn));
1499
1500 write32le(P: buf + 3, V: sym.getGotPltVA(ctx) - pltEntryAddr - 7);
1501 write32le(P: buf + 8, V: ctx.in.plt->getVA() - pltEntryAddr - 12);
1502}
1503
1504void elf::setX86_64TargetInfo(Ctx &ctx) {
1505 if (ctx.arg.zRetpolineplt) {
1506 if (ctx.arg.zNow)
1507 ctx.target.reset(p: new RetpolineZNow(ctx));
1508 else
1509 ctx.target.reset(p: new Retpoline(ctx));
1510 return;
1511 }
1512
1513 if (ctx.arg.andFeatures & GNU_PROPERTY_X86_FEATURE_1_IBT)
1514 ctx.target.reset(p: new IntelIBT(ctx));
1515 else
1516 ctx.target.reset(p: new X86_64(ctx));
1517}
1518