1//===- RISCV.cpp ----------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "InputFiles.h"
10#include "OutputSections.h"
11#include "RelocScan.h"
12#include "Symbols.h"
13#include "SyntheticSections.h"
14#include "Target.h"
15#include "llvm/Support/ELFAttributes.h"
16#include "llvm/Support/LEB128.h"
17#include "llvm/Support/RISCVAttributeParser.h"
18#include "llvm/Support/RISCVAttributes.h"
19#include "llvm/Support/TimeProfiler.h"
20#include "llvm/TargetParser/RISCVISAInfo.h"
21
22using namespace llvm;
23using namespace llvm::object;
24using namespace llvm::support::endian;
25using namespace llvm::ELF;
26using namespace lld;
27using namespace lld::elf;
28
29namespace {
30
31class RISCV final : public TargetInfo {
32public:
33 RISCV(Ctx &);
34 uint32_t calcEFlags() const override;
35 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;
36 void writeGotHeader(uint8_t *buf) const override;
37 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
38 void writeIgotPlt(uint8_t *buf, const Symbol &s) const override;
39 void writePltHeader(uint8_t *buf) const override;
40 void writePlt(uint8_t *buf, const Symbol &sym,
41 uint64_t pltEntryAddr) const override;
42 template <class ELFT, class RelTy>
43 void scanSectionImpl(InputSectionBase &, Relocs<RelTy>);
44 void scanSection(InputSectionBase &) override;
45 RelType getDynRel(RelType type) const override;
46 RelExpr getRelExpr(RelType type, const Symbol &s,
47 const uint8_t *loc) const override;
48 void relocate(uint8_t *loc, const Relocation &rel,
49 uint64_t val) const override;
50 void relocateAlloc(InputSection &sec, uint8_t *buf) const override;
51 bool relaxOnce(int pass) const override;
52 template <class ELFT, class RelTy>
53 bool synthesizeAlignForInput(uint64_t &dot, InputSection *sec,
54 Relocs<RelTy> rels);
55 template <class ELFT, class RelTy>
56 void finalizeSynthesizeAligns(uint64_t &dot, InputSection *sec,
57 Relocs<RelTy> rels);
58 template <class ELFT>
59 bool synthesizeAlignAux(uint64_t &dot, InputSection *sec);
60 bool synthesizeAlign(uint64_t &dot, InputSection *sec) override;
61 void finalizeRelax(int passes) const override;
62
63 // The following two variables are used by synthesized ALIGN relocations.
64 InputSection *baseSec = nullptr;
65 // r_offset and r_addend pairs.
66 SmallVector<std::pair<uint64_t, uint64_t>, 0> synthesizedAligns;
67};
68
69} // end anonymous namespace
70
71// These are internal relocation numbers for GP/X0 relaxation. They aren't part
72// of the psABI spec.
73#define INTERNAL_R_RISCV_GPREL_I 256
74#define INTERNAL_R_RISCV_GPREL_S 257
75#define INTERNAL_R_RISCV_X0REL_I 258
76#define INTERNAL_R_RISCV_X0REL_S 259
77
78const uint64_t dtpOffset = 0x800;
79
80namespace {
81enum Op {
82 ADDI = 0x13,
83 AUIPC = 0x17,
84 JALR = 0x67,
85 LD = 0x3003,
86 LUI = 0x37,
87 LW = 0x2003,
88 SRLI = 0x5013,
89 SUB = 0x40000033,
90};
91
92enum Reg {
93 X_X0 = 0,
94 X_RA = 1,
95 X_GP = 3,
96 X_TP = 4,
97 X_T0 = 5,
98 X_T1 = 6,
99 X_T2 = 7,
100 X_A0 = 10,
101 X_T3 = 28,
102};
103} // namespace
104
105static uint32_t hi20(uint32_t val) { return (val + 0x800) >> 12; }
106static uint32_t lo12(uint32_t val) { return val & 4095; }
107
108static uint32_t itype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t imm) {
109 return op | (rd << 7) | (rs1 << 15) | (imm << 20);
110}
111static uint32_t rtype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t rs2) {
112 return op | (rd << 7) | (rs1 << 15) | (rs2 << 20);
113}
114static uint32_t utype(uint32_t op, uint32_t rd, uint32_t imm) {
115 return op | (rd << 7) | (imm << 12);
116}
117
118// Extract bits v[begin:end], where range is inclusive, and begin must be < 63.
119static uint32_t extractBits(uint64_t v, uint32_t begin, uint32_t end) {
120 return (v & ((1ULL << (begin + 1)) - 1)) >> end;
121}
122
123static uint32_t setLO12_I(uint32_t insn, uint32_t imm) {
124 return (insn & 0xfffff) | (imm << 20);
125}
126static uint32_t setLO12_S(uint32_t insn, uint32_t imm) {
127 return (insn & 0x1fff07f) | (extractBits(v: imm, begin: 11, end: 5) << 25) |
128 (extractBits(v: imm, begin: 4, end: 0) << 7);
129}
130
131RISCV::RISCV(Ctx &ctx) : TargetInfo(ctx) {
132 copyRel = R_RISCV_COPY;
133 pltRel = R_RISCV_JUMP_SLOT;
134 relativeRel = R_RISCV_RELATIVE;
135 iRelativeRel = R_RISCV_IRELATIVE;
136 if (ctx.arg.is64) {
137 symbolicRel = R_RISCV_64;
138 tlsModuleIndexRel = R_RISCV_TLS_DTPMOD64;
139 tlsOffsetRel = R_RISCV_TLS_DTPREL64;
140 tlsGotRel = R_RISCV_TLS_TPREL64;
141 } else {
142 symbolicRel = R_RISCV_32;
143 tlsModuleIndexRel = R_RISCV_TLS_DTPMOD32;
144 tlsOffsetRel = R_RISCV_TLS_DTPREL32;
145 tlsGotRel = R_RISCV_TLS_TPREL32;
146 }
147 gotRel = symbolicRel;
148 tlsDescRel = R_RISCV_TLSDESC;
149
150 // .got[0] = _DYNAMIC
151 gotHeaderEntriesNum = 1;
152
153 // .got.plt[0] = _dl_runtime_resolve, .got.plt[1] = link_map
154 gotPltHeaderEntriesNum = 2;
155
156 pltHeaderSize = 32;
157 pltEntrySize = 16;
158 ipltEntrySize = 16;
159}
160
161static uint32_t getEFlags(Ctx &ctx, InputFile *f) {
162 if (ctx.arg.is64)
163 return cast<ObjFile<ELF64LE>>(Val: f)->getObj().getHeader().e_flags;
164 return cast<ObjFile<ELF32LE>>(Val: f)->getObj().getHeader().e_flags;
165}
166
167uint32_t RISCV::calcEFlags() const {
168 // If there are only binary input files (from -b binary), use a
169 // value of 0 for the ELF header flags.
170 if (ctx.objectFiles.empty())
171 return 0;
172
173 uint32_t target = getEFlags(ctx, f: ctx.objectFiles.front());
174 for (InputFile *f : ctx.objectFiles) {
175 uint32_t eflags = getEFlags(ctx, f);
176 if (eflags & EF_RISCV_RVC)
177 target |= EF_RISCV_RVC;
178
179 if ((eflags & EF_RISCV_FLOAT_ABI) != (target & EF_RISCV_FLOAT_ABI))
180 Err(ctx) << f
181 << ": cannot link object files with different "
182 "floating-point ABI from "
183 << ctx.objectFiles[0];
184
185 if ((eflags & EF_RISCV_RVE) != (target & EF_RISCV_RVE))
186 Err(ctx) << f << ": cannot link object files with different EF_RISCV_RVE";
187 }
188
189 return target;
190}
191
192int64_t RISCV::getImplicitAddend(const uint8_t *buf, RelType type) const {
193 switch (type) {
194 default:
195 InternalErr(ctx, buf) << "cannot read addend for relocation " << type;
196 return 0;
197 case R_RISCV_32:
198 case R_RISCV_TLS_DTPMOD32:
199 case R_RISCV_TLS_DTPREL32:
200 case R_RISCV_TLS_TPREL32:
201 return SignExtend64<32>(x: read32le(P: buf));
202 case R_RISCV_64:
203 case R_RISCV_TLS_DTPMOD64:
204 case R_RISCV_TLS_DTPREL64:
205 case R_RISCV_TLS_TPREL64:
206 return read64le(P: buf);
207 case R_RISCV_RELATIVE:
208 case R_RISCV_IRELATIVE:
209 return ctx.arg.is64 ? read64le(P: buf) : read32le(P: buf);
210 case R_RISCV_NONE:
211 case R_RISCV_JUMP_SLOT:
212 // These relocations are defined as not having an implicit addend.
213 return 0;
214 case R_RISCV_TLSDESC:
215 return ctx.arg.is64 ? read64le(P: buf + 8) : read32le(P: buf + 4);
216 }
217}
218
219void RISCV::writeGotHeader(uint8_t *buf) const {
220 if (ctx.arg.is64)
221 write64le(P: buf, V: ctx.mainPart->dynamic->getVA());
222 else
223 write32le(P: buf, V: ctx.mainPart->dynamic->getVA());
224}
225
226void RISCV::writeGotPlt(uint8_t *buf, const Symbol &s) const {
227 if (ctx.arg.is64)
228 write64le(P: buf, V: ctx.in.plt->getVA());
229 else
230 write32le(P: buf, V: ctx.in.plt->getVA());
231}
232
233void RISCV::writeIgotPlt(uint8_t *buf, const Symbol &s) const {
234 if (ctx.arg.writeAddends) {
235 if (ctx.arg.is64)
236 write64le(P: buf, V: s.getVA(ctx));
237 else
238 write32le(P: buf, V: s.getVA(ctx));
239 }
240}
241
242void RISCV::writePltHeader(uint8_t *buf) const {
243 // 1: auipc t2, %pcrel_hi(.got.plt)
244 // sub t1, t1, t3
245 // l[wd] t3, %pcrel_lo(1b)(t2); t3 = _dl_runtime_resolve
246 // addi t1, t1, -pltHeaderSize-12; t1 = &.plt[i] - &.plt[0]
247 // addi t0, t2, %pcrel_lo(1b)
248 // srli t1, t1, (rv64?1:2); t1 = &.got.plt[i] - &.got.plt[0]
249 // l[wd] t0, Wordsize(t0); t0 = link_map
250 // jr t3
251 uint32_t offset = ctx.in.gotPlt->getVA() - ctx.in.plt->getVA();
252 uint32_t load = ctx.arg.is64 ? LD : LW;
253 write32le(P: buf + 0, V: utype(op: AUIPC, rd: X_T2, imm: hi20(val: offset)));
254 write32le(P: buf + 4, V: rtype(op: SUB, rd: X_T1, rs1: X_T1, rs2: X_T3));
255 write32le(P: buf + 8, V: itype(op: load, rd: X_T3, rs1: X_T2, imm: lo12(val: offset)));
256 write32le(P: buf + 12, V: itype(op: ADDI, rd: X_T1, rs1: X_T1, imm: -ctx.target->pltHeaderSize - 12));
257 write32le(P: buf + 16, V: itype(op: ADDI, rd: X_T0, rs1: X_T2, imm: lo12(val: offset)));
258 write32le(P: buf + 20, V: itype(op: SRLI, rd: X_T1, rs1: X_T1, imm: ctx.arg.is64 ? 1 : 2));
259 write32le(P: buf + 24, V: itype(op: load, rd: X_T0, rs1: X_T0, imm: ctx.arg.wordsize));
260 write32le(P: buf + 28, V: itype(op: JALR, rd: 0, rs1: X_T3, imm: 0));
261}
262
263void RISCV::writePlt(uint8_t *buf, const Symbol &sym,
264 uint64_t pltEntryAddr) const {
265 // 1: auipc t3, %pcrel_hi(f@.got.plt)
266 // l[wd] t3, %pcrel_lo(1b)(t3)
267 // jalr t1, t3
268 // nop
269 uint32_t offset = sym.getGotPltVA(ctx) - pltEntryAddr;
270 write32le(P: buf + 0, V: utype(op: AUIPC, rd: X_T3, imm: hi20(val: offset)));
271 write32le(P: buf + 4, V: itype(op: ctx.arg.is64 ? LD : LW, rd: X_T3, rs1: X_T3, imm: lo12(val: offset)));
272 write32le(P: buf + 8, V: itype(op: JALR, rd: X_T1, rs1: X_T3, imm: 0));
273 write32le(P: buf + 12, V: itype(op: ADDI, rd: 0, rs1: 0, imm: 0));
274}
275
276RelType RISCV::getDynRel(RelType type) const {
277 return type == ctx.target->symbolicRel ? type
278 : static_cast<RelType>(R_RISCV_NONE);
279}
280
281RelExpr RISCV::getRelExpr(const RelType type, const Symbol &s,
282 const uint8_t *loc) const {
283 switch (type) {
284 case R_RISCV_NONE:
285 case R_RISCV_VENDOR:
286 return R_NONE;
287 case R_RISCV_32:
288 case R_RISCV_64:
289 case R_RISCV_HI20:
290 case R_RISCV_LO12_I:
291 case R_RISCV_LO12_S:
292 return R_ABS;
293 case R_RISCV_ADD8:
294 case R_RISCV_ADD16:
295 case R_RISCV_ADD32:
296 case R_RISCV_ADD64:
297 case R_RISCV_SET6:
298 case R_RISCV_SET8:
299 case R_RISCV_SET16:
300 case R_RISCV_SET32:
301 case R_RISCV_SUB6:
302 case R_RISCV_SUB8:
303 case R_RISCV_SUB16:
304 case R_RISCV_SUB32:
305 case R_RISCV_SUB64:
306 return RE_RISCV_ADD;
307 case R_RISCV_JAL:
308 case R_RISCV_BRANCH:
309 case R_RISCV_PCREL_HI20:
310 case R_RISCV_RVC_BRANCH:
311 case R_RISCV_RVC_JUMP:
312 case R_RISCV_32_PCREL:
313 return R_PC;
314 case R_RISCV_CALL:
315 case R_RISCV_CALL_PLT:
316 case R_RISCV_PLT32:
317 return R_PLT_PC;
318 case R_RISCV_GOT_HI20:
319 case R_RISCV_GOT32_PCREL:
320 return R_GOT_PC;
321 case R_RISCV_PCREL_LO12_I:
322 case R_RISCV_PCREL_LO12_S:
323 return RE_RISCV_PC_INDIRECT;
324 case R_RISCV_TLSDESC_HI20:
325 case R_RISCV_TLSDESC_LOAD_LO12:
326 case R_RISCV_TLSDESC_ADD_LO12:
327 return R_TLSDESC_PC;
328 case R_RISCV_TLSDESC_CALL:
329 return R_TLSDESC_CALL;
330 case R_RISCV_TLS_GD_HI20:
331 return R_TLSGD_PC;
332 case R_RISCV_TLS_GOT_HI20:
333 return R_GOT_PC;
334 case R_RISCV_TPREL_HI20:
335 case R_RISCV_TPREL_LO12_I:
336 case R_RISCV_TPREL_LO12_S:
337 return R_TPREL;
338 case R_RISCV_ALIGN:
339 return R_RELAX_HINT;
340 case R_RISCV_TPREL_ADD:
341 case R_RISCV_RELAX:
342 return ctx.arg.relax ? R_RELAX_HINT : R_NONE;
343 case R_RISCV_SET_ULEB128:
344 case R_RISCV_SUB_ULEB128:
345 return RE_RISCV_LEB128;
346 default:
347 Err(ctx) << getErrorLoc(ctx, loc) << "unknown relocation (" << type.v
348 << ") against symbol " << &s;
349 return R_NONE;
350 }
351}
352
353template <class ELFT, class RelTy>
354void RISCV::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
355 RelocScan rs(ctx, &sec);
356 // Many relocations end up in sec.relocations.
357 sec.relocations.reserve(N: rels.size());
358
359 StringRef rvVendor;
360 for (auto it = rels.begin(); it != rels.end(); ++it) {
361 RelType type = it->getType(false);
362 uint32_t symIndex = it->getSymbol(false);
363 Symbol &sym = sec.getFile<ELFT>()->getSymbol(symIndex);
364 const uint8_t *loc = sec.content().data() + it->r_offset;
365
366 if (type == R_RISCV_VENDOR) {
367 if (!rvVendor.empty())
368 Err(ctx) << getErrorLoc(ctx, loc)
369 << "malformed consecutive R_RISCV_VENDOR relocations";
370 rvVendor = sym.getName();
371 continue;
372 } else if (!rvVendor.empty()) {
373 Err(ctx) << getErrorLoc(ctx, loc)
374 << "unknown vendor-specific relocation (" << type.v
375 << ") in namespace '" << rvVendor << "' against symbol '" << &sym
376 << "'";
377 rvVendor = "";
378 continue;
379 }
380
381 rs.scan<ELFT, RelTy>(it, type, rs.getAddend<ELFT>(*it, type));
382 }
383
384 // Sort relocations by offset for more efficient searching for
385 // R_RISCV_PCREL_HI20.
386 llvm::stable_sort(sec.relocs(),
387 [](const Relocation &lhs, const Relocation &rhs) {
388 return lhs.offset < rhs.offset;
389 });
390}
391
392void RISCV::scanSection(InputSectionBase &sec) {
393 if (ctx.arg.is64)
394 elf::scanSection1<RISCV, ELF64LE>(target&: *this, sec);
395 else
396 elf::scanSection1<RISCV, ELF32LE>(target&: *this, sec);
397}
398
399void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
400 const unsigned bits = ctx.arg.wordsize * 8;
401
402 switch (rel.type) {
403 case R_RISCV_32:
404 write32le(P: loc, V: val);
405 return;
406 case R_RISCV_64:
407 write64le(P: loc, V: val);
408 return;
409
410 case R_RISCV_RVC_BRANCH: {
411 checkInt(ctx, loc, v: val, n: 9, rel);
412 checkAlignment(ctx, loc, v: val, n: 2, rel);
413 uint16_t insn = read16le(P: loc) & 0xE383;
414 uint16_t imm8 = extractBits(v: val, begin: 8, end: 8) << 12;
415 uint16_t imm4_3 = extractBits(v: val, begin: 4, end: 3) << 10;
416 uint16_t imm7_6 = extractBits(v: val, begin: 7, end: 6) << 5;
417 uint16_t imm2_1 = extractBits(v: val, begin: 2, end: 1) << 3;
418 uint16_t imm5 = extractBits(v: val, begin: 5, end: 5) << 2;
419 insn |= imm8 | imm4_3 | imm7_6 | imm2_1 | imm5;
420
421 write16le(P: loc, V: insn);
422 return;
423 }
424
425 case R_RISCV_RVC_JUMP: {
426 checkInt(ctx, loc, v: val, n: 12, rel);
427 checkAlignment(ctx, loc, v: val, n: 2, rel);
428 uint16_t insn = read16le(P: loc) & 0xE003;
429 uint16_t imm11 = extractBits(v: val, begin: 11, end: 11) << 12;
430 uint16_t imm4 = extractBits(v: val, begin: 4, end: 4) << 11;
431 uint16_t imm9_8 = extractBits(v: val, begin: 9, end: 8) << 9;
432 uint16_t imm10 = extractBits(v: val, begin: 10, end: 10) << 8;
433 uint16_t imm6 = extractBits(v: val, begin: 6, end: 6) << 7;
434 uint16_t imm7 = extractBits(v: val, begin: 7, end: 7) << 6;
435 uint16_t imm3_1 = extractBits(v: val, begin: 3, end: 1) << 3;
436 uint16_t imm5 = extractBits(v: val, begin: 5, end: 5) << 2;
437 insn |= imm11 | imm4 | imm9_8 | imm10 | imm6 | imm7 | imm3_1 | imm5;
438
439 write16le(P: loc, V: insn);
440 return;
441 }
442
443 case R_RISCV_JAL: {
444 checkInt(ctx, loc, v: val, n: 21, rel);
445 checkAlignment(ctx, loc, v: val, n: 2, rel);
446
447 uint32_t insn = read32le(P: loc) & 0xFFF;
448 uint32_t imm20 = extractBits(v: val, begin: 20, end: 20) << 31;
449 uint32_t imm10_1 = extractBits(v: val, begin: 10, end: 1) << 21;
450 uint32_t imm11 = extractBits(v: val, begin: 11, end: 11) << 20;
451 uint32_t imm19_12 = extractBits(v: val, begin: 19, end: 12) << 12;
452 insn |= imm20 | imm10_1 | imm11 | imm19_12;
453
454 write32le(P: loc, V: insn);
455 return;
456 }
457
458 case R_RISCV_BRANCH: {
459 checkInt(ctx, loc, v: val, n: 13, rel);
460 checkAlignment(ctx, loc, v: val, n: 2, rel);
461
462 uint32_t insn = read32le(P: loc) & 0x1FFF07F;
463 uint32_t imm12 = extractBits(v: val, begin: 12, end: 12) << 31;
464 uint32_t imm10_5 = extractBits(v: val, begin: 10, end: 5) << 25;
465 uint32_t imm4_1 = extractBits(v: val, begin: 4, end: 1) << 8;
466 uint32_t imm11 = extractBits(v: val, begin: 11, end: 11) << 7;
467 insn |= imm12 | imm10_5 | imm4_1 | imm11;
468
469 write32le(P: loc, V: insn);
470 return;
471 }
472
473 // auipc + jalr pair
474 case R_RISCV_CALL:
475 case R_RISCV_CALL_PLT: {
476 int64_t hi = SignExtend64(X: val + 0x800, B: bits) >> 12;
477 checkInt(ctx, loc, v: hi, n: 20, rel);
478 if (isInt<20>(x: hi)) {
479 relocateNoSym(loc, type: R_RISCV_PCREL_HI20, val);
480 relocateNoSym(loc: loc + 4, type: R_RISCV_PCREL_LO12_I, val);
481 }
482 return;
483 }
484
485 case R_RISCV_GOT_HI20:
486 case R_RISCV_PCREL_HI20:
487 case R_RISCV_TLSDESC_HI20:
488 case R_RISCV_TLS_GD_HI20:
489 case R_RISCV_TLS_GOT_HI20:
490 case R_RISCV_TPREL_HI20:
491 case R_RISCV_HI20: {
492 uint64_t hi = val + 0x800;
493 checkInt(ctx, loc, v: SignExtend64(X: hi, B: bits) >> 12, n: 20, rel);
494 write32le(P: loc, V: (read32le(P: loc) & 0xFFF) | (hi & 0xFFFFF000));
495 return;
496 }
497
498 case R_RISCV_PCREL_LO12_I:
499 case R_RISCV_TLSDESC_LOAD_LO12:
500 case R_RISCV_TLSDESC_ADD_LO12:
501 case R_RISCV_TPREL_LO12_I:
502 case R_RISCV_LO12_I: {
503 uint64_t hi = (val + 0x800) >> 12;
504 uint64_t lo = val - (hi << 12);
505 write32le(P: loc, V: setLO12_I(insn: read32le(P: loc), imm: lo & 0xfff));
506 return;
507 }
508
509 case R_RISCV_PCREL_LO12_S:
510 case R_RISCV_TPREL_LO12_S:
511 case R_RISCV_LO12_S: {
512 uint64_t hi = (val + 0x800) >> 12;
513 uint64_t lo = val - (hi << 12);
514 write32le(P: loc, V: setLO12_S(insn: read32le(P: loc), imm: lo));
515 return;
516 }
517
518 case INTERNAL_R_RISCV_X0REL_I:
519 case INTERNAL_R_RISCV_X0REL_S: {
520 checkInt(ctx, loc, v: val, n: 12, rel);
521 uint32_t insn = (read32le(P: loc) & ~(31 << 15)) | (X_X0 << 15);
522 if (rel.type == INTERNAL_R_RISCV_X0REL_I)
523 insn = setLO12_I(insn, imm: val);
524 else
525 insn = setLO12_S(insn, imm: val);
526 write32le(P: loc, V: insn);
527 return;
528 }
529
530 case INTERNAL_R_RISCV_GPREL_I:
531 case INTERNAL_R_RISCV_GPREL_S: {
532 Defined *gp = ctx.sym.riscvGlobalPointer;
533 int64_t displace = SignExtend64(X: val - gp->getVA(ctx), B: bits);
534 checkInt(ctx, loc, v: displace, n: 12, rel);
535 uint32_t insn = (read32le(P: loc) & ~(31 << 15)) | (X_GP << 15);
536 if (rel.type == INTERNAL_R_RISCV_GPREL_I)
537 insn = setLO12_I(insn, imm: displace);
538 else
539 insn = setLO12_S(insn, imm: displace);
540 write32le(P: loc, V: insn);
541 return;
542 }
543
544 case R_RISCV_ADD8:
545 *loc += val;
546 return;
547 case R_RISCV_ADD16:
548 write16le(P: loc, V: read16le(P: loc) + val);
549 return;
550 case R_RISCV_ADD32:
551 write32le(P: loc, V: read32le(P: loc) + val);
552 return;
553 case R_RISCV_ADD64:
554 write64le(P: loc, V: read64le(P: loc) + val);
555 return;
556 case R_RISCV_SUB6:
557 *loc = (*loc & 0xc0) | (((*loc & 0x3f) - val) & 0x3f);
558 return;
559 case R_RISCV_SUB8:
560 *loc -= val;
561 return;
562 case R_RISCV_SUB16:
563 write16le(P: loc, V: read16le(P: loc) - val);
564 return;
565 case R_RISCV_SUB32:
566 write32le(P: loc, V: read32le(P: loc) - val);
567 return;
568 case R_RISCV_SUB64:
569 write64le(P: loc, V: read64le(P: loc) - val);
570 return;
571 case R_RISCV_SET6:
572 *loc = (*loc & 0xc0) | (val & 0x3f);
573 return;
574 case R_RISCV_SET8:
575 *loc = val;
576 return;
577 case R_RISCV_SET16:
578 write16le(P: loc, V: val);
579 return;
580 case R_RISCV_SET32:
581 case R_RISCV_32_PCREL:
582 case R_RISCV_PLT32:
583 case R_RISCV_GOT32_PCREL:
584 checkInt(ctx, loc, v: val, n: 32, rel);
585 write32le(P: loc, V: val);
586 return;
587
588 case R_RISCV_TLS_DTPREL32:
589 write32le(P: loc, V: val - dtpOffset);
590 break;
591 case R_RISCV_TLS_DTPREL64:
592 write64le(P: loc, V: val - dtpOffset);
593 break;
594
595 case R_RISCV_RELAX:
596 return;
597 case R_RISCV_TLSDESC:
598 // The addend is stored in the second word.
599 if (ctx.arg.is64)
600 write64le(P: loc + 8, V: val);
601 else
602 write32le(P: loc + 4, V: val);
603 break;
604 default:
605 llvm_unreachable("unknown relocation");
606 }
607}
608
609static bool relaxable(ArrayRef<Relocation> relocs, size_t i) {
610 return i + 1 != relocs.size() && relocs[i + 1].type == R_RISCV_RELAX;
611}
612
613static void tlsdescToIe(Ctx &ctx, uint8_t *loc, const Relocation &rel,
614 uint64_t val) {
615 switch (rel.type) {
616 case R_RISCV_TLSDESC_HI20:
617 case R_RISCV_TLSDESC_LOAD_LO12:
618 write32le(P: loc, V: 0x00000013); // nop
619 break;
620 case R_RISCV_TLSDESC_ADD_LO12:
621 write32le(P: loc, V: utype(op: AUIPC, rd: X_A0, imm: hi20(val))); // auipc a0,<hi20>
622 break;
623 case R_RISCV_TLSDESC_CALL:
624 if (ctx.arg.is64)
625 write32le(P: loc, V: itype(op: LD, rd: X_A0, rs1: X_A0, imm: lo12(val))); // ld a0,<lo12>(a0)
626 else
627 write32le(P: loc, V: itype(op: LW, rd: X_A0, rs1: X_A0, imm: lo12(val))); // lw a0,<lo12>(a0)
628 break;
629 default:
630 llvm_unreachable("unsupported relocation for TLSDESC to IE");
631 }
632}
633
634static void tlsdescToLe(uint8_t *loc, const Relocation &rel, uint64_t val) {
635 switch (rel.type) {
636 case R_RISCV_TLSDESC_HI20:
637 case R_RISCV_TLSDESC_LOAD_LO12:
638 write32le(P: loc, V: 0x00000013); // nop
639 return;
640 case R_RISCV_TLSDESC_ADD_LO12:
641 if (isInt<12>(x: val))
642 write32le(P: loc, V: 0x00000013); // nop
643 else
644 write32le(P: loc, V: utype(op: LUI, rd: X_A0, imm: hi20(val))); // lui a0,<hi20>
645 return;
646 case R_RISCV_TLSDESC_CALL:
647 if (isInt<12>(x: val))
648 write32le(P: loc, V: itype(op: ADDI, rd: X_A0, rs1: 0, imm: val)); // addi a0,zero,<lo12>
649 else
650 write32le(P: loc, V: itype(op: ADDI, rd: X_A0, rs1: X_A0, imm: lo12(val))); // addi a0,a0,<lo12>
651 return;
652 default:
653 llvm_unreachable("unsupported relocation for TLSDESC to LE");
654 }
655}
656
657void RISCV::relocateAlloc(InputSection &sec, uint8_t *buf) const {
658 uint64_t secAddr = sec.getOutputSection()->addr + sec.outSecOff;
659 uint64_t tlsdescVal = 0;
660 bool tlsdescRelax = false, isToLe = false;
661 const ArrayRef<Relocation> relocs = sec.relocs();
662 for (size_t i = 0, size = relocs.size(); i != size; ++i) {
663 const Relocation &rel = relocs[i];
664 uint8_t *loc = buf + rel.offset;
665 uint64_t val = sec.getRelocTargetVA(ctx, r: rel, p: secAddr + rel.offset);
666
667 switch (rel.expr) {
668 case R_RELAX_HINT:
669 continue;
670 case R_TLSDESC_PC:
671 // For R_RISCV_TLSDESC_HI20, store &got(sym)-PC to be used by the
672 // following two instructions L[DW] and ADDI.
673 if (rel.type == R_RISCV_TLSDESC_HI20)
674 tlsdescVal = val;
675 else
676 val = tlsdescVal;
677 break;
678 case R_RELAX_TLS_GD_TO_IE:
679 // Only R_RISCV_TLSDESC_HI20 reaches here. tlsdescVal will be finalized
680 // after we see R_RISCV_TLSDESC_ADD_LO12 in the R_RELAX_TLS_GD_TO_LE case.
681 // The net effect is that tlsdescVal will be smaller than `val` to take
682 // into account of NOP instructions (in the absence of R_RISCV_RELAX)
683 // before AUIPC.
684 tlsdescVal = val + rel.offset;
685 isToLe = false;
686 tlsdescRelax = relaxable(relocs, i);
687 if (!tlsdescRelax)
688 tlsdescToIe(ctx, loc, rel, val);
689 continue;
690 case R_RELAX_TLS_GD_TO_LE:
691 // See the comment in handleTlsRelocation. For TLSDESC=>IE,
692 // R_RISCV_TLSDESC_{LOAD_LO12,ADD_LO12,CALL} also reach here. If isToLe is
693 // false, this is actually TLSDESC=>IE optimization.
694 if (rel.type == R_RISCV_TLSDESC_HI20) {
695 tlsdescVal = val;
696 isToLe = true;
697 tlsdescRelax = relaxable(relocs, i);
698 } else {
699 if (!isToLe && rel.type == R_RISCV_TLSDESC_ADD_LO12)
700 tlsdescVal -= rel.offset;
701 val = tlsdescVal;
702 }
703 // When NOP conversion is eligible and relaxation applies, don't write a
704 // NOP in case an unrelated instruction follows the current instruction.
705 if (tlsdescRelax &&
706 (rel.type == R_RISCV_TLSDESC_HI20 ||
707 rel.type == R_RISCV_TLSDESC_LOAD_LO12 ||
708 (rel.type == R_RISCV_TLSDESC_ADD_LO12 && isToLe && !hi20(val))))
709 continue;
710 if (isToLe)
711 tlsdescToLe(loc, rel, val);
712 else
713 tlsdescToIe(ctx, loc, rel, val);
714 continue;
715 case RE_RISCV_LEB128:
716 if (i + 1 < size) {
717 const Relocation &rel1 = relocs[i + 1];
718 if (rel.type == R_RISCV_SET_ULEB128 &&
719 rel1.type == R_RISCV_SUB_ULEB128 && rel.offset == rel1.offset) {
720 auto val = rel.sym->getVA(ctx, addend: rel.addend) -
721 rel1.sym->getVA(ctx, addend: rel1.addend);
722 if (overwriteULEB128(bufLoc: loc, val) >= 0x80)
723 Err(ctx) << sec.getLocation(offset: rel.offset) << ": ULEB128 value " << val
724 << " exceeds available space; references '" << rel.sym
725 << "'";
726 ++i;
727 continue;
728 }
729 }
730 Err(ctx) << sec.getLocation(offset: rel.offset)
731 << ": R_RISCV_SET_ULEB128 not paired with R_RISCV_SUB_SET128";
732 return;
733 default:
734 break;
735 }
736 relocate(loc, rel, val);
737 }
738}
739
740void elf::initSymbolAnchors(Ctx &ctx) {
741 SmallVector<InputSection *, 0> storage;
742 for (OutputSection *osec : ctx.outputSections) {
743 if (!(osec->flags & SHF_EXECINSTR))
744 continue;
745 for (InputSection *sec : getInputSections(os: *osec, storage)) {
746 sec->relaxAux = make<RelaxAux>();
747 if (sec->relocs().size()) {
748 sec->relaxAux->relocDeltas =
749 std::make_unique<uint32_t[]>(num: sec->relocs().size());
750 sec->relaxAux->relocTypes =
751 std::make_unique<RelType[]>(num: sec->relocs().size());
752 }
753 }
754 }
755 // Store symbol anchors for adjusting st_value/st_size during relaxation.
756 // We include symbols where d->file == file for the prevailing copies.
757 //
758 // For a defined symbol foo, we may have `d->file != file` with --wrap=foo.
759 // We should process foo, as the defining object file's symbol table may not
760 // contain foo after redirectSymbols changed the foo entry to __wrap_foo. Use
761 // `d->scriptDefined` to include such symbols.
762 //
763 // `relaxAux->anchors` may contain duplicate symbols, but that is fine.
764 auto addAnchor = [](Defined *d) {
765 if (auto *sec = dyn_cast_or_null<InputSection>(Val: d->section))
766 if (sec->flags & SHF_EXECINSTR && sec->relaxAux) {
767 // If sec is discarded, relaxAux will be nullptr.
768 sec->relaxAux->anchors.push_back(Elt: {.offset: d->value, .d: d, .end: false});
769 sec->relaxAux->anchors.push_back(Elt: {.offset: d->value + d->size, .d: d, .end: true});
770 }
771 };
772 for (InputFile *file : ctx.objectFiles)
773 for (Symbol *sym : file->getSymbols()) {
774 auto *d = dyn_cast<Defined>(Val: sym);
775 if (d && (d->file == file || d->scriptDefined))
776 addAnchor(d);
777 }
778 // Add anchors for IRELATIVE symbols (see `handleNonPreemptibleIfunc`).
779 // Their values must be adjusted so IRELATIVE addends remain correct.
780 for (Defined *d : ctx.irelativeSyms)
781 addAnchor(d);
782 // Sort anchors by offset so that we can find the closest relocation
783 // efficiently. For a zero size symbol, ensure that its start anchor precedes
784 // its end anchor. For two symbols with anchors at the same offset, their
785 // order does not matter.
786 for (OutputSection *osec : ctx.outputSections) {
787 if (!(osec->flags & SHF_EXECINSTR))
788 continue;
789 for (InputSection *sec : getInputSections(os: *osec, storage)) {
790 llvm::sort(C&: sec->relaxAux->anchors, Comp: [](auto &a, auto &b) {
791 return std::make_pair(a.offset, a.end) <
792 std::make_pair(b.offset, b.end);
793 });
794 }
795 }
796}
797
798// Relax R_RISCV_CALL/R_RISCV_CALL_PLT auipc+jalr to c.j, c.jal, or jal.
799static void relaxCall(Ctx &ctx, const InputSection &sec, size_t i, uint64_t loc,
800 Relocation &r, uint32_t &remove) {
801 const bool rvc = getEFlags(ctx, f: sec.file) & EF_RISCV_RVC;
802 const Symbol &sym = *r.sym;
803 const uint64_t insnPair = read64le(P: sec.content().data() + r.offset);
804 const uint32_t rd = extractBits(v: insnPair, begin: 32 + 11, end: 32 + 7);
805 const uint64_t dest =
806 (r.expr == R_PLT_PC ? sym.getPltVA(ctx) : sym.getVA(ctx)) + r.addend;
807 const int64_t displace = dest - loc;
808
809 // When the caller specifies the old value of `remove`, disallow its
810 // increment.
811 if (remove >= 6 && rvc && isInt<12>(x: displace) && rd == X_X0) {
812 sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP;
813 sec.relaxAux->writes.push_back(Elt: 0xa001); // c.j
814 remove = 6;
815 } else if (remove >= 6 && rvc && isInt<12>(x: displace) && rd == X_RA &&
816 !ctx.arg.is64) { // RV32C only
817 sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP;
818 sec.relaxAux->writes.push_back(Elt: 0x2001); // c.jal
819 remove = 6;
820 } else if (remove >= 4 && isInt<21>(x: displace)) {
821 sec.relaxAux->relocTypes[i] = R_RISCV_JAL;
822 sec.relaxAux->writes.push_back(Elt: 0x6f | rd << 7); // jal
823 remove = 4;
824 } else {
825 remove = 0;
826 }
827}
828
829// Relax local-exec TLS when hi20 is zero.
830static void relaxTlsLe(Ctx &ctx, const InputSection &sec, size_t i,
831 uint64_t loc, Relocation &r, uint32_t &remove) {
832 uint64_t val = r.sym->getVA(ctx, addend: r.addend);
833 if (hi20(val) != 0)
834 return;
835 uint32_t insn = read32le(P: sec.content().data() + r.offset);
836 switch (r.type) {
837 case R_RISCV_TPREL_HI20:
838 case R_RISCV_TPREL_ADD:
839 // Remove lui rd, %tprel_hi(x) and add rd, rd, tp, %tprel_add(x).
840 sec.relaxAux->relocTypes[i] = R_RISCV_RELAX;
841 remove = 4;
842 break;
843 case R_RISCV_TPREL_LO12_I:
844 // addi rd, rd, %tprel_lo(x) => addi rd, tp, st_value(x)
845 sec.relaxAux->relocTypes[i] = R_RISCV_32;
846 insn = (insn & ~(31 << 15)) | (X_TP << 15);
847 sec.relaxAux->writes.push_back(Elt: setLO12_I(insn, imm: val));
848 break;
849 case R_RISCV_TPREL_LO12_S:
850 // sw rs, %tprel_lo(x)(rd) => sw rs, st_value(x)(rd)
851 sec.relaxAux->relocTypes[i] = R_RISCV_32;
852 insn = (insn & ~(31 << 15)) | (X_TP << 15);
853 sec.relaxAux->writes.push_back(Elt: setLO12_S(insn, imm: val));
854 break;
855 }
856}
857
858static void relaxHi20Lo12(Ctx &ctx, const InputSection &sec, size_t i,
859 uint64_t loc, Relocation &r, uint32_t &remove) {
860
861 // Fold into use of x0+offset
862 if (isInt<12>(x: r.sym->getVA(ctx, addend: r.addend))) {
863 switch (r.type) {
864 case R_RISCV_HI20:
865 // Remove lui rd, %hi20(x).
866 sec.relaxAux->relocTypes[i] = R_RISCV_RELAX;
867 remove = 4;
868 break;
869 case R_RISCV_LO12_I:
870 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_X0REL_I;
871 break;
872 case R_RISCV_LO12_S:
873 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_X0REL_S;
874 break;
875 }
876 return;
877 }
878
879 const Defined *gp = ctx.sym.riscvGlobalPointer;
880 if (!gp)
881 return;
882
883 if (!isInt<12>(x: r.sym->getVA(ctx, addend: r.addend) - gp->getVA(ctx)))
884 return;
885
886 switch (r.type) {
887 case R_RISCV_HI20:
888 // Remove lui rd, %hi20(x).
889 sec.relaxAux->relocTypes[i] = R_RISCV_RELAX;
890 remove = 4;
891 break;
892 case R_RISCV_LO12_I:
893 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_GPREL_I;
894 break;
895 case R_RISCV_LO12_S:
896 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_GPREL_S;
897 break;
898 }
899}
900
901static bool relax(Ctx &ctx, int pass, InputSection &sec) {
902 const uint64_t secAddr = sec.getVA();
903 const MutableArrayRef<Relocation> relocs = sec.relocs();
904 auto &aux = *sec.relaxAux;
905 bool changed = false;
906 ArrayRef<SymbolAnchor> sa = ArrayRef(aux.anchors);
907 uint64_t delta = 0;
908 bool tlsdescRelax = false, toLeShortForm = false;
909
910 std::fill_n(first: aux.relocTypes.get(), n: relocs.size(), value: R_RISCV_NONE);
911 aux.writes.clear();
912 for (auto [i, r] : llvm::enumerate(First: relocs)) {
913 const uint64_t loc = secAddr + r.offset - delta;
914 uint32_t &cur = aux.relocDeltas[i], remove = 0;
915 switch (r.type) {
916 case R_RISCV_ALIGN: {
917 const uint64_t nextLoc = loc + r.addend;
918 const uint64_t align = PowerOf2Ceil(A: r.addend + 2);
919 // All bytes beyond the alignment boundary should be removed.
920 remove = nextLoc - ((loc + align - 1) & -align);
921 // If we can't satisfy this alignment, we've found a bad input.
922 if (LLVM_UNLIKELY(static_cast<int32_t>(remove) < 0)) {
923 Err(ctx) << getErrorLoc(ctx, loc: (const uint8_t *)loc)
924 << "insufficient padding bytes for " << r.type << ": "
925 << r.addend
926 << " bytes available "
927 "for requested alignment of "
928 << align << " bytes";
929 remove = 0;
930 }
931 break;
932 }
933 case R_RISCV_CALL:
934 case R_RISCV_CALL_PLT:
935 // Prevent oscillation between states by disallowing the increment of
936 // `remove` after a few passes. The previous `remove` value is
937 // `cur-delta`.
938 if (relaxable(relocs, i)) {
939 remove = pass < 4 ? 6 : cur - delta;
940 relaxCall(ctx, sec, i, loc, r, remove);
941 }
942 break;
943 case R_RISCV_TPREL_HI20:
944 case R_RISCV_TPREL_ADD:
945 case R_RISCV_TPREL_LO12_I:
946 case R_RISCV_TPREL_LO12_S:
947 if (relaxable(relocs, i))
948 relaxTlsLe(ctx, sec, i, loc, r, remove);
949 break;
950 case R_RISCV_HI20:
951 case R_RISCV_LO12_I:
952 case R_RISCV_LO12_S:
953 if (relaxable(relocs, i))
954 relaxHi20Lo12(ctx, sec, i, loc, r, remove);
955 break;
956 case R_RISCV_TLSDESC_HI20:
957 // For TLSDESC=>LE, we can use the short form if hi20 is zero.
958 tlsdescRelax = relaxable(relocs, i);
959 toLeShortForm = tlsdescRelax && r.expr == R_RELAX_TLS_GD_TO_LE &&
960 !hi20(val: r.sym->getVA(ctx, addend: r.addend));
961 [[fallthrough]];
962 case R_RISCV_TLSDESC_LOAD_LO12:
963 // For TLSDESC=>LE/IE, AUIPC and L[DW] are removed if relaxable.
964 if (tlsdescRelax && r.expr != R_TLSDESC_PC)
965 remove = 4;
966 break;
967 case R_RISCV_TLSDESC_ADD_LO12:
968 if (toLeShortForm)
969 remove = 4;
970 break;
971 }
972
973 // For all anchors whose offsets are <= r.offset, they are preceded by
974 // the previous relocation whose `relocDeltas` value equals `delta`.
975 // Decrease their st_value and update their st_size.
976 for (; sa.size() && sa[0].offset <= r.offset; sa = sa.slice(N: 1)) {
977 if (sa[0].end)
978 sa[0].d->size = sa[0].offset - delta - sa[0].d->value;
979 else
980 sa[0].d->value = sa[0].offset - delta;
981 }
982 delta += remove;
983 if (delta != cur) {
984 cur = delta;
985 changed = true;
986 }
987 }
988
989 for (const SymbolAnchor &a : sa) {
990 if (a.end)
991 a.d->size = a.offset - delta - a.d->value;
992 else
993 a.d->value = a.offset - delta;
994 }
995 // Inform assignAddresses that the size has changed.
996 if (!isUInt<32>(x: delta))
997 Err(ctx) << "section size decrease is too large: " << delta;
998 sec.bytesDropped = delta;
999 return changed;
1000}
1001
1002// When relaxing just R_RISCV_ALIGN, relocDeltas is usually changed only once in
1003// the absence of a linker script. For call and load/store R_RISCV_RELAX, code
1004// shrinkage may reduce displacement and make more relocations eligible for
1005// relaxation. Code shrinkage may increase displacement to a call/load/store
1006// target at a higher fixed address, invalidating an earlier relaxation. Any
1007// change in section sizes can have cascading effect and require another
1008// relaxation pass.
1009bool RISCV::relaxOnce(int pass) const {
1010 llvm::TimeTraceScope timeScope("RISC-V relaxOnce");
1011 if (pass == 0)
1012 initSymbolAnchors(ctx);
1013
1014 SmallVector<InputSection *, 0> storage;
1015 bool changed = false;
1016 for (OutputSection *osec : ctx.outputSections) {
1017 if (!(osec->flags & SHF_EXECINSTR))
1018 continue;
1019 for (InputSection *sec : getInputSections(os: *osec, storage))
1020 changed |= relax(ctx, pass, sec&: *sec);
1021 }
1022 return changed;
1023}
1024
1025// If the section alignment is >= 4, advance `dot` to insert NOPs and synthesize
1026// an ALIGN relocation. Otherwise, return false to use default handling.
1027template <class ELFT, class RelTy>
1028bool RISCV::synthesizeAlignForInput(uint64_t &dot, InputSection *sec,
1029 Relocs<RelTy> rels) {
1030 if (!baseSec) {
1031 // Record the first input section with RELAX relocations. We will synthesize
1032 // ALIGN relocations here.
1033 for (auto rel : rels) {
1034 if (rel.getType(false) == R_RISCV_RELAX) {
1035 baseSec = sec;
1036 break;
1037 }
1038 }
1039 } else if (sec->addralign >= 4) {
1040 // If the alignment is >= 4 and the section does not start with an ALIGN
1041 // relocation, synthesize one.
1042 bool hasAlignRel = llvm::any_of(rels, [](const RelTy &rel) {
1043 return rel.r_offset == 0 && rel.getType(false) == R_RISCV_ALIGN;
1044 });
1045 if (!hasAlignRel) {
1046 synthesizedAligns.emplace_back(Args: dot - baseSec->getVA(),
1047 Args: sec->addralign - 2);
1048 dot += sec->addralign - 2;
1049 return true;
1050 }
1051 }
1052 return false;
1053}
1054
1055// Finalize the relocation section by appending synthesized ALIGN relocations
1056// after processing all input sections.
1057template <class ELFT, class RelTy>
1058void RISCV::finalizeSynthesizeAligns(uint64_t &dot, InputSection *sec,
1059 Relocs<RelTy> rels) {
1060 auto *f = cast<ObjFile<ELFT>>(baseSec->file);
1061 auto shdr = f->template getELFShdrs<ELFT>()[baseSec->relSecIdx];
1062 // Create a copy of InputSection.
1063 sec = make<InputSection>(*f, shdr, baseSec->name);
1064 auto *baseRelSec = cast<InputSection>(f->getSections()[baseSec->relSecIdx]);
1065 *sec = *baseRelSec;
1066 baseSec = nullptr;
1067
1068 // Allocate buffer for original and synthesized relocations in RELA format.
1069 // If CREL is used, OutputSection::finalizeNonAllocCrel will convert RELA to
1070 // CREL.
1071 auto newSize = rels.size() + synthesizedAligns.size();
1072 auto *relas = makeThreadLocalN<typename ELFT::Rela>(newSize);
1073 sec->size = newSize * sizeof(typename ELFT::Rela);
1074 sec->content_ = reinterpret_cast<uint8_t *>(relas);
1075 sec->type = SHT_RELA;
1076 // Copy original relocations to the new buffer, potentially converting CREL to
1077 // RELA.
1078 for (auto [i, r] : llvm::enumerate(rels)) {
1079 relas[i].r_offset = r.r_offset;
1080 relas[i].setSymbolAndType(r.getSymbol(0), r.getType(0), false);
1081 if constexpr (RelTy::HasAddend)
1082 relas[i].r_addend = r.r_addend;
1083 }
1084 // Append synthesized ALIGN relocations to the buffer.
1085 for (auto [i, r] : llvm::enumerate(First&: synthesizedAligns)) {
1086 auto &rela = relas[rels.size() + i];
1087 rela.r_offset = r.first;
1088 rela.setSymbolAndType(0, R_RISCV_ALIGN, false);
1089 rela.r_addend = r.second;
1090 }
1091 synthesizedAligns.clear();
1092 // Replace the old relocation section with the new one in the output section.
1093 // addOrphanSections ensures that the output relocation section is processed
1094 // after osec.
1095 for (SectionCommand *cmd : sec->getParent()->commands) {
1096 auto *isd = dyn_cast<InputSectionDescription>(Val: cmd);
1097 if (!isd)
1098 continue;
1099 for (auto *&isec : isd->sections)
1100 if (isec == baseRelSec)
1101 isec = sec;
1102 }
1103}
1104
1105template <class ELFT>
1106bool RISCV::synthesizeAlignAux(uint64_t &dot, InputSection *sec) {
1107 bool ret = false;
1108 if (sec) {
1109 invokeOnRelocs(*sec, ret = synthesizeAlignForInput<ELFT>, dot, sec);
1110 } else if (baseSec) {
1111 invokeOnRelocs(*baseSec, finalizeSynthesizeAligns<ELFT>, dot, sec);
1112 }
1113 return ret;
1114}
1115
1116// Without linker relaxation enabled for a particular relocatable file or
1117// section, the assembler will not generate R_RISCV_ALIGN relocations for
1118// alignment directives. This becomes problematic in a two-stage linking
1119// process: ld -r a.o b.o -o ab.o; ld ab.o -o ab. This function synthesizes an
1120// R_RISCV_ALIGN relocation at section start when needed.
1121//
1122// When called with an input section (`sec` is not null): If the section
1123// alignment is >= 4, advance `dot` to insert NOPs and synthesize an ALIGN
1124// relocation.
1125//
1126// When called after all input sections are processed (`sec` is null): The
1127// output relocation section is updated with all the newly synthesized ALIGN
1128// relocations.
1129bool RISCV::synthesizeAlign(uint64_t &dot, InputSection *sec) {
1130 assert(ctx.arg.relocatable);
1131 if (ctx.arg.is64)
1132 return synthesizeAlignAux<ELF64LE>(dot, sec);
1133 return synthesizeAlignAux<ELF32LE>(dot, sec);
1134}
1135
1136void RISCV::finalizeRelax(int passes) const {
1137 llvm::TimeTraceScope timeScope("Finalize RISC-V relaxation");
1138 Log(ctx) << "relaxation passes: " << passes;
1139 SmallVector<InputSection *, 0> storage;
1140 for (OutputSection *osec : ctx.outputSections) {
1141 if (!(osec->flags & SHF_EXECINSTR))
1142 continue;
1143 for (InputSection *sec : getInputSections(os: *osec, storage)) {
1144 RelaxAux &aux = *sec->relaxAux;
1145 if (!aux.relocDeltas)
1146 continue;
1147
1148 MutableArrayRef<Relocation> rels = sec->relocs();
1149 ArrayRef<uint8_t> old = sec->content();
1150 size_t newSize = old.size() - aux.relocDeltas[rels.size() - 1];
1151 size_t writesIdx = 0;
1152 uint8_t *p = ctx.bAlloc.Allocate<uint8_t>(Num: newSize);
1153 uint64_t offset = 0;
1154 int64_t delta = 0;
1155 sec->content_ = p;
1156 sec->size = newSize;
1157 sec->bytesDropped = 0;
1158
1159 // Update section content: remove NOPs for R_RISCV_ALIGN and rewrite
1160 // instructions for relaxed relocations.
1161 for (size_t i = 0, e = rels.size(); i != e; ++i) {
1162 uint32_t remove = aux.relocDeltas[i] - delta;
1163 delta = aux.relocDeltas[i];
1164 if (remove == 0 && aux.relocTypes[i] == R_RISCV_NONE)
1165 continue;
1166
1167 // Copy from last location to the current relocated location.
1168 const Relocation &r = rels[i];
1169 uint64_t size = r.offset - offset;
1170 memcpy(dest: p, src: old.data() + offset, n: size);
1171 p += size;
1172
1173 // For R_RISCV_ALIGN, we will place `offset` in a location (among NOPs)
1174 // to satisfy the alignment requirement. If both `remove` and r.addend
1175 // are multiples of 4, it is as if we have skipped some NOPs. Otherwise
1176 // we are in the middle of a 4-byte NOP, and we need to rewrite the NOP
1177 // sequence.
1178 int64_t skip = 0;
1179 if (r.type == R_RISCV_ALIGN) {
1180 if (remove % 4 || r.addend % 4) {
1181 skip = r.addend - remove;
1182 int64_t j = 0;
1183 for (; j + 4 <= skip; j += 4)
1184 write32le(P: p + j, V: 0x00000013); // nop
1185 if (j != skip) {
1186 assert(j + 2 == skip);
1187 write16le(P: p + j, V: 0x0001); // c.nop
1188 }
1189 }
1190 } else if (RelType newType = aux.relocTypes[i]) {
1191 switch (newType) {
1192 case INTERNAL_R_RISCV_GPREL_I:
1193 case INTERNAL_R_RISCV_GPREL_S:
1194 case INTERNAL_R_RISCV_X0REL_I:
1195 case INTERNAL_R_RISCV_X0REL_S:
1196 break;
1197 case R_RISCV_RELAX:
1198 // Used by relaxTlsLe to indicate the relocation is ignored.
1199 break;
1200 case R_RISCV_RVC_JUMP:
1201 skip = 2;
1202 write16le(P: p, V: aux.writes[writesIdx++]);
1203 break;
1204 case R_RISCV_JAL:
1205 skip = 4;
1206 write32le(P: p, V: aux.writes[writesIdx++]);
1207 break;
1208 case R_RISCV_32:
1209 // Used by relaxTlsLe to write a uint32_t then suppress the handling
1210 // in relocateAlloc.
1211 skip = 4;
1212 write32le(P: p, V: aux.writes[writesIdx++]);
1213 aux.relocTypes[i] = R_RISCV_NONE;
1214 break;
1215 default:
1216 llvm_unreachable("unsupported type");
1217 }
1218 }
1219
1220 p += skip;
1221 offset = r.offset + skip + remove;
1222 }
1223 memcpy(dest: p, src: old.data() + offset, n: old.size() - offset);
1224
1225 // Subtract the previous relocDeltas value from the relocation offset.
1226 // For a pair of R_RISCV_CALL/R_RISCV_RELAX with the same offset, decrease
1227 // their r_offset by the same delta.
1228 delta = 0;
1229 for (size_t i = 0, e = rels.size(); i != e;) {
1230 uint64_t cur = rels[i].offset;
1231 do {
1232 rels[i].offset -= delta;
1233 if (aux.relocTypes[i] != R_RISCV_NONE)
1234 rels[i].type = aux.relocTypes[i];
1235 } while (++i != e && rels[i].offset == cur);
1236 delta = aux.relocDeltas[i - 1];
1237 }
1238 }
1239 }
1240}
1241
1242namespace {
1243// Representation of the merged .riscv.attributes input sections. The psABI
1244// specifies merge policy for attributes. E.g. if we link an object without an
1245// extension with an object with the extension, the output Tag_RISCV_arch shall
1246// contain the extension. Some tools like objdump parse .riscv.attributes and
1247// disabling some instructions if the first Tag_RISCV_arch does not contain an
1248// extension.
1249class RISCVAttributesSection final : public SyntheticSection {
1250public:
1251 RISCVAttributesSection(Ctx &ctx)
1252 : SyntheticSection(ctx, ".riscv.attributes", SHT_RISCV_ATTRIBUTES, 0, 1) {
1253 }
1254
1255 size_t getSize() const override { return size; }
1256 void writeTo(uint8_t *buf) override;
1257
1258 static constexpr StringRef vendor = "riscv";
1259 DenseMap<unsigned, unsigned> intAttr;
1260 DenseMap<unsigned, StringRef> strAttr;
1261 size_t size = 0;
1262};
1263} // namespace
1264
1265static void mergeArch(Ctx &ctx, RISCVISAUtils::OrderedExtensionMap &mergedExts,
1266 unsigned &mergedXlen, const InputSectionBase *sec,
1267 StringRef s) {
1268 auto maybeInfo = RISCVISAInfo::parseNormalizedArchString(Arch: s);
1269 if (!maybeInfo) {
1270 Err(ctx) << sec << ": " << s << ": " << maybeInfo.takeError();
1271 return;
1272 }
1273
1274 // Merge extensions.
1275 RISCVISAInfo &info = **maybeInfo;
1276 if (mergedExts.empty()) {
1277 mergedExts = info.getExtensions();
1278 mergedXlen = info.getXLen();
1279 } else {
1280 for (const auto &ext : info.getExtensions()) {
1281 auto p = mergedExts.insert(x: ext);
1282 if (!p.second) {
1283 if (std::tie(args&: p.first->second.Major, args&: p.first->second.Minor) <
1284 std::tie(args: ext.second.Major, args: ext.second.Minor))
1285 p.first->second = ext.second;
1286 }
1287 }
1288 }
1289}
1290
1291static void mergeAtomic(Ctx &ctx, DenseMap<unsigned, unsigned>::iterator it,
1292 const InputSectionBase *oldSection,
1293 const InputSectionBase *newSection,
1294 RISCVAttrs::RISCVAtomicAbiTag oldTag,
1295 RISCVAttrs::RISCVAtomicAbiTag newTag) {
1296 using RISCVAttrs::RISCVAtomicAbiTag;
1297 // Same tags stay the same, and UNKNOWN is compatible with anything
1298 if (oldTag == newTag || newTag == RISCVAtomicAbiTag::UNKNOWN)
1299 return;
1300
1301 auto reportAbiError = [&]() {
1302 Err(ctx) << "atomic abi mismatch for " << oldSection->name << "\n>>> "
1303 << oldSection << ": atomic_abi=" << static_cast<unsigned>(oldTag)
1304 << "\n>>> " << newSection
1305 << ": atomic_abi=" << static_cast<unsigned>(newTag);
1306 };
1307
1308 auto reportUnknownAbiError = [&](const InputSectionBase *section,
1309 RISCVAtomicAbiTag tag) {
1310 switch (tag) {
1311 case RISCVAtomicAbiTag::UNKNOWN:
1312 case RISCVAtomicAbiTag::A6C:
1313 case RISCVAtomicAbiTag::A6S:
1314 case RISCVAtomicAbiTag::A7:
1315 return;
1316 };
1317 Err(ctx) << "unknown atomic abi for " << section->name << "\n>>> "
1318 << section << ": atomic_abi=" << static_cast<unsigned>(tag);
1319 };
1320 switch (oldTag) {
1321 case RISCVAtomicAbiTag::UNKNOWN:
1322 it->getSecond() = static_cast<unsigned>(newTag);
1323 return;
1324 case RISCVAtomicAbiTag::A6C:
1325 switch (newTag) {
1326 case RISCVAtomicAbiTag::A6S:
1327 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A6C);
1328 return;
1329 case RISCVAtomicAbiTag::A7:
1330 reportAbiError();
1331 return;
1332 case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN:
1333 case RISCVAttrs::RISCVAtomicAbiTag::A6C:
1334 return;
1335 };
1336 break;
1337
1338 case RISCVAtomicAbiTag::A6S:
1339 switch (newTag) {
1340 case RISCVAtomicAbiTag::A6C:
1341 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A6C);
1342 return;
1343 case RISCVAtomicAbiTag::A7:
1344 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A7);
1345 return;
1346 case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN:
1347 case RISCVAttrs::RISCVAtomicAbiTag::A6S:
1348 return;
1349 };
1350 break;
1351
1352 case RISCVAtomicAbiTag::A7:
1353 switch (newTag) {
1354 case RISCVAtomicAbiTag::A6S:
1355 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A7);
1356 return;
1357 case RISCVAtomicAbiTag::A6C:
1358 reportAbiError();
1359 return;
1360 case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN:
1361 case RISCVAttrs::RISCVAtomicAbiTag::A7:
1362 return;
1363 };
1364 break;
1365 };
1366
1367 // If we get here, then we have an invalid tag, so report it.
1368 // Putting these checks at the end allows us to only do these checks when we
1369 // need to, since this is expected to be a rare occurrence.
1370 reportUnknownAbiError(oldSection, oldTag);
1371 reportUnknownAbiError(newSection, newTag);
1372}
1373
1374static RISCVAttributesSection *
1375mergeAttributesSection(Ctx &ctx,
1376 const SmallVector<InputSectionBase *, 0> &sections) {
1377 using RISCVAttrs::RISCVAtomicAbiTag;
1378 RISCVISAUtils::OrderedExtensionMap exts;
1379 const InputSectionBase *firstStackAlign = nullptr;
1380 const InputSectionBase *firstAtomicAbi = nullptr;
1381 unsigned firstStackAlignValue = 0, xlen = 0;
1382 bool hasArch = false;
1383
1384 ctx.in.riscvAttributes = std::make_unique<RISCVAttributesSection>(args&: ctx);
1385 auto &merged = static_cast<RISCVAttributesSection &>(*ctx.in.riscvAttributes);
1386
1387 // Collect all tags values from attributes section.
1388 const auto &attributesTags = RISCVAttrs::getRISCVAttributeTags();
1389 for (const InputSectionBase *sec : sections) {
1390 RISCVAttributeParser parser;
1391 if (Error e = parser.parse(section: sec->content(), endian: llvm::endianness::little))
1392 Warn(ctx) << sec << ": " << std::move(e);
1393 for (const auto &tag : attributesTags) {
1394 switch (RISCVAttrs::AttrType(tag.attr)) {
1395 // Integer attributes.
1396 case RISCVAttrs::STACK_ALIGN:
1397 if (auto i = parser.getAttributeValue(tag: tag.attr)) {
1398 auto r = merged.intAttr.try_emplace(Key: tag.attr, Args&: *i);
1399 if (r.second) {
1400 firstStackAlign = sec;
1401 firstStackAlignValue = *i;
1402 } else if (r.first->second != *i) {
1403 Err(ctx) << sec << " has stack_align=" << *i << " but "
1404 << firstStackAlign
1405 << " has stack_align=" << firstStackAlignValue;
1406 }
1407 }
1408 continue;
1409 case RISCVAttrs::UNALIGNED_ACCESS:
1410 if (auto i = parser.getAttributeValue(tag: tag.attr))
1411 merged.intAttr[tag.attr] |= *i;
1412 continue;
1413
1414 // String attributes.
1415 case RISCVAttrs::ARCH:
1416 if (auto s = parser.getAttributeString(tag: tag.attr)) {
1417 hasArch = true;
1418 mergeArch(ctx, mergedExts&: exts, mergedXlen&: xlen, sec, s: *s);
1419 }
1420 continue;
1421
1422 // Attributes which use the default handling.
1423 case RISCVAttrs::PRIV_SPEC:
1424 case RISCVAttrs::PRIV_SPEC_MINOR:
1425 case RISCVAttrs::PRIV_SPEC_REVISION:
1426 break;
1427
1428 case RISCVAttrs::AttrType::ATOMIC_ABI:
1429 if (auto i = parser.getAttributeValue(tag: tag.attr)) {
1430 auto r = merged.intAttr.try_emplace(Key: tag.attr, Args&: *i);
1431 if (r.second)
1432 firstAtomicAbi = sec;
1433 else
1434 mergeAtomic(ctx, it: r.first, oldSection: firstAtomicAbi, newSection: sec,
1435 oldTag: static_cast<RISCVAtomicAbiTag>(r.first->getSecond()),
1436 newTag: static_cast<RISCVAtomicAbiTag>(*i));
1437 }
1438 continue;
1439 }
1440
1441 // Fallback for deprecated priv_spec* and other unknown attributes: retain
1442 // the attribute if all input sections agree on the value. GNU ld uses 0
1443 // and empty strings as default values which are not dumped to the output.
1444 // TODO Adjust after resolution to
1445 // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/issues/352
1446 if (tag.attr % 2 == 0) {
1447 if (auto i = parser.getAttributeValue(tag: tag.attr)) {
1448 auto r = merged.intAttr.try_emplace(Key: tag.attr, Args&: *i);
1449 if (!r.second && r.first->second != *i)
1450 r.first->second = 0;
1451 }
1452 } else if (auto s = parser.getAttributeString(tag: tag.attr)) {
1453 auto r = merged.strAttr.try_emplace(Key: tag.attr, Args&: *s);
1454 if (!r.second && r.first->second != *s)
1455 r.first->second = {};
1456 }
1457 }
1458 }
1459
1460 if (hasArch && xlen != 0) {
1461 if (auto result = RISCVISAInfo::createFromExtMap(XLen: xlen, Exts: exts)) {
1462 merged.strAttr.try_emplace(Key: RISCVAttrs::ARCH,
1463 Args: ctx.saver.save(S: (*result)->toString()));
1464 } else {
1465 Err(ctx) << result.takeError();
1466 }
1467 }
1468
1469 // The total size of headers: format-version [ <section-length> "vendor-name"
1470 // [ <file-tag> <size>.
1471 size_t size = 5 + merged.vendor.size() + 1 + 5;
1472 for (auto &attr : merged.intAttr)
1473 if (attr.second != 0)
1474 size += getULEB128Size(Value: attr.first) + getULEB128Size(Value: attr.second);
1475 for (auto &attr : merged.strAttr)
1476 if (!attr.second.empty())
1477 size += getULEB128Size(Value: attr.first) + attr.second.size() + 1;
1478 merged.size = size;
1479 return &merged;
1480}
1481
1482void RISCVAttributesSection::writeTo(uint8_t *buf) {
1483 const size_t size = getSize();
1484 uint8_t *const end = buf + size;
1485 *buf = ELFAttrs::Format_Version;
1486 write32(ctx, p: buf + 1, v: size - 1);
1487 buf += 5;
1488
1489 memcpy(dest: buf, src: vendor.data(), n: vendor.size());
1490 buf += vendor.size() + 1;
1491
1492 *buf = ELFAttrs::File;
1493 write32(ctx, p: buf + 1, v: end - buf);
1494 buf += 5;
1495
1496 for (auto &attr : intAttr) {
1497 if (attr.second == 0)
1498 continue;
1499 buf += encodeULEB128(Value: attr.first, p: buf);
1500 buf += encodeULEB128(Value: attr.second, p: buf);
1501 }
1502 for (auto &attr : strAttr) {
1503 if (attr.second.empty())
1504 continue;
1505 buf += encodeULEB128(Value: attr.first, p: buf);
1506 memcpy(dest: buf, src: attr.second.data(), n: attr.second.size());
1507 buf += attr.second.size() + 1;
1508 }
1509}
1510
1511void elf::mergeRISCVAttributesSections(Ctx &ctx) {
1512 // Find the first input SHT_RISCV_ATTRIBUTES; return if not found.
1513 size_t place =
1514 llvm::find_if(Range&: ctx.inputSections,
1515 P: [](auto *s) { return s->type == SHT_RISCV_ATTRIBUTES; }) -
1516 ctx.inputSections.begin();
1517 if (place == ctx.inputSections.size())
1518 return;
1519
1520 // Extract all SHT_RISCV_ATTRIBUTES sections into `sections`.
1521 SmallVector<InputSectionBase *, 0> sections;
1522 llvm::erase_if(C&: ctx.inputSections, P: [&](InputSectionBase *s) {
1523 if (s->type != SHT_RISCV_ATTRIBUTES)
1524 return false;
1525 sections.push_back(Elt: s);
1526 return true;
1527 });
1528
1529 // Add the merged section.
1530 ctx.inputSections.insert(I: ctx.inputSections.begin() + place,
1531 Elt: mergeAttributesSection(ctx, sections));
1532}
1533
1534void elf::setRISCVTargetInfo(Ctx &ctx) { ctx.target.reset(p: new RISCV(ctx)); }
1535