1//===- RISCV.cpp ----------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "InputFiles.h"
10#include "OutputSections.h"
11#include "RISCVInternalRelocations.h"
12#include "RelocScan.h"
13#include "Symbols.h"
14#include "SyntheticSections.h"
15#include "Target.h"
16#include "llvm/Support/ELFAttributes.h"
17#include "llvm/Support/LEB128.h"
18#include "llvm/Support/RISCVAttributeParser.h"
19#include "llvm/Support/RISCVAttributes.h"
20#include "llvm/Support/TimeProfiler.h"
21#include "llvm/TargetParser/RISCVISAInfo.h"
22
23using namespace llvm;
24using namespace llvm::object;
25using namespace llvm::support::endian;
26using namespace llvm::ELF;
27using namespace lld;
28using namespace lld::elf;
29
30namespace {
31
32class RISCV final : public TargetInfo {
33public:
34 RISCV(Ctx &);
35 uint32_t calcEFlags() const override;
36 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;
37 void writeGotHeader(uint8_t *buf) const override;
38 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
39 void writeIgotPlt(uint8_t *buf, const Symbol &s) const override;
40 void writePltHeader(uint8_t *buf) const override;
41 void writePlt(uint8_t *buf, const Symbol &sym,
42 uint64_t pltEntryAddr) const override;
43 template <class ELFT, class RelTy>
44 void scanSectionImpl(InputSectionBase &, Relocs<RelTy>);
45 void scanSection(InputSectionBase &) override;
46 RelType getDynRel(RelType type) const override;
47 RelExpr getRelExpr(RelType type, const Symbol &s,
48 const uint8_t *loc) const override;
49 void relocate(uint8_t *loc, const Relocation &rel,
50 uint64_t val) const override;
51 void relocateAlloc(InputSection &sec, uint8_t *buf) const override;
52 bool relaxOnce(int pass) const override;
53 template <class ELFT, class RelTy>
54 bool synthesizeAlignForInput(uint64_t &dot, InputSection *sec,
55 Relocs<RelTy> rels);
56 template <class ELFT, class RelTy>
57 void finalizeSynthesizeAligns(uint64_t &dot, InputSection *sec,
58 Relocs<RelTy> rels);
59 template <class ELFT>
60 bool synthesizeAlignAux(uint64_t &dot, InputSection *sec);
61 bool synthesizeAlign(uint64_t &dot, InputSection *sec) override;
62 void finalizeRelax(int passes) const override;
63
64 // The following two variables are used by synthesized ALIGN relocations.
65 InputSection *baseSec = nullptr;
66 // r_offset and r_addend pairs.
67 SmallVector<std::pair<uint64_t, uint64_t>, 0> synthesizedAligns;
68};
69
70} // end anonymous namespace
71
72// These are internal relocation numbers for GP/X0 relaxation. They aren't part
73// of the psABI spec.
74#define INTERNAL_R_RISCV_GPREL_I 256
75#define INTERNAL_R_RISCV_GPREL_S 257
76#define INTERNAL_R_RISCV_X0REL_I 258
77#define INTERNAL_R_RISCV_X0REL_S 259
78
79const uint64_t dtpOffset = 0x800;
80
81namespace {
82enum Op {
83 ADDI = 0x13,
84 AUIPC = 0x17,
85 JALR = 0x67,
86 LD = 0x3003,
87 LUI = 0x37,
88 LW = 0x2003,
89 SRLI = 0x5013,
90 SUB = 0x40000033,
91};
92
93enum Reg {
94 X_X0 = 0,
95 X_RA = 1,
96 X_GP = 3,
97 X_TP = 4,
98 X_T0 = 5,
99 X_T1 = 6,
100 X_T2 = 7,
101 X_A0 = 10,
102 X_T3 = 28,
103};
104} // namespace
105
106static uint32_t hi20(uint32_t val) { return (val + 0x800) >> 12; }
107static uint32_t lo12(uint32_t val) { return val & 4095; }
108
109static uint32_t itype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t imm) {
110 return op | (rd << 7) | (rs1 << 15) | (imm << 20);
111}
112static uint32_t rtype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t rs2) {
113 return op | (rd << 7) | (rs1 << 15) | (rs2 << 20);
114}
115static uint32_t utype(uint32_t op, uint32_t rd, uint32_t imm) {
116 return op | (rd << 7) | (imm << 12);
117}
118
119// Extract bits v[begin:end], where range is inclusive, and begin must be < 63.
120static uint32_t extractBits(uint64_t v, uint32_t begin, uint32_t end) {
121 return (v & ((1ULL << (begin + 1)) - 1)) >> end;
122}
123
124static uint32_t setLO12_I(uint32_t insn, uint32_t imm) {
125 return (insn & 0xfffff) | (imm << 20);
126}
127static uint32_t setLO12_S(uint32_t insn, uint32_t imm) {
128 return (insn & 0x1fff07f) | (extractBits(v: imm, begin: 11, end: 5) << 25) |
129 (extractBits(v: imm, begin: 4, end: 0) << 7);
130}
131
132RISCV::RISCV(Ctx &ctx) : TargetInfo(ctx) {
133 copyRel = R_RISCV_COPY;
134 pltRel = R_RISCV_JUMP_SLOT;
135 relativeRel = R_RISCV_RELATIVE;
136 iRelativeRel = R_RISCV_IRELATIVE;
137 if (ctx.arg.is64) {
138 symbolicRel = R_RISCV_64;
139 tlsModuleIndexRel = R_RISCV_TLS_DTPMOD64;
140 tlsOffsetRel = R_RISCV_TLS_DTPREL64;
141 tlsGotRel = R_RISCV_TLS_TPREL64;
142 } else {
143 symbolicRel = R_RISCV_32;
144 tlsModuleIndexRel = R_RISCV_TLS_DTPMOD32;
145 tlsOffsetRel = R_RISCV_TLS_DTPREL32;
146 tlsGotRel = R_RISCV_TLS_TPREL32;
147 }
148 gotRel = symbolicRel;
149 tlsDescRel = R_RISCV_TLSDESC;
150
151 // .got[0] = _DYNAMIC
152 gotHeaderEntriesNum = 1;
153
154 // .got.plt[0] = _dl_runtime_resolve, .got.plt[1] = link_map
155 gotPltHeaderEntriesNum = 2;
156
157 pltHeaderSize = 32;
158 pltEntrySize = 16;
159 ipltEntrySize = 16;
160}
161
162static uint32_t getEFlags(Ctx &ctx, InputFile *f) {
163 if (ctx.arg.is64)
164 return cast<ObjFile<ELF64LE>>(Val: f)->getObj().getHeader().e_flags;
165 return cast<ObjFile<ELF32LE>>(Val: f)->getObj().getHeader().e_flags;
166}
167
168uint32_t RISCV::calcEFlags() const {
169 // If there are only binary input files (from -b binary), use a
170 // value of 0 for the ELF header flags.
171 if (ctx.objectFiles.empty())
172 return 0;
173
174 uint32_t target = getEFlags(ctx, f: ctx.objectFiles.front());
175 for (InputFile *f : ctx.objectFiles) {
176 uint32_t eflags = getEFlags(ctx, f);
177 if (eflags & EF_RISCV_RVC)
178 target |= EF_RISCV_RVC;
179
180 if ((eflags & EF_RISCV_FLOAT_ABI) != (target & EF_RISCV_FLOAT_ABI))
181 Err(ctx) << f
182 << ": cannot link object files with different "
183 "floating-point ABI from "
184 << ctx.objectFiles[0];
185
186 if ((eflags & EF_RISCV_RVE) != (target & EF_RISCV_RVE))
187 Err(ctx) << f << ": cannot link object files with different EF_RISCV_RVE";
188 }
189
190 return target;
191}
192
193int64_t RISCV::getImplicitAddend(const uint8_t *buf, RelType type) const {
194 switch (type) {
195 default:
196 InternalErr(ctx, buf) << "cannot read addend for relocation " << type;
197 return 0;
198 case R_RISCV_32:
199 case R_RISCV_TLS_DTPMOD32:
200 case R_RISCV_TLS_DTPREL32:
201 case R_RISCV_TLS_TPREL32:
202 return SignExtend64<32>(x: read32le(P: buf));
203 case R_RISCV_64:
204 case R_RISCV_TLS_DTPMOD64:
205 case R_RISCV_TLS_DTPREL64:
206 case R_RISCV_TLS_TPREL64:
207 return read64le(P: buf);
208 case R_RISCV_RELATIVE:
209 case R_RISCV_IRELATIVE:
210 return ctx.arg.is64 ? read64le(P: buf) : read32le(P: buf);
211 case R_RISCV_NONE:
212 case R_RISCV_JUMP_SLOT:
213 // These relocations are defined as not having an implicit addend.
214 return 0;
215 case R_RISCV_TLSDESC:
216 return ctx.arg.is64 ? read64le(P: buf + 8) : read32le(P: buf + 4);
217 }
218}
219
220void RISCV::writeGotHeader(uint8_t *buf) const {
221 if (ctx.arg.is64)
222 write64le(P: buf, V: ctx.mainPart->dynamic->getVA());
223 else
224 write32le(P: buf, V: ctx.mainPart->dynamic->getVA());
225}
226
227void RISCV::writeGotPlt(uint8_t *buf, const Symbol &s) const {
228 if (ctx.arg.is64)
229 write64le(P: buf, V: ctx.in.plt->getVA());
230 else
231 write32le(P: buf, V: ctx.in.plt->getVA());
232}
233
234void RISCV::writeIgotPlt(uint8_t *buf, const Symbol &s) const {
235 if (ctx.arg.writeAddends) {
236 if (ctx.arg.is64)
237 write64le(P: buf, V: s.getVA(ctx));
238 else
239 write32le(P: buf, V: s.getVA(ctx));
240 }
241}
242
243void RISCV::writePltHeader(uint8_t *buf) const {
244 // 1: auipc t2, %pcrel_hi(.got.plt)
245 // sub t1, t1, t3
246 // l[wd] t3, %pcrel_lo(1b)(t2); t3 = _dl_runtime_resolve
247 // addi t1, t1, -pltHeaderSize-12; t1 = &.plt[i] - &.plt[0]
248 // addi t0, t2, %pcrel_lo(1b)
249 // srli t1, t1, (rv64?1:2); t1 = &.got.plt[i] - &.got.plt[0]
250 // l[wd] t0, Wordsize(t0); t0 = link_map
251 // jr t3
252 uint32_t offset = ctx.in.gotPlt->getVA() - ctx.in.plt->getVA();
253 uint32_t load = ctx.arg.is64 ? LD : LW;
254 write32le(P: buf + 0, V: utype(op: AUIPC, rd: X_T2, imm: hi20(val: offset)));
255 write32le(P: buf + 4, V: rtype(op: SUB, rd: X_T1, rs1: X_T1, rs2: X_T3));
256 write32le(P: buf + 8, V: itype(op: load, rd: X_T3, rs1: X_T2, imm: lo12(val: offset)));
257 write32le(P: buf + 12, V: itype(op: ADDI, rd: X_T1, rs1: X_T1, imm: -ctx.target->pltHeaderSize - 12));
258 write32le(P: buf + 16, V: itype(op: ADDI, rd: X_T0, rs1: X_T2, imm: lo12(val: offset)));
259 write32le(P: buf + 20, V: itype(op: SRLI, rd: X_T1, rs1: X_T1, imm: ctx.arg.is64 ? 1 : 2));
260 write32le(P: buf + 24, V: itype(op: load, rd: X_T0, rs1: X_T0, imm: ctx.arg.wordsize));
261 write32le(P: buf + 28, V: itype(op: JALR, rd: 0, rs1: X_T3, imm: 0));
262}
263
264void RISCV::writePlt(uint8_t *buf, const Symbol &sym,
265 uint64_t pltEntryAddr) const {
266 // 1: auipc t3, %pcrel_hi(f@.got.plt)
267 // l[wd] t3, %pcrel_lo(1b)(t3)
268 // jalr t1, t3
269 // nop
270 uint32_t offset = sym.getGotPltVA(ctx) - pltEntryAddr;
271 write32le(P: buf + 0, V: utype(op: AUIPC, rd: X_T3, imm: hi20(val: offset)));
272 write32le(P: buf + 4, V: itype(op: ctx.arg.is64 ? LD : LW, rd: X_T3, rs1: X_T3, imm: lo12(val: offset)));
273 write32le(P: buf + 8, V: itype(op: JALR, rd: X_T1, rs1: X_T3, imm: 0));
274 write32le(P: buf + 12, V: itype(op: ADDI, rd: 0, rs1: 0, imm: 0));
275}
276
277RelType RISCV::getDynRel(RelType type) const {
278 return type == ctx.target->symbolicRel ? type
279 : static_cast<RelType>(R_RISCV_NONE);
280}
281
282RelExpr RISCV::getRelExpr(const RelType type, const Symbol &s,
283 const uint8_t *loc) const {
284 switch (type) {
285 case R_RISCV_NONE:
286 case R_RISCV_VENDOR:
287 return R_NONE;
288 case R_RISCV_32:
289 case R_RISCV_64:
290 case R_RISCV_HI20:
291 case R_RISCV_LO12_I:
292 case R_RISCV_LO12_S:
293 return R_ABS;
294 case R_RISCV_ADD8:
295 case R_RISCV_ADD16:
296 case R_RISCV_ADD32:
297 case R_RISCV_ADD64:
298 case R_RISCV_SET6:
299 case R_RISCV_SET8:
300 case R_RISCV_SET16:
301 case R_RISCV_SET32:
302 case R_RISCV_SUB6:
303 case R_RISCV_SUB8:
304 case R_RISCV_SUB16:
305 case R_RISCV_SUB32:
306 case R_RISCV_SUB64:
307 return RE_RISCV_ADD;
308 case R_RISCV_JAL:
309 case R_RISCV_BRANCH:
310 case R_RISCV_PCREL_HI20:
311 case R_RISCV_RVC_BRANCH:
312 case R_RISCV_RVC_JUMP:
313 case R_RISCV_32_PCREL:
314 return R_PC;
315 case R_RISCV_CALL:
316 case R_RISCV_CALL_PLT:
317 case R_RISCV_PLT32:
318 return R_PLT_PC;
319 case R_RISCV_GOT_HI20:
320 case R_RISCV_GOT32_PCREL:
321 return R_GOT_PC;
322 case R_RISCV_PCREL_LO12_I:
323 case R_RISCV_PCREL_LO12_S:
324 return RE_RISCV_PC_INDIRECT;
325 case R_RISCV_TLSDESC_HI20:
326 case R_RISCV_TLSDESC_LOAD_LO12:
327 case R_RISCV_TLSDESC_ADD_LO12:
328 return R_TLSDESC_PC;
329 case R_RISCV_TLSDESC_CALL:
330 return R_TLSDESC_CALL;
331 case R_RISCV_TLS_GD_HI20:
332 return R_TLSGD_PC;
333 case R_RISCV_TLS_GOT_HI20:
334 return R_GOT_PC;
335 case R_RISCV_TPREL_HI20:
336 case R_RISCV_TPREL_LO12_I:
337 case R_RISCV_TPREL_LO12_S:
338 return R_TPREL;
339 case R_RISCV_ALIGN:
340 return R_RELAX_HINT;
341 case R_RISCV_TPREL_ADD:
342 case R_RISCV_RELAX:
343 return ctx.arg.relax ? R_RELAX_HINT : R_NONE;
344 case R_RISCV_SET_ULEB128:
345 case R_RISCV_SUB_ULEB128:
346 return RE_RISCV_LEB128;
347 default:
348 if (type.v & INTERNAL_RISCV_VENDOR_MASK) {
349 Err(ctx) << getErrorLoc(ctx, loc)
350 << "unsupported vendor-specific relocation " << type
351 << " against symbol " << &s;
352 return R_NONE;
353 }
354 Err(ctx) << getErrorLoc(ctx, loc) << "unknown relocation ("
355 << (type.v & ~INTERNAL_RISCV_VENDOR_MASK) << ") against symbol "
356 << &s;
357 return R_NONE;
358 }
359}
360
361void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
362 const unsigned bits = ctx.arg.wordsize * 8;
363
364 switch (rel.type) {
365 case R_RISCV_32:
366 write32le(P: loc, V: val);
367 return;
368 case R_RISCV_64:
369 write64le(P: loc, V: val);
370 return;
371
372 case R_RISCV_RVC_BRANCH: {
373 checkInt(ctx, loc, v: val, n: 9, rel);
374 checkAlignment(ctx, loc, v: val, n: 2, rel);
375 uint16_t insn = read16le(P: loc) & 0xE383;
376 uint16_t imm8 = extractBits(v: val, begin: 8, end: 8) << 12;
377 uint16_t imm4_3 = extractBits(v: val, begin: 4, end: 3) << 10;
378 uint16_t imm7_6 = extractBits(v: val, begin: 7, end: 6) << 5;
379 uint16_t imm2_1 = extractBits(v: val, begin: 2, end: 1) << 3;
380 uint16_t imm5 = extractBits(v: val, begin: 5, end: 5) << 2;
381 insn |= imm8 | imm4_3 | imm7_6 | imm2_1 | imm5;
382
383 write16le(P: loc, V: insn);
384 return;
385 }
386
387 case R_RISCV_RVC_JUMP: {
388 checkInt(ctx, loc, v: val, n: 12, rel);
389 checkAlignment(ctx, loc, v: val, n: 2, rel);
390 uint16_t insn = read16le(P: loc) & 0xE003;
391 uint16_t imm11 = extractBits(v: val, begin: 11, end: 11) << 12;
392 uint16_t imm4 = extractBits(v: val, begin: 4, end: 4) << 11;
393 uint16_t imm9_8 = extractBits(v: val, begin: 9, end: 8) << 9;
394 uint16_t imm10 = extractBits(v: val, begin: 10, end: 10) << 8;
395 uint16_t imm6 = extractBits(v: val, begin: 6, end: 6) << 7;
396 uint16_t imm7 = extractBits(v: val, begin: 7, end: 7) << 6;
397 uint16_t imm3_1 = extractBits(v: val, begin: 3, end: 1) << 3;
398 uint16_t imm5 = extractBits(v: val, begin: 5, end: 5) << 2;
399 insn |= imm11 | imm4 | imm9_8 | imm10 | imm6 | imm7 | imm3_1 | imm5;
400
401 write16le(P: loc, V: insn);
402 return;
403 }
404
405 case R_RISCV_JAL: {
406 checkInt(ctx, loc, v: val, n: 21, rel);
407 checkAlignment(ctx, loc, v: val, n: 2, rel);
408
409 uint32_t insn = read32le(P: loc) & 0xFFF;
410 uint32_t imm20 = extractBits(v: val, begin: 20, end: 20) << 31;
411 uint32_t imm10_1 = extractBits(v: val, begin: 10, end: 1) << 21;
412 uint32_t imm11 = extractBits(v: val, begin: 11, end: 11) << 20;
413 uint32_t imm19_12 = extractBits(v: val, begin: 19, end: 12) << 12;
414 insn |= imm20 | imm10_1 | imm11 | imm19_12;
415
416 write32le(P: loc, V: insn);
417 return;
418 }
419
420 case R_RISCV_BRANCH: {
421 checkInt(ctx, loc, v: val, n: 13, rel);
422 checkAlignment(ctx, loc, v: val, n: 2, rel);
423
424 uint32_t insn = read32le(P: loc) & 0x1FFF07F;
425 uint32_t imm12 = extractBits(v: val, begin: 12, end: 12) << 31;
426 uint32_t imm10_5 = extractBits(v: val, begin: 10, end: 5) << 25;
427 uint32_t imm4_1 = extractBits(v: val, begin: 4, end: 1) << 8;
428 uint32_t imm11 = extractBits(v: val, begin: 11, end: 11) << 7;
429 insn |= imm12 | imm10_5 | imm4_1 | imm11;
430
431 write32le(P: loc, V: insn);
432 return;
433 }
434
435 // auipc + jalr pair
436 case R_RISCV_CALL:
437 case R_RISCV_CALL_PLT: {
438 int64_t hi = SignExtend64(X: val + 0x800, B: bits) >> 12;
439 checkInt(ctx, loc, v: hi, n: 20, rel);
440 if (isInt<20>(x: hi)) {
441 relocateNoSym(loc, type: R_RISCV_PCREL_HI20, val);
442 relocateNoSym(loc: loc + 4, type: R_RISCV_PCREL_LO12_I, val);
443 }
444 return;
445 }
446
447 case R_RISCV_GOT_HI20:
448 case R_RISCV_PCREL_HI20:
449 case R_RISCV_TLSDESC_HI20:
450 case R_RISCV_TLS_GD_HI20:
451 case R_RISCV_TLS_GOT_HI20:
452 case R_RISCV_TPREL_HI20:
453 case R_RISCV_HI20: {
454 uint64_t hi = val + 0x800;
455 checkInt(ctx, loc, v: SignExtend64(X: hi, B: bits) >> 12, n: 20, rel);
456 write32le(P: loc, V: (read32le(P: loc) & 0xFFF) | (hi & 0xFFFFF000));
457 return;
458 }
459
460 case R_RISCV_PCREL_LO12_I:
461 case R_RISCV_TLSDESC_LOAD_LO12:
462 case R_RISCV_TLSDESC_ADD_LO12:
463 case R_RISCV_TPREL_LO12_I:
464 case R_RISCV_LO12_I: {
465 uint64_t hi = (val + 0x800) >> 12;
466 uint64_t lo = val - (hi << 12);
467 write32le(P: loc, V: setLO12_I(insn: read32le(P: loc), imm: lo & 0xfff));
468 return;
469 }
470
471 case R_RISCV_PCREL_LO12_S:
472 case R_RISCV_TPREL_LO12_S:
473 case R_RISCV_LO12_S: {
474 uint64_t hi = (val + 0x800) >> 12;
475 uint64_t lo = val - (hi << 12);
476 write32le(P: loc, V: setLO12_S(insn: read32le(P: loc), imm: lo));
477 return;
478 }
479
480 case INTERNAL_R_RISCV_X0REL_I:
481 case INTERNAL_R_RISCV_X0REL_S: {
482 checkInt(ctx, loc, v: val, n: 12, rel);
483 uint32_t insn = (read32le(P: loc) & ~(31 << 15)) | (X_X0 << 15);
484 if (rel.type == INTERNAL_R_RISCV_X0REL_I)
485 insn = setLO12_I(insn, imm: val);
486 else
487 insn = setLO12_S(insn, imm: val);
488 write32le(P: loc, V: insn);
489 return;
490 }
491
492 case INTERNAL_R_RISCV_GPREL_I:
493 case INTERNAL_R_RISCV_GPREL_S: {
494 Defined *gp = ctx.sym.riscvGlobalPointer;
495 int64_t displace = SignExtend64(X: val - gp->getVA(ctx), B: bits);
496 checkInt(ctx, loc, v: displace, n: 12, rel);
497 uint32_t insn = (read32le(P: loc) & ~(31 << 15)) | (X_GP << 15);
498 if (rel.type == INTERNAL_R_RISCV_GPREL_I)
499 insn = setLO12_I(insn, imm: displace);
500 else
501 insn = setLO12_S(insn, imm: displace);
502 write32le(P: loc, V: insn);
503 return;
504 }
505
506 case R_RISCV_ADD8:
507 *loc += val;
508 return;
509 case R_RISCV_ADD16:
510 write16le(P: loc, V: read16le(P: loc) + val);
511 return;
512 case R_RISCV_ADD32:
513 write32le(P: loc, V: read32le(P: loc) + val);
514 return;
515 case R_RISCV_ADD64:
516 write64le(P: loc, V: read64le(P: loc) + val);
517 return;
518 case R_RISCV_SUB6:
519 *loc = (*loc & 0xc0) | (((*loc & 0x3f) - val) & 0x3f);
520 return;
521 case R_RISCV_SUB8:
522 *loc -= val;
523 return;
524 case R_RISCV_SUB16:
525 write16le(P: loc, V: read16le(P: loc) - val);
526 return;
527 case R_RISCV_SUB32:
528 write32le(P: loc, V: read32le(P: loc) - val);
529 return;
530 case R_RISCV_SUB64:
531 write64le(P: loc, V: read64le(P: loc) - val);
532 return;
533 case R_RISCV_SET6:
534 *loc = (*loc & 0xc0) | (val & 0x3f);
535 return;
536 case R_RISCV_SET8:
537 *loc = val;
538 return;
539 case R_RISCV_SET16:
540 write16le(P: loc, V: val);
541 return;
542 case R_RISCV_SET32:
543 case R_RISCV_32_PCREL:
544 case R_RISCV_PLT32:
545 case R_RISCV_GOT32_PCREL:
546 checkInt(ctx, loc, v: val, n: 32, rel);
547 write32le(P: loc, V: val);
548 return;
549
550 case R_RISCV_TLS_DTPREL32:
551 write32le(P: loc, V: val - dtpOffset);
552 break;
553 case R_RISCV_TLS_DTPREL64:
554 write64le(P: loc, V: val - dtpOffset);
555 break;
556
557 case R_RISCV_RELAX:
558 return;
559 case R_RISCV_TLSDESC:
560 // The addend is stored in the second word.
561 if (ctx.arg.is64)
562 write64le(P: loc + 8, V: val);
563 else
564 write32le(P: loc + 4, V: val);
565 break;
566 default:
567 llvm_unreachable("unknown relocation");
568 }
569}
570
571static bool relaxable(ArrayRef<Relocation> relocs, size_t i) {
572 return i + 1 != relocs.size() && relocs[i + 1].type == R_RISCV_RELAX;
573}
574
575static void tlsdescToIe(Ctx &ctx, uint8_t *loc, const Relocation &rel,
576 uint64_t val) {
577 switch (rel.type) {
578 case R_RISCV_TLSDESC_HI20:
579 case R_RISCV_TLSDESC_LOAD_LO12:
580 write32le(P: loc, V: 0x00000013); // nop
581 break;
582 case R_RISCV_TLSDESC_ADD_LO12:
583 write32le(P: loc, V: utype(op: AUIPC, rd: X_A0, imm: hi20(val))); // auipc a0,<hi20>
584 break;
585 case R_RISCV_TLSDESC_CALL:
586 if (ctx.arg.is64)
587 write32le(P: loc, V: itype(op: LD, rd: X_A0, rs1: X_A0, imm: lo12(val))); // ld a0,<lo12>(a0)
588 else
589 write32le(P: loc, V: itype(op: LW, rd: X_A0, rs1: X_A0, imm: lo12(val))); // lw a0,<lo12>(a0)
590 break;
591 default:
592 llvm_unreachable("unsupported relocation for TLSDESC to IE");
593 }
594}
595
596static void tlsdescToLe(uint8_t *loc, const Relocation &rel, uint64_t val) {
597 switch (rel.type) {
598 case R_RISCV_TLSDESC_HI20:
599 case R_RISCV_TLSDESC_LOAD_LO12:
600 write32le(P: loc, V: 0x00000013); // nop
601 return;
602 case R_RISCV_TLSDESC_ADD_LO12:
603 if (isInt<12>(x: val))
604 write32le(P: loc, V: 0x00000013); // nop
605 else
606 write32le(P: loc, V: utype(op: LUI, rd: X_A0, imm: hi20(val))); // lui a0,<hi20>
607 return;
608 case R_RISCV_TLSDESC_CALL:
609 if (isInt<12>(x: val))
610 write32le(P: loc, V: itype(op: ADDI, rd: X_A0, rs1: 0, imm: val)); // addi a0,zero,<lo12>
611 else
612 write32le(P: loc, V: itype(op: ADDI, rd: X_A0, rs1: X_A0, imm: lo12(val))); // addi a0,a0,<lo12>
613 return;
614 default:
615 llvm_unreachable("unsupported relocation for TLSDESC to LE");
616 }
617}
618
619void RISCV::relocateAlloc(InputSection &sec, uint8_t *buf) const {
620 uint64_t secAddr = sec.getOutputSection()->addr + sec.outSecOff;
621 uint64_t tlsdescVal = 0;
622 bool tlsdescRelax = false, isToLe = false;
623 const ArrayRef<Relocation> relocs = sec.relocs();
624 for (size_t i = 0, size = relocs.size(); i != size; ++i) {
625 const Relocation &rel = relocs[i];
626 uint8_t *loc = buf + rel.offset;
627 uint64_t val = sec.getRelocTargetVA(ctx, r: rel, p: secAddr + rel.offset);
628
629 switch (rel.expr) {
630 case R_RELAX_HINT:
631 continue;
632 case R_TLSDESC_PC:
633 // For R_RISCV_TLSDESC_HI20, store &got(sym)-PC to be used by the
634 // following two instructions L[DW] and ADDI.
635 if (rel.type == R_RISCV_TLSDESC_HI20)
636 tlsdescVal = val;
637 else
638 val = tlsdescVal;
639 break;
640 case R_RELAX_TLS_GD_TO_IE:
641 // Only R_RISCV_TLSDESC_HI20 reaches here. tlsdescVal will be finalized
642 // after we see R_RISCV_TLSDESC_ADD_LO12 in the R_RELAX_TLS_GD_TO_LE case.
643 // The net effect is that tlsdescVal will be smaller than `val` to take
644 // into account of NOP instructions (in the absence of R_RISCV_RELAX)
645 // before AUIPC.
646 tlsdescVal = val + rel.offset;
647 isToLe = false;
648 tlsdescRelax = relaxable(relocs, i);
649 if (!tlsdescRelax)
650 tlsdescToIe(ctx, loc, rel, val);
651 continue;
652 case R_RELAX_TLS_GD_TO_LE:
653 // See the comment in handleTlsRelocation. For TLSDESC=>IE,
654 // R_RISCV_TLSDESC_{LOAD_LO12,ADD_LO12,CALL} also reach here. If isToLe is
655 // false, this is actually TLSDESC=>IE optimization.
656 if (rel.type == R_RISCV_TLSDESC_HI20) {
657 tlsdescVal = val;
658 isToLe = true;
659 tlsdescRelax = relaxable(relocs, i);
660 } else {
661 if (!isToLe && rel.type == R_RISCV_TLSDESC_ADD_LO12)
662 tlsdescVal -= rel.offset;
663 val = tlsdescVal;
664 }
665 // When NOP conversion is eligible and relaxation applies, don't write a
666 // NOP in case an unrelated instruction follows the current instruction.
667 if (tlsdescRelax &&
668 (rel.type == R_RISCV_TLSDESC_HI20 ||
669 rel.type == R_RISCV_TLSDESC_LOAD_LO12 ||
670 (rel.type == R_RISCV_TLSDESC_ADD_LO12 && isToLe && !hi20(val))))
671 continue;
672 if (isToLe)
673 tlsdescToLe(loc, rel, val);
674 else
675 tlsdescToIe(ctx, loc, rel, val);
676 continue;
677 case RE_RISCV_LEB128:
678 if (i + 1 < size) {
679 const Relocation &rel1 = relocs[i + 1];
680 if (rel.type == R_RISCV_SET_ULEB128 &&
681 rel1.type == R_RISCV_SUB_ULEB128 && rel.offset == rel1.offset) {
682 auto val = rel.sym->getVA(ctx, addend: rel.addend) -
683 rel1.sym->getVA(ctx, addend: rel1.addend);
684 if (overwriteULEB128(bufLoc: loc, val) >= 0x80)
685 Err(ctx) << sec.getLocation(offset: rel.offset) << ": ULEB128 value " << val
686 << " exceeds available space; references '" << rel.sym
687 << "'";
688 ++i;
689 continue;
690 }
691 }
692 Err(ctx) << sec.getLocation(offset: rel.offset)
693 << ": R_RISCV_SET_ULEB128 not paired with R_RISCV_SUB_SET128";
694 return;
695 default:
696 break;
697 }
698 relocate(loc, rel, val);
699 }
700}
701
702void elf::initSymbolAnchors(Ctx &ctx) {
703 SmallVector<InputSection *, 0> storage;
704 for (OutputSection *osec : ctx.outputSections) {
705 if (!(osec->flags & SHF_EXECINSTR))
706 continue;
707 for (InputSection *sec : getInputSections(os: *osec, storage)) {
708 sec->relaxAux = make<RelaxAux>();
709 if (sec->relocs().size()) {
710 sec->relaxAux->relocDeltas =
711 std::make_unique<uint32_t[]>(num: sec->relocs().size());
712 sec->relaxAux->relocTypes =
713 std::make_unique<RelType[]>(num: sec->relocs().size());
714 }
715 }
716 }
717 // Store symbol anchors for adjusting st_value/st_size during relaxation.
718 // We include symbols where d->file == file for the prevailing copies.
719 //
720 // For a defined symbol foo, we may have `d->file != file` with --wrap=foo.
721 // We should process foo, as the defining object file's symbol table may not
722 // contain foo after redirectSymbols changed the foo entry to __wrap_foo. Use
723 // `d->scriptDefined` to include such symbols.
724 //
725 // `relaxAux->anchors` may contain duplicate symbols, but that is fine.
726 auto addAnchor = [](Defined *d) {
727 if (auto *sec = dyn_cast_or_null<InputSection>(Val: d->section))
728 if (sec->flags & SHF_EXECINSTR && sec->relaxAux) {
729 // If sec is discarded, relaxAux will be nullptr.
730 sec->relaxAux->anchors.push_back(Elt: {.offset: d->value, .d: d, .end: false});
731 sec->relaxAux->anchors.push_back(Elt: {.offset: d->value + d->size, .d: d, .end: true});
732 }
733 };
734 for (InputFile *file : ctx.objectFiles)
735 for (Symbol *sym : file->getSymbols()) {
736 auto *d = dyn_cast<Defined>(Val: sym);
737 if (d && (d->file == file || d->scriptDefined))
738 addAnchor(d);
739 }
740 // Add anchors for IRELATIVE symbols (see `handleNonPreemptibleIfunc`).
741 // Their values must be adjusted so IRELATIVE addends remain correct.
742 for (Defined *d : ctx.irelativeSyms)
743 addAnchor(d);
744 // Sort anchors by offset so that we can find the closest relocation
745 // efficiently. For a zero size symbol, ensure that its start anchor precedes
746 // its end anchor. For two symbols with anchors at the same offset, their
747 // order does not matter.
748 for (OutputSection *osec : ctx.outputSections) {
749 if (!(osec->flags & SHF_EXECINSTR))
750 continue;
751 for (InputSection *sec : getInputSections(os: *osec, storage)) {
752 llvm::sort(C&: sec->relaxAux->anchors, Comp: [](auto &a, auto &b) {
753 return std::make_pair(a.offset, a.end) <
754 std::make_pair(b.offset, b.end);
755 });
756 }
757 }
758}
759
760// Relax R_RISCV_CALL/R_RISCV_CALL_PLT auipc+jalr to c.j, c.jal, or jal.
761static void relaxCall(Ctx &ctx, const InputSection &sec, size_t i, uint64_t loc,
762 Relocation &r, uint32_t &remove) {
763 const bool rvc = getEFlags(ctx, f: sec.file) & EF_RISCV_RVC;
764 const Symbol &sym = *r.sym;
765 const uint64_t insnPair = read64le(P: sec.content().data() + r.offset);
766 const uint32_t rd = extractBits(v: insnPair, begin: 32 + 11, end: 32 + 7);
767 const uint64_t dest =
768 (r.expr == R_PLT_PC ? sym.getPltVA(ctx) : sym.getVA(ctx)) + r.addend;
769 const int64_t displace = dest - loc;
770
771 // When the caller specifies the old value of `remove`, disallow its
772 // increment.
773 if (remove >= 6 && rvc && isInt<12>(x: displace) && rd == X_X0) {
774 sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP;
775 sec.relaxAux->writes.push_back(Elt: 0xa001); // c.j
776 remove = 6;
777 } else if (remove >= 6 && rvc && isInt<12>(x: displace) && rd == X_RA &&
778 !ctx.arg.is64) { // RV32C only
779 sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP;
780 sec.relaxAux->writes.push_back(Elt: 0x2001); // c.jal
781 remove = 6;
782 } else if (remove >= 4 && isInt<21>(x: displace)) {
783 sec.relaxAux->relocTypes[i] = R_RISCV_JAL;
784 sec.relaxAux->writes.push_back(Elt: 0x6f | rd << 7); // jal
785 remove = 4;
786 } else {
787 remove = 0;
788 }
789}
790
791// Relax local-exec TLS when hi20 is zero.
792static void relaxTlsLe(Ctx &ctx, const InputSection &sec, size_t i,
793 uint64_t loc, Relocation &r, uint32_t &remove) {
794 uint64_t val = r.sym->getVA(ctx, addend: r.addend);
795 if (hi20(val) != 0)
796 return;
797 uint32_t insn = read32le(P: sec.content().data() + r.offset);
798 switch (r.type) {
799 case R_RISCV_TPREL_HI20:
800 case R_RISCV_TPREL_ADD:
801 // Remove lui rd, %tprel_hi(x) and add rd, rd, tp, %tprel_add(x).
802 sec.relaxAux->relocTypes[i] = R_RISCV_RELAX;
803 remove = 4;
804 break;
805 case R_RISCV_TPREL_LO12_I:
806 // addi rd, rd, %tprel_lo(x) => addi rd, tp, st_value(x)
807 sec.relaxAux->relocTypes[i] = R_RISCV_32;
808 insn = (insn & ~(31 << 15)) | (X_TP << 15);
809 sec.relaxAux->writes.push_back(Elt: setLO12_I(insn, imm: val));
810 break;
811 case R_RISCV_TPREL_LO12_S:
812 // sw rs, %tprel_lo(x)(rd) => sw rs, st_value(x)(rd)
813 sec.relaxAux->relocTypes[i] = R_RISCV_32;
814 insn = (insn & ~(31 << 15)) | (X_TP << 15);
815 sec.relaxAux->writes.push_back(Elt: setLO12_S(insn, imm: val));
816 break;
817 }
818}
819
820static void relaxHi20Lo12(Ctx &ctx, const InputSection &sec, size_t i,
821 uint64_t loc, Relocation &r, uint32_t &remove) {
822
823 // Fold into use of x0+offset
824 if (isInt<12>(x: r.sym->getVA(ctx, addend: r.addend))) {
825 switch (r.type) {
826 case R_RISCV_HI20:
827 // Remove lui rd, %hi20(x).
828 sec.relaxAux->relocTypes[i] = R_RISCV_RELAX;
829 remove = 4;
830 break;
831 case R_RISCV_LO12_I:
832 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_X0REL_I;
833 break;
834 case R_RISCV_LO12_S:
835 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_X0REL_S;
836 break;
837 }
838 return;
839 }
840
841 const Defined *gp = ctx.sym.riscvGlobalPointer;
842 if (!gp)
843 return;
844
845 if (!isInt<12>(x: r.sym->getVA(ctx, addend: r.addend) - gp->getVA(ctx)))
846 return;
847
848 switch (r.type) {
849 case R_RISCV_HI20:
850 // Remove lui rd, %hi20(x).
851 sec.relaxAux->relocTypes[i] = R_RISCV_RELAX;
852 remove = 4;
853 break;
854 case R_RISCV_LO12_I:
855 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_GPREL_I;
856 break;
857 case R_RISCV_LO12_S:
858 sec.relaxAux->relocTypes[i] = INTERNAL_R_RISCV_GPREL_S;
859 break;
860 }
861}
862
863static bool relax(Ctx &ctx, int pass, InputSection &sec) {
864 const uint64_t secAddr = sec.getVA();
865 const MutableArrayRef<Relocation> relocs = sec.relocs();
866 auto &aux = *sec.relaxAux;
867 bool changed = false;
868 ArrayRef<SymbolAnchor> sa = ArrayRef(aux.anchors);
869 uint64_t delta = 0;
870 bool tlsdescRelax = false, toLeShortForm = false;
871
872 std::fill_n(first: aux.relocTypes.get(), n: relocs.size(), value: R_RISCV_NONE);
873 aux.writes.clear();
874 for (auto [i, r] : llvm::enumerate(First: riscv_vendor_relocs(arr: relocs))) {
875 const uint64_t loc = secAddr + r.offset - delta;
876 uint32_t &cur = aux.relocDeltas[i], remove = 0;
877 switch (r.type) {
878 case R_RISCV_ALIGN: {
879 const uint64_t nextLoc = loc + r.addend;
880 const uint64_t align = PowerOf2Ceil(A: r.addend + 2);
881 // All bytes beyond the alignment boundary should be removed.
882 remove = nextLoc - ((loc + align - 1) & -align);
883 // If we can't satisfy this alignment, we've found a bad input.
884 if (LLVM_UNLIKELY(static_cast<int32_t>(remove) < 0)) {
885 Err(ctx) << getErrorLoc(ctx, loc: (const uint8_t *)loc)
886 << "insufficient padding bytes for " << r.type << ": "
887 << r.addend
888 << " bytes available "
889 "for requested alignment of "
890 << align << " bytes";
891 remove = 0;
892 }
893 break;
894 }
895 case R_RISCV_CALL:
896 case R_RISCV_CALL_PLT:
897 // Prevent oscillation between states by disallowing the increment of
898 // `remove` after a few passes. The previous `remove` value is
899 // `cur-delta`.
900 if (relaxable(relocs, i)) {
901 remove = pass < 4 ? 6 : cur - delta;
902 relaxCall(ctx, sec, i, loc, r, remove);
903 }
904 break;
905 case R_RISCV_TPREL_HI20:
906 case R_RISCV_TPREL_ADD:
907 case R_RISCV_TPREL_LO12_I:
908 case R_RISCV_TPREL_LO12_S:
909 if (relaxable(relocs, i))
910 relaxTlsLe(ctx, sec, i, loc, r, remove);
911 break;
912 case R_RISCV_HI20:
913 case R_RISCV_LO12_I:
914 case R_RISCV_LO12_S:
915 if (relaxable(relocs, i))
916 relaxHi20Lo12(ctx, sec, i, loc, r, remove);
917 break;
918 case R_RISCV_TLSDESC_HI20:
919 // For TLSDESC=>LE, we can use the short form if hi20 is zero.
920 tlsdescRelax = relaxable(relocs, i);
921 toLeShortForm = tlsdescRelax && r.expr == R_RELAX_TLS_GD_TO_LE &&
922 !hi20(val: r.sym->getVA(ctx, addend: r.addend));
923 [[fallthrough]];
924 case R_RISCV_TLSDESC_LOAD_LO12:
925 // For TLSDESC=>LE/IE, AUIPC and L[DW] are removed if relaxable.
926 if (tlsdescRelax && r.expr != R_TLSDESC_PC)
927 remove = 4;
928 break;
929 case R_RISCV_TLSDESC_ADD_LO12:
930 if (toLeShortForm)
931 remove = 4;
932 break;
933 }
934
935 // For all anchors whose offsets are <= r.offset, they are preceded by
936 // the previous relocation whose `relocDeltas` value equals `delta`.
937 // Decrease their st_value and update their st_size.
938 for (; sa.size() && sa[0].offset <= r.offset; sa = sa.slice(N: 1)) {
939 if (sa[0].end)
940 sa[0].d->size = sa[0].offset - delta - sa[0].d->value;
941 else
942 sa[0].d->value = sa[0].offset - delta;
943 }
944 delta += remove;
945 if (delta != cur) {
946 cur = delta;
947 changed = true;
948 }
949 }
950
951 for (const SymbolAnchor &a : sa) {
952 if (a.end)
953 a.d->size = a.offset - delta - a.d->value;
954 else
955 a.d->value = a.offset - delta;
956 }
957 // Inform assignAddresses that the size has changed.
958 if (!isUInt<32>(x: delta))
959 Err(ctx) << "section size decrease is too large: " << delta;
960 sec.bytesDropped = delta;
961 return changed;
962}
963
964// When relaxing just R_RISCV_ALIGN, relocDeltas is usually changed only once in
965// the absence of a linker script. For call and load/store R_RISCV_RELAX, code
966// shrinkage may reduce displacement and make more relocations eligible for
967// relaxation. Code shrinkage may increase displacement to a call/load/store
968// target at a higher fixed address, invalidating an earlier relaxation. Any
969// change in section sizes can have cascading effect and require another
970// relaxation pass.
971bool RISCV::relaxOnce(int pass) const {
972 llvm::TimeTraceScope timeScope("RISC-V relaxOnce");
973 if (pass == 0)
974 initSymbolAnchors(ctx);
975
976 SmallVector<InputSection *, 0> storage;
977 bool changed = false;
978 for (OutputSection *osec : ctx.outputSections) {
979 if (!(osec->flags & SHF_EXECINSTR))
980 continue;
981 for (InputSection *sec : getInputSections(os: *osec, storage))
982 changed |= relax(ctx, pass, sec&: *sec);
983 }
984 return changed;
985}
986
987// If the section alignment is >= 4, advance `dot` to insert NOPs and synthesize
988// an ALIGN relocation. Otherwise, return false to use default handling.
989template <class ELFT, class RelTy>
990bool RISCV::synthesizeAlignForInput(uint64_t &dot, InputSection *sec,
991 Relocs<RelTy> rels) {
992 if (!baseSec) {
993 // Record the first input section with RELAX relocations. We will synthesize
994 // ALIGN relocations here.
995 for (auto rel : rels) {
996 if (rel.getType(false) == R_RISCV_RELAX) {
997 baseSec = sec;
998 break;
999 }
1000 }
1001 } else if (sec->addralign >= 4) {
1002 // If the alignment is >= 4 and the section does not start with an ALIGN
1003 // relocation, synthesize one.
1004 bool hasAlignRel = llvm::any_of(rels, [](const RelTy &rel) {
1005 return rel.r_offset == 0 && rel.getType(false) == R_RISCV_ALIGN;
1006 });
1007 if (!hasAlignRel) {
1008 synthesizedAligns.emplace_back(Args: dot - baseSec->getVA(),
1009 Args: sec->addralign - 2);
1010 dot += sec->addralign - 2;
1011 return true;
1012 }
1013 }
1014 return false;
1015}
1016
1017// Finalize the relocation section by appending synthesized ALIGN relocations
1018// after processing all input sections.
1019template <class ELFT, class RelTy>
1020void RISCV::finalizeSynthesizeAligns(uint64_t &dot, InputSection *sec,
1021 Relocs<RelTy> rels) {
1022 auto *f = cast<ObjFile<ELFT>>(baseSec->file);
1023 auto shdr = f->template getELFShdrs<ELFT>()[baseSec->relSecIdx];
1024 // Create a copy of InputSection.
1025 sec = make<InputSection>(*f, shdr, baseSec->name);
1026 auto *baseRelSec = cast<InputSection>(f->getSections()[baseSec->relSecIdx]);
1027 *sec = *baseRelSec;
1028 baseSec = nullptr;
1029
1030 // Allocate buffer for original and synthesized relocations in RELA format.
1031 // If CREL is used, OutputSection::finalizeNonAllocCrel will convert RELA to
1032 // CREL.
1033 auto newSize = rels.size() + synthesizedAligns.size();
1034 auto *relas = makeThreadLocalN<typename ELFT::Rela>(newSize);
1035 sec->size = newSize * sizeof(typename ELFT::Rela);
1036 sec->content_ = reinterpret_cast<uint8_t *>(relas);
1037 sec->type = SHT_RELA;
1038 // Copy original relocations to the new buffer, potentially converting CREL to
1039 // RELA.
1040 for (auto [i, r] : llvm::enumerate(rels)) {
1041 relas[i].r_offset = r.r_offset;
1042 relas[i].setSymbolAndType(r.getSymbol(0), r.getType(0), false);
1043 if constexpr (RelTy::HasAddend)
1044 relas[i].r_addend = r.r_addend;
1045 }
1046 // Append synthesized ALIGN relocations to the buffer.
1047 for (auto [i, r] : llvm::enumerate(First&: synthesizedAligns)) {
1048 auto &rela = relas[rels.size() + i];
1049 rela.r_offset = r.first;
1050 rela.setSymbolAndType(0, R_RISCV_ALIGN, false);
1051 rela.r_addend = r.second;
1052 }
1053 synthesizedAligns.clear();
1054 // Replace the old relocation section with the new one in the output section.
1055 // addOrphanSections ensures that the output relocation section is processed
1056 // after osec.
1057 for (SectionCommand *cmd : sec->getParent()->commands) {
1058 auto *isd = dyn_cast<InputSectionDescription>(Val: cmd);
1059 if (!isd)
1060 continue;
1061 for (auto *&isec : isd->sections)
1062 if (isec == baseRelSec)
1063 isec = sec;
1064 }
1065}
1066
1067template <class ELFT>
1068bool RISCV::synthesizeAlignAux(uint64_t &dot, InputSection *sec) {
1069 bool ret = false;
1070 if (sec) {
1071 invokeOnRelocs(*sec, ret = synthesizeAlignForInput<ELFT>, dot, sec);
1072 } else if (baseSec) {
1073 invokeOnRelocs(*baseSec, finalizeSynthesizeAligns<ELFT>, dot, sec);
1074 }
1075 return ret;
1076}
1077
1078// Without linker relaxation enabled for a particular relocatable file or
1079// section, the assembler will not generate R_RISCV_ALIGN relocations for
1080// alignment directives. This becomes problematic in a two-stage linking
1081// process: ld -r a.o b.o -o ab.o; ld ab.o -o ab. This function synthesizes an
1082// R_RISCV_ALIGN relocation at section start when needed.
1083//
1084// When called with an input section (`sec` is not null): If the section
1085// alignment is >= 4, advance `dot` to insert NOPs and synthesize an ALIGN
1086// relocation.
1087//
1088// When called after all input sections are processed (`sec` is null): The
1089// output relocation section is updated with all the newly synthesized ALIGN
1090// relocations.
1091bool RISCV::synthesizeAlign(uint64_t &dot, InputSection *sec) {
1092 assert(ctx.arg.relocatable);
1093 if (ctx.arg.is64)
1094 return synthesizeAlignAux<ELF64LE>(dot, sec);
1095 return synthesizeAlignAux<ELF32LE>(dot, sec);
1096}
1097
1098void RISCV::finalizeRelax(int passes) const {
1099 llvm::TimeTraceScope timeScope("Finalize RISC-V relaxation");
1100 Log(ctx) << "relaxation passes: " << passes;
1101 SmallVector<InputSection *, 0> storage;
1102 for (OutputSection *osec : ctx.outputSections) {
1103 if (!(osec->flags & SHF_EXECINSTR))
1104 continue;
1105 for (InputSection *sec : getInputSections(os: *osec, storage)) {
1106 RelaxAux &aux = *sec->relaxAux;
1107 if (!aux.relocDeltas)
1108 continue;
1109
1110 MutableArrayRef<Relocation> rels = sec->relocs();
1111 ArrayRef<uint8_t> old = sec->content();
1112 size_t newSize = old.size() - aux.relocDeltas[rels.size() - 1];
1113 size_t writesIdx = 0;
1114 uint8_t *p = ctx.bAlloc.Allocate<uint8_t>(Num: newSize);
1115 uint64_t offset = 0;
1116 int64_t delta = 0;
1117 sec->content_ = p;
1118 sec->size = newSize;
1119 sec->bytesDropped = 0;
1120
1121 // Update section content: remove NOPs for R_RISCV_ALIGN and rewrite
1122 // instructions for relaxed relocations.
1123 for (size_t i = 0, e = rels.size(); i != e; ++i) {
1124 uint32_t remove = aux.relocDeltas[i] - delta;
1125 delta = aux.relocDeltas[i];
1126 if (remove == 0 && aux.relocTypes[i] == R_RISCV_NONE)
1127 continue;
1128
1129 // Copy from last location to the current relocated location.
1130 const Relocation &r = rels[i];
1131 uint64_t size = r.offset - offset;
1132 memcpy(dest: p, src: old.data() + offset, n: size);
1133 p += size;
1134
1135 // For R_RISCV_ALIGN, we will place `offset` in a location (among NOPs)
1136 // to satisfy the alignment requirement. If both `remove` and r.addend
1137 // are multiples of 4, it is as if we have skipped some NOPs. Otherwise
1138 // we are in the middle of a 4-byte NOP, and we need to rewrite the NOP
1139 // sequence.
1140 int64_t skip = 0;
1141 if (r.type == R_RISCV_ALIGN) {
1142 if (remove % 4 || r.addend % 4) {
1143 skip = r.addend - remove;
1144 int64_t j = 0;
1145 for (; j + 4 <= skip; j += 4)
1146 write32le(P: p + j, V: 0x00000013); // nop
1147 if (j != skip) {
1148 assert(j + 2 == skip);
1149 write16le(P: p + j, V: 0x0001); // c.nop
1150 }
1151 }
1152 } else if (RelType newType = aux.relocTypes[i]) {
1153 switch (newType) {
1154 case INTERNAL_R_RISCV_GPREL_I:
1155 case INTERNAL_R_RISCV_GPREL_S:
1156 case INTERNAL_R_RISCV_X0REL_I:
1157 case INTERNAL_R_RISCV_X0REL_S:
1158 break;
1159 case R_RISCV_RELAX:
1160 // Used by relaxTlsLe to indicate the relocation is ignored.
1161 break;
1162 case R_RISCV_RVC_JUMP:
1163 skip = 2;
1164 write16le(P: p, V: aux.writes[writesIdx++]);
1165 break;
1166 case R_RISCV_JAL:
1167 skip = 4;
1168 write32le(P: p, V: aux.writes[writesIdx++]);
1169 break;
1170 case R_RISCV_32:
1171 // Used by relaxTlsLe to write a uint32_t then suppress the handling
1172 // in relocateAlloc.
1173 skip = 4;
1174 write32le(P: p, V: aux.writes[writesIdx++]);
1175 aux.relocTypes[i] = R_RISCV_NONE;
1176 break;
1177 default:
1178 llvm_unreachable("unsupported type");
1179 }
1180 }
1181
1182 p += skip;
1183 offset = r.offset + skip + remove;
1184 }
1185 memcpy(dest: p, src: old.data() + offset, n: old.size() - offset);
1186
1187 // Subtract the previous relocDeltas value from the relocation offset.
1188 // For a pair of R_RISCV_CALL/R_RISCV_RELAX with the same offset, decrease
1189 // their r_offset by the same delta.
1190 delta = 0;
1191 for (size_t i = 0, e = rels.size(); i != e;) {
1192 uint64_t cur = rels[i].offset;
1193 do {
1194 rels[i].offset -= delta;
1195 if (aux.relocTypes[i] != R_RISCV_NONE)
1196 rels[i].type = aux.relocTypes[i];
1197 } while (++i != e && rels[i].offset == cur);
1198 delta = aux.relocDeltas[i - 1];
1199 }
1200 }
1201 }
1202}
1203
1204namespace {
1205// Representation of the merged .riscv.attributes input sections. The psABI
1206// specifies merge policy for attributes. E.g. if we link an object without an
1207// extension with an object with the extension, the output Tag_RISCV_arch shall
1208// contain the extension. Some tools like objdump parse .riscv.attributes and
1209// disabling some instructions if the first Tag_RISCV_arch does not contain an
1210// extension.
1211class RISCVAttributesSection final : public SyntheticSection {
1212public:
1213 RISCVAttributesSection(Ctx &ctx)
1214 : SyntheticSection(ctx, ".riscv.attributes", SHT_RISCV_ATTRIBUTES, 0, 1) {
1215 }
1216
1217 size_t getSize() const override { return size; }
1218 void writeTo(uint8_t *buf) override;
1219
1220 static constexpr StringRef vendor = "riscv";
1221 DenseMap<unsigned, unsigned> intAttr;
1222 DenseMap<unsigned, StringRef> strAttr;
1223 size_t size = 0;
1224};
1225} // namespace
1226
1227static void mergeArch(Ctx &ctx, RISCVISAUtils::OrderedExtensionMap &mergedExts,
1228 unsigned &mergedXlen, const InputSectionBase *sec,
1229 StringRef s) {
1230 auto maybeInfo = RISCVISAInfo::parseNormalizedArchString(Arch: s);
1231 if (!maybeInfo) {
1232 Err(ctx) << sec << ": " << s << ": " << maybeInfo.takeError();
1233 return;
1234 }
1235
1236 // Merge extensions.
1237 RISCVISAInfo &info = **maybeInfo;
1238 if (mergedExts.empty()) {
1239 mergedExts = info.getExtensions();
1240 mergedXlen = info.getXLen();
1241 } else {
1242 for (const auto &ext : info.getExtensions()) {
1243 auto p = mergedExts.insert(x: ext);
1244 if (!p.second) {
1245 if (std::tie(args&: p.first->second.Major, args&: p.first->second.Minor) <
1246 std::tie(args: ext.second.Major, args: ext.second.Minor))
1247 p.first->second = ext.second;
1248 }
1249 }
1250 }
1251}
1252
1253static void mergeAtomic(Ctx &ctx, DenseMap<unsigned, unsigned>::iterator it,
1254 const InputSectionBase *oldSection,
1255 const InputSectionBase *newSection,
1256 RISCVAttrs::RISCVAtomicAbiTag oldTag,
1257 RISCVAttrs::RISCVAtomicAbiTag newTag) {
1258 using RISCVAttrs::RISCVAtomicAbiTag;
1259 // Same tags stay the same, and UNKNOWN is compatible with anything
1260 if (oldTag == newTag || newTag == RISCVAtomicAbiTag::UNKNOWN)
1261 return;
1262
1263 auto reportAbiError = [&]() {
1264 Err(ctx) << "atomic abi mismatch for " << oldSection->name << "\n>>> "
1265 << oldSection << ": atomic_abi=" << static_cast<unsigned>(oldTag)
1266 << "\n>>> " << newSection
1267 << ": atomic_abi=" << static_cast<unsigned>(newTag);
1268 };
1269
1270 auto reportUnknownAbiError = [&](const InputSectionBase *section,
1271 RISCVAtomicAbiTag tag) {
1272 switch (tag) {
1273 case RISCVAtomicAbiTag::UNKNOWN:
1274 case RISCVAtomicAbiTag::A6C:
1275 case RISCVAtomicAbiTag::A6S:
1276 case RISCVAtomicAbiTag::A7:
1277 return;
1278 };
1279 Err(ctx) << "unknown atomic abi for " << section->name << "\n>>> "
1280 << section << ": atomic_abi=" << static_cast<unsigned>(tag);
1281 };
1282 switch (oldTag) {
1283 case RISCVAtomicAbiTag::UNKNOWN:
1284 it->getSecond() = static_cast<unsigned>(newTag);
1285 return;
1286 case RISCVAtomicAbiTag::A6C:
1287 switch (newTag) {
1288 case RISCVAtomicAbiTag::A6S:
1289 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A6C);
1290 return;
1291 case RISCVAtomicAbiTag::A7:
1292 reportAbiError();
1293 return;
1294 case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN:
1295 case RISCVAttrs::RISCVAtomicAbiTag::A6C:
1296 return;
1297 };
1298 break;
1299
1300 case RISCVAtomicAbiTag::A6S:
1301 switch (newTag) {
1302 case RISCVAtomicAbiTag::A6C:
1303 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A6C);
1304 return;
1305 case RISCVAtomicAbiTag::A7:
1306 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A7);
1307 return;
1308 case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN:
1309 case RISCVAttrs::RISCVAtomicAbiTag::A6S:
1310 return;
1311 };
1312 break;
1313
1314 case RISCVAtomicAbiTag::A7:
1315 switch (newTag) {
1316 case RISCVAtomicAbiTag::A6S:
1317 it->getSecond() = static_cast<unsigned>(RISCVAtomicAbiTag::A7);
1318 return;
1319 case RISCVAtomicAbiTag::A6C:
1320 reportAbiError();
1321 return;
1322 case RISCVAttrs::RISCVAtomicAbiTag::UNKNOWN:
1323 case RISCVAttrs::RISCVAtomicAbiTag::A7:
1324 return;
1325 };
1326 break;
1327 };
1328
1329 // If we get here, then we have an invalid tag, so report it.
1330 // Putting these checks at the end allows us to only do these checks when we
1331 // need to, since this is expected to be a rare occurrence.
1332 reportUnknownAbiError(oldSection, oldTag);
1333 reportUnknownAbiError(newSection, newTag);
1334}
1335
1336static RISCVAttributesSection *
1337mergeAttributesSection(Ctx &ctx,
1338 const SmallVector<InputSectionBase *, 0> &sections) {
1339 using RISCVAttrs::RISCVAtomicAbiTag;
1340 RISCVISAUtils::OrderedExtensionMap exts;
1341 const InputSectionBase *firstStackAlign = nullptr;
1342 const InputSectionBase *firstAtomicAbi = nullptr;
1343 unsigned firstStackAlignValue = 0, xlen = 0;
1344 bool hasArch = false;
1345
1346 ctx.in.riscvAttributes = std::make_unique<RISCVAttributesSection>(args&: ctx);
1347 auto &merged = static_cast<RISCVAttributesSection &>(*ctx.in.riscvAttributes);
1348
1349 // Collect all tags values from attributes section.
1350 const auto &attributesTags = RISCVAttrs::getRISCVAttributeTags();
1351 for (const InputSectionBase *sec : sections) {
1352 RISCVAttributeParser parser;
1353 if (Error e = parser.parse(section: sec->content(), endian: llvm::endianness::little))
1354 Warn(ctx) << sec << ": " << std::move(e);
1355 for (const auto &tag : attributesTags) {
1356 switch (RISCVAttrs::AttrType(tag.attr)) {
1357 // Integer attributes.
1358 case RISCVAttrs::STACK_ALIGN:
1359 if (auto i = parser.getAttributeValue(tag: tag.attr)) {
1360 auto r = merged.intAttr.try_emplace(Key: tag.attr, Args&: *i);
1361 if (r.second) {
1362 firstStackAlign = sec;
1363 firstStackAlignValue = *i;
1364 } else if (r.first->second != *i) {
1365 Err(ctx) << sec << " has stack_align=" << *i << " but "
1366 << firstStackAlign
1367 << " has stack_align=" << firstStackAlignValue;
1368 }
1369 }
1370 continue;
1371 case RISCVAttrs::UNALIGNED_ACCESS:
1372 if (auto i = parser.getAttributeValue(tag: tag.attr))
1373 merged.intAttr[tag.attr] |= *i;
1374 continue;
1375
1376 // String attributes.
1377 case RISCVAttrs::ARCH:
1378 if (auto s = parser.getAttributeString(tag: tag.attr)) {
1379 hasArch = true;
1380 mergeArch(ctx, mergedExts&: exts, mergedXlen&: xlen, sec, s: *s);
1381 }
1382 continue;
1383
1384 // Attributes which use the default handling.
1385 case RISCVAttrs::PRIV_SPEC:
1386 case RISCVAttrs::PRIV_SPEC_MINOR:
1387 case RISCVAttrs::PRIV_SPEC_REVISION:
1388 break;
1389
1390 case RISCVAttrs::AttrType::ATOMIC_ABI:
1391 if (auto i = parser.getAttributeValue(tag: tag.attr)) {
1392 auto r = merged.intAttr.try_emplace(Key: tag.attr, Args&: *i);
1393 if (r.second)
1394 firstAtomicAbi = sec;
1395 else
1396 mergeAtomic(ctx, it: r.first, oldSection: firstAtomicAbi, newSection: sec,
1397 oldTag: static_cast<RISCVAtomicAbiTag>(r.first->getSecond()),
1398 newTag: static_cast<RISCVAtomicAbiTag>(*i));
1399 }
1400 continue;
1401 }
1402
1403 // Fallback for deprecated priv_spec* and other unknown attributes: retain
1404 // the attribute if all input sections agree on the value. GNU ld uses 0
1405 // and empty strings as default values which are not dumped to the output.
1406 // TODO Adjust after resolution to
1407 // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/issues/352
1408 if (tag.attr % 2 == 0) {
1409 if (auto i = parser.getAttributeValue(tag: tag.attr)) {
1410 auto r = merged.intAttr.try_emplace(Key: tag.attr, Args&: *i);
1411 if (!r.second && r.first->second != *i)
1412 r.first->second = 0;
1413 }
1414 } else if (auto s = parser.getAttributeString(tag: tag.attr)) {
1415 auto r = merged.strAttr.try_emplace(Key: tag.attr, Args&: *s);
1416 if (!r.second && r.first->second != *s)
1417 r.first->second = {};
1418 }
1419 }
1420 }
1421
1422 if (hasArch && xlen != 0) {
1423 if (auto result = RISCVISAInfo::createFromExtMap(XLen: xlen, Exts: exts)) {
1424 merged.strAttr.try_emplace(Key: RISCVAttrs::ARCH,
1425 Args: ctx.saver.save(S: (*result)->toString()));
1426 } else {
1427 Err(ctx) << result.takeError();
1428 }
1429 }
1430
1431 // The total size of headers: format-version [ <section-length> "vendor-name"
1432 // [ <file-tag> <size>.
1433 size_t size = 5 + merged.vendor.size() + 1 + 5;
1434 for (auto &attr : merged.intAttr)
1435 if (attr.second != 0)
1436 size += getULEB128Size(Value: attr.first) + getULEB128Size(Value: attr.second);
1437 for (auto &attr : merged.strAttr)
1438 if (!attr.second.empty())
1439 size += getULEB128Size(Value: attr.first) + attr.second.size() + 1;
1440 merged.size = size;
1441 return &merged;
1442}
1443
1444void RISCVAttributesSection::writeTo(uint8_t *buf) {
1445 const size_t size = getSize();
1446 uint8_t *const end = buf + size;
1447 *buf = ELFAttrs::Format_Version;
1448 write32(ctx, p: buf + 1, v: size - 1);
1449 buf += 5;
1450
1451 memcpy(dest: buf, src: vendor.data(), n: vendor.size());
1452 buf += vendor.size() + 1;
1453
1454 *buf = ELFAttrs::File;
1455 write32(ctx, p: buf + 1, v: end - buf);
1456 buf += 5;
1457
1458 for (auto &attr : intAttr) {
1459 if (attr.second == 0)
1460 continue;
1461 buf += encodeULEB128(Value: attr.first, p: buf);
1462 buf += encodeULEB128(Value: attr.second, p: buf);
1463 }
1464 for (auto &attr : strAttr) {
1465 if (attr.second.empty())
1466 continue;
1467 buf += encodeULEB128(Value: attr.first, p: buf);
1468 memcpy(dest: buf, src: attr.second.data(), n: attr.second.size());
1469 buf += attr.second.size() + 1;
1470 }
1471}
1472
1473void elf::mergeRISCVAttributesSections(Ctx &ctx) {
1474 // Find the first input SHT_RISCV_ATTRIBUTES; return if not found.
1475 size_t place =
1476 llvm::find_if(Range&: ctx.inputSections,
1477 P: [](auto *s) { return s->type == SHT_RISCV_ATTRIBUTES; }) -
1478 ctx.inputSections.begin();
1479 if (place == ctx.inputSections.size())
1480 return;
1481
1482 // Extract all SHT_RISCV_ATTRIBUTES sections into `sections`.
1483 SmallVector<InputSectionBase *, 0> sections;
1484 llvm::erase_if(C&: ctx.inputSections, P: [&](InputSectionBase *s) {
1485 if (s->type != SHT_RISCV_ATTRIBUTES)
1486 return false;
1487 sections.push_back(Elt: s);
1488 return true;
1489 });
1490
1491 // Add the merged section.
1492 ctx.inputSections.insert(I: ctx.inputSections.begin() + place,
1493 Elt: mergeAttributesSection(ctx, sections));
1494}
1495
1496void elf::setRISCVTargetInfo(Ctx &ctx) { ctx.target.reset(p: new RISCV(ctx)); }
1497
1498template <class ELFT, class RelTy>
1499void RISCV::scanSectionImpl(InputSectionBase &sec, Relocs<RelTy> rels) {
1500 RelocScan rs(ctx, &sec);
1501 // Many relocations end up in sec.relocations.
1502 sec.relocations.reserve(N: rels.size());
1503
1504 StringRef rvVendor;
1505 for (auto it = rels.begin(); it != rels.end(); ++it) {
1506 RelType type = it->getType(false);
1507 uint32_t symIndex = it->getSymbol(false);
1508 Symbol &sym = sec.getFile<ELFT>()->getSymbol(symIndex);
1509 const uint8_t *loc = sec.content().data() + it->r_offset;
1510
1511 if (type == R_RISCV_VENDOR) {
1512 if (!rvVendor.empty())
1513 Err(ctx) << getErrorLoc(ctx, loc)
1514 << "malformed consecutive R_RISCV_VENDOR relocations";
1515 rvVendor = sym.getName();
1516 continue;
1517 } else if (!rvVendor.empty()) {
1518 uint32_t VendorFlag = getRISCVVendorRelMarker(rvVendor);
1519 if (!VendorFlag) {
1520 Err(ctx) << getErrorLoc(ctx, loc)
1521 << "unknown vendor-specific relocation (" << type.v
1522 << ") in namespace '" << rvVendor << "' against symbol '"
1523 << &sym << "'";
1524 rvVendor = "";
1525 continue;
1526 }
1527
1528 rvVendor = "";
1529 assert((type.v < 256) && "Out of range relocation detected!");
1530 type.v |= VendorFlag;
1531 }
1532
1533 rs.scan<ELFT, RelTy>(it, type, rs.getAddend<ELFT>(*it, type));
1534 }
1535
1536 // Sort relocations by offset for more efficient searching for
1537 // R_RISCV_PCREL_HI20.
1538 llvm::stable_sort(sec.relocs(),
1539 [](const Relocation &lhs, const Relocation &rhs) {
1540 return lhs.offset < rhs.offset;
1541 });
1542}
1543
1544void RISCV::scanSection(InputSectionBase &sec) {
1545 if (ctx.arg.is64)
1546 elf::scanSection1<RISCV, ELF64LE>(target&: *this, sec);
1547 else
1548 elf::scanSection1<RISCV, ELF32LE>(target&: *this, sec);
1549}
1550
1551uint32_t elf::getRISCVVendorRelMarker(StringRef rvVendor) {
1552 return StringSwitch<uint32_t>(rvVendor)
1553 .Case(S: "QUALCOMM", Value: INTERNAL_RISCV_VENDOR_QUALCOMM)
1554 .Case(S: "ANDES", Value: INTERNAL_RISCV_VENDOR_ANDES)
1555 .Default(Value: 0);
1556}
1557
1558std::optional<StringRef> elf::getRISCVVendorString(RelType ty) {
1559 if ((ty.v & INTERNAL_RISCV_VENDOR_MASK) == INTERNAL_RISCV_VENDOR_QUALCOMM)
1560 return "QUALCOMM";
1561 if ((ty.v & INTERNAL_RISCV_VENDOR_MASK) == INTERNAL_RISCV_VENDOR_ANDES)
1562 return "ANDES";
1563 return std::nullopt;
1564}
1565