1//===--------- aarch32.cpp - Generic JITLink arm/thumb utilities ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Generic utilities for graphs representing arm/thumb objects.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ExecutionEngine/JITLink/aarch32.h"
14
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/ExecutionEngine/JITLink/JITLink.h"
17#include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
18#include "llvm/Support/Compiler.h"
19#include "llvm/Support/Endian.h"
20#include "llvm/Support/ManagedStatic.h"
21#include "llvm/Support/MathExtras.h"
22
23#define DEBUG_TYPE "jitlink"
24
25namespace llvm {
26namespace jitlink {
27namespace aarch32 {
28
29/// Check whether the given target flags are set for this Symbol.
30bool hasTargetFlags(Symbol &Sym, TargetFlagsType Flags) {
31 return static_cast<TargetFlagsType>(Sym.getTargetFlags()) & Flags;
32}
33
34/// Encode 22-bit immediate value for branch instructions without J1J2 range
35/// extension (formats B T4, BL T1 and BLX T2).
36///
37/// 00000:Imm11H:Imm11L:0 -> [ 00000:Imm11H, 00000:Imm11L ]
38/// J1^ ^J2 will always be 1
39///
40HalfWords encodeImmBT4BlT1BlxT2(int64_t Value) {
41 constexpr uint32_t J1J2 = 0x2800;
42 uint32_t Imm11H = (Value >> 12) & 0x07ff;
43 uint32_t Imm11L = (Value >> 1) & 0x07ff;
44 return HalfWords{Imm11H, Imm11L | J1J2};
45}
46
47/// Decode 22-bit immediate value for branch instructions without J1J2 range
48/// extension (formats B T4, BL T1 and BLX T2).
49///
50/// [ 00000:Imm11H, 00000:Imm11L ] -> 00000:Imm11H:Imm11L:0
51/// J1^ ^J2 will always be 1
52///
53int64_t decodeImmBT4BlT1BlxT2(uint32_t Hi, uint32_t Lo) {
54 uint32_t Imm11H = Hi & 0x07ff;
55 uint32_t Imm11L = Lo & 0x07ff;
56 return SignExtend64<22>(x: Imm11H << 12 | Imm11L << 1);
57}
58
59/// Encode 25-bit immediate value for branch instructions with J1J2 range
60/// extension (formats B T4, BL T1 and BLX T2).
61///
62/// S:I1:I2:Imm10:Imm11:0 -> [ 00000:S:Imm10, 00:J1:0:J2:Imm11 ]
63///
64LLVM_ABI HalfWords encodeImmBT4BlT1BlxT2_J1J2(int64_t Value) {
65 uint32_t S = (Value >> 14) & 0x0400;
66 uint32_t J1 = (((~(Value >> 10)) ^ (Value >> 11)) & 0x2000);
67 uint32_t J2 = (((~(Value >> 11)) ^ (Value >> 13)) & 0x0800);
68 uint32_t Imm10 = (Value >> 12) & 0x03ff;
69 uint32_t Imm11 = (Value >> 1) & 0x07ff;
70 return HalfWords{S | Imm10, J1 | J2 | Imm11};
71}
72
73/// Decode 25-bit immediate value for branch instructions with J1J2 range
74/// extension (formats B T4, BL T1 and BLX T2).
75///
76/// [ 00000:S:Imm10, 00:J1:0:J2:Imm11] -> S:I1:I2:Imm10:Imm11:0
77///
78LLVM_ABI int64_t decodeImmBT4BlT1BlxT2_J1J2(uint32_t Hi, uint32_t Lo) {
79 uint32_t S = Hi & 0x0400;
80 uint32_t I1 = ~((Lo ^ (Hi << 3)) << 10) & 0x00800000;
81 uint32_t I2 = ~((Lo ^ (Hi << 1)) << 11) & 0x00400000;
82 uint32_t Imm10 = Hi & 0x03ff;
83 uint32_t Imm11 = Lo & 0x07ff;
84 return SignExtend64<25>(x: S << 14 | I1 | I2 | Imm10 << 12 | Imm11 << 1);
85}
86
87/// Encode 26-bit immediate value for branch instructions
88/// (formats B A1, BL A1 and BLX A2).
89///
90/// Imm24:00 -> 00000000:Imm24
91///
92LLVM_ABI uint32_t encodeImmBA1BlA1BlxA2(int64_t Value) {
93 return (Value >> 2) & 0x00ffffff;
94}
95
96/// Decode 26-bit immediate value for branch instructions
97/// (formats B A1, BL A1 and BLX A2).
98///
99/// 00000000:Imm24 -> Imm24:00
100///
101LLVM_ABI int64_t decodeImmBA1BlA1BlxA2(int64_t Value) {
102 return SignExtend64<26>(x: (Value & 0x00ffffff) << 2);
103}
104
105/// Encode 16-bit immediate value for move instruction formats MOVT T1 and
106/// MOVW T3.
107///
108/// Imm4:Imm1:Imm3:Imm8 -> [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ]
109///
110LLVM_ABI HalfWords encodeImmMovtT1MovwT3(uint16_t Value) {
111 uint32_t Imm4 = (Value >> 12) & 0x0f;
112 uint32_t Imm1 = (Value >> 11) & 0x01;
113 uint32_t Imm3 = (Value >> 8) & 0x07;
114 uint32_t Imm8 = Value & 0xff;
115 return HalfWords{Imm1 << 10 | Imm4, Imm3 << 12 | Imm8};
116}
117
118/// Decode 16-bit immediate value from move instruction formats MOVT T1 and
119/// MOVW T3.
120///
121/// [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ] -> Imm4:Imm1:Imm3:Imm8
122///
123LLVM_ABI uint16_t decodeImmMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
124 uint32_t Imm4 = Hi & 0x0f;
125 uint32_t Imm1 = (Hi >> 10) & 0x01;
126 uint32_t Imm3 = (Lo >> 12) & 0x07;
127 uint32_t Imm8 = Lo & 0xff;
128 uint32_t Imm16 = Imm4 << 12 | Imm1 << 11 | Imm3 << 8 | Imm8;
129 assert(Imm16 <= 0xffff && "Decoded value out-of-range");
130 return Imm16;
131}
132
133/// Encode register ID for instruction formats MOVT T1 and MOVW T3.
134///
135/// Rd4 -> [0000000000000000, 0000:Rd4:00000000]
136///
137LLVM_ABI HalfWords encodeRegMovtT1MovwT3(int64_t Value) {
138 uint32_t Rd4 = (Value & 0x0f) << 8;
139 return HalfWords{0, Rd4};
140}
141
142/// Decode register ID from instruction formats MOVT T1 and MOVW T3.
143///
144/// [0000000000000000, 0000:Rd4:00000000] -> Rd4
145///
146LLVM_ABI int64_t decodeRegMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
147 uint32_t Rd4 = (Lo >> 8) & 0x0f;
148 return Rd4;
149}
150
151/// Encode 16-bit immediate value for move instruction formats MOVT A1 and
152/// MOVW A2.
153///
154/// Imm4:Imm12 -> 000000000000:Imm4:0000:Imm12
155///
156LLVM_ABI uint32_t encodeImmMovtA1MovwA2(uint16_t Value) {
157 uint32_t Imm4 = (Value >> 12) & 0x0f;
158 uint32_t Imm12 = Value & 0x0fff;
159 return (Imm4 << 16) | Imm12;
160}
161
162/// Decode 16-bit immediate value for move instruction formats MOVT A1 and
163/// MOVW A2.
164///
165/// 000000000000:Imm4:0000:Imm12 -> Imm4:Imm12
166///
167LLVM_ABI uint16_t decodeImmMovtA1MovwA2(uint64_t Value) {
168 uint32_t Imm4 = (Value >> 16) & 0x0f;
169 uint32_t Imm12 = Value & 0x0fff;
170 return (Imm4 << 12) | Imm12;
171}
172
173/// Encode register ID for instruction formats MOVT A1 and
174/// MOVW A2.
175///
176/// Rd4 -> 0000000000000000:Rd4:000000000000
177///
178LLVM_ABI uint32_t encodeRegMovtA1MovwA2(int64_t Value) {
179 uint32_t Rd4 = (Value & 0x00000f) << 12;
180 return Rd4;
181}
182
183/// Decode register ID for instruction formats MOVT A1 and
184/// MOVW A2.
185///
186/// 0000000000000000:Rd4:000000000000 -> Rd4
187///
188LLVM_ABI int64_t decodeRegMovtA1MovwA2(uint64_t Value) {
189 uint32_t Rd4 = (Value >> 12) & 0x00000f;
190 return Rd4;
191}
192
193namespace {
194
195/// 32-bit Thumb instructions are stored as two little-endian halfwords.
196/// An instruction at address A encodes bytes A+1, A in the first halfword (Hi),
197/// followed by bytes A+3, A+2 in the second halfword (Lo).
198struct WritableThumbRelocation {
199 /// Create a writable reference to a Thumb32 fixup.
200 WritableThumbRelocation(char *FixupPtr)
201 : Hi{*reinterpret_cast<support::ulittle16_t *>(FixupPtr)},
202 Lo{*reinterpret_cast<support::ulittle16_t *>(FixupPtr + 2)} {}
203
204 support::ulittle16_t &Hi; // First halfword
205 support::ulittle16_t &Lo; // Second halfword
206};
207
208struct ThumbRelocation {
209 /// Create a read-only reference to a Thumb32 fixup.
210 ThumbRelocation(const char *FixupPtr)
211 : Hi{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr)},
212 Lo{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr + 2)} {}
213
214 /// Create a read-only Thumb32 fixup from a writeable one.
215 ThumbRelocation(WritableThumbRelocation &Writable)
216 : Hi{Writable.Hi}, Lo(Writable.Lo) {}
217
218 const support::ulittle16_t &Hi; // First halfword
219 const support::ulittle16_t &Lo; // Second halfword
220};
221
222struct WritableArmRelocation {
223 WritableArmRelocation(char *FixupPtr)
224 : Wd{*reinterpret_cast<support::ulittle32_t *>(FixupPtr)} {}
225
226 support::ulittle32_t &Wd;
227};
228
229struct ArmRelocation {
230 ArmRelocation(const char *FixupPtr)
231 : Wd{*reinterpret_cast<const support::ulittle32_t *>(FixupPtr)} {}
232
233 ArmRelocation(WritableArmRelocation &Writable) : Wd{Writable.Wd} {}
234
235 const support::ulittle32_t &Wd;
236};
237
238Error makeUnexpectedOpcodeError(const LinkGraph &G, const ThumbRelocation &R,
239 Edge::Kind Kind) {
240 return make_error<JITLinkError>(
241 Args: formatv(Fmt: "Invalid opcode [ {0:x4}, {1:x4} ] for relocation: {2}",
242 Vals: static_cast<uint16_t>(R.Hi), Vals: static_cast<uint16_t>(R.Lo),
243 Vals: G.getEdgeKindName(K: Kind)));
244}
245
246Error makeUnexpectedOpcodeError(const LinkGraph &G, const ArmRelocation &R,
247 Edge::Kind Kind) {
248 return make_error<JITLinkError>(
249 Args: formatv(Fmt: "Invalid opcode {0:x8} for relocation: {1}",
250 Vals: static_cast<uint32_t>(R.Wd), Vals: G.getEdgeKindName(K: Kind)));
251}
252
253template <EdgeKind_aarch32 K> constexpr bool isArm() {
254 return FirstArmRelocation <= K && K <= LastArmRelocation;
255}
256template <EdgeKind_aarch32 K> constexpr bool isThumb() {
257 return FirstThumbRelocation <= K && K <= LastThumbRelocation;
258}
259
260template <EdgeKind_aarch32 K> static bool checkOpcodeArm(uint32_t Wd) {
261 return (Wd & FixupInfo<K>::OpcodeMask) == FixupInfo<K>::Opcode;
262}
263
264template <EdgeKind_aarch32 K>
265static bool checkOpcodeThumb(uint16_t Hi, uint16_t Lo) {
266 return (Hi & FixupInfo<K>::OpcodeMask.Hi) == FixupInfo<K>::Opcode.Hi &&
267 (Lo & FixupInfo<K>::OpcodeMask.Lo) == FixupInfo<K>::Opcode.Lo;
268}
269
270class FixupInfoTable {
271 static constexpr size_t Items = LastRelocation + 1;
272
273public:
274 FixupInfoTable() {
275 populateEntries<FirstArmRelocation, LastArmRelocation>();
276 populateEntries<FirstThumbRelocation, LastThumbRelocation>();
277 }
278
279 const FixupInfoBase *getEntry(Edge::Kind K) {
280 assert(K < Data.size() && "Index out of bounds");
281 return Data.at(n: K).get();
282 }
283
284private:
285 template <EdgeKind_aarch32 K, EdgeKind_aarch32 LastK> void populateEntries() {
286 assert(K < Data.size() && "Index out of range");
287 assert(Data.at(K) == nullptr && "Initialized entries are immutable");
288 Data[K] = initEntry<K>();
289 if constexpr (K < LastK) {
290 constexpr auto Next = static_cast<EdgeKind_aarch32>(K + 1);
291 populateEntries<Next, LastK>();
292 }
293 }
294
295 template <EdgeKind_aarch32 K>
296 static std::unique_ptr<FixupInfoBase> initEntry() {
297 auto Entry = std::make_unique<FixupInfo<K>>();
298 static_assert(isArm<K>() != isThumb<K>(), "Classes are mutually exclusive");
299 if constexpr (isArm<K>())
300 Entry->checkOpcode = checkOpcodeArm<K>;
301 if constexpr (isThumb<K>())
302 Entry->checkOpcode = checkOpcodeThumb<K>;
303 return Entry;
304 }
305
306private:
307 std::array<std::unique_ptr<FixupInfoBase>, Items> Data;
308};
309
310ManagedStatic<FixupInfoTable> DynFixupInfos;
311
312} // namespace
313
314static Error checkOpcode(LinkGraph &G, const ArmRelocation &R,
315 Edge::Kind Kind) {
316 assert(Kind >= FirstArmRelocation && Kind <= LastArmRelocation &&
317 "Edge kind must be Arm relocation");
318 const FixupInfoBase *Entry = DynFixupInfos->getEntry(K: Kind);
319 const FixupInfoArm &Info = *static_cast<const FixupInfoArm *>(Entry);
320 assert(Info.checkOpcode && "Opcode check is mandatory for Arm edges");
321 if (!Info.checkOpcode(R.Wd))
322 return makeUnexpectedOpcodeError(G, R, Kind);
323
324 return Error::success();
325}
326
327static Error checkOpcode(LinkGraph &G, const ThumbRelocation &R,
328 Edge::Kind Kind) {
329 assert(Kind >= FirstThumbRelocation && Kind <= LastThumbRelocation &&
330 "Edge kind must be Thumb relocation");
331 const FixupInfoBase *Entry = DynFixupInfos->getEntry(K: Kind);
332 const FixupInfoThumb &Info = *static_cast<const FixupInfoThumb *>(Entry);
333 assert(Info.checkOpcode && "Opcode check is mandatory for Thumb edges");
334 if (!Info.checkOpcode(R.Hi, R.Lo))
335 return makeUnexpectedOpcodeError(G, R, Kind);
336
337 return Error::success();
338}
339
340const FixupInfoBase *FixupInfoBase::getDynFixupInfo(Edge::Kind K) {
341 return DynFixupInfos->getEntry(K);
342}
343
344template <EdgeKind_aarch32 Kind>
345bool checkRegister(const ThumbRelocation &R, HalfWords Reg) {
346 uint16_t Hi = R.Hi & FixupInfo<Kind>::RegMask.Hi;
347 uint16_t Lo = R.Lo & FixupInfo<Kind>::RegMask.Lo;
348 return Hi == Reg.Hi && Lo == Reg.Lo;
349}
350
351template <EdgeKind_aarch32 Kind>
352bool checkRegister(const ArmRelocation &R, uint32_t Reg) {
353 uint32_t Wd = R.Wd & FixupInfo<Kind>::RegMask;
354 return Wd == Reg;
355}
356
357template <EdgeKind_aarch32 Kind>
358void writeRegister(WritableThumbRelocation &R, HalfWords Reg) {
359 static constexpr HalfWords Mask = FixupInfo<Kind>::RegMask;
360 assert((Mask.Hi & Reg.Hi) == Reg.Hi && (Mask.Lo & Reg.Lo) == Reg.Lo &&
361 "Value bits exceed bit range of given mask");
362 R.Hi = (R.Hi & ~Mask.Hi) | Reg.Hi;
363 R.Lo = (R.Lo & ~Mask.Lo) | Reg.Lo;
364}
365
366template <EdgeKind_aarch32 Kind>
367void writeRegister(WritableArmRelocation &R, uint32_t Reg) {
368 static constexpr uint32_t Mask = FixupInfo<Kind>::RegMask;
369 assert((Mask & Reg) == Reg && "Value bits exceed bit range of given mask");
370 R.Wd = (R.Wd & ~Mask) | Reg;
371}
372
373template <EdgeKind_aarch32 Kind>
374void writeImmediate(WritableThumbRelocation &R, HalfWords Imm) {
375 static constexpr HalfWords Mask = FixupInfo<Kind>::ImmMask;
376 assert((Mask.Hi & Imm.Hi) == Imm.Hi && (Mask.Lo & Imm.Lo) == Imm.Lo &&
377 "Value bits exceed bit range of given mask");
378 R.Hi = (R.Hi & ~Mask.Hi) | Imm.Hi;
379 R.Lo = (R.Lo & ~Mask.Lo) | Imm.Lo;
380}
381
382template <EdgeKind_aarch32 Kind>
383void writeImmediate(WritableArmRelocation &R, uint32_t Imm) {
384 static constexpr uint32_t Mask = FixupInfo<Kind>::ImmMask;
385 assert((Mask & Imm) == Imm && "Value bits exceed bit range of given mask");
386 R.Wd = (R.Wd & ~Mask) | Imm;
387}
388
389Expected<int64_t> readAddendData(LinkGraph &G, Block &B, Edge::OffsetT Offset,
390 Edge::Kind Kind) {
391 endianness Endian = G.getEndianness();
392 const char *BlockWorkingMem = B.getContent().data();
393 const char *FixupPtr = BlockWorkingMem + Offset;
394
395 switch (Kind) {
396 case Data_Delta32:
397 case Data_Pointer32:
398 case Data_RequestGOTAndTransformToDelta32:
399 return SignExtend64<32>(x: support::endian::read32(P: FixupPtr, E: Endian));
400 case Data_PRel31:
401 return SignExtend64<31>(x: support::endian::read32(P: FixupPtr, E: Endian));
402 default:
403 return make_error<JITLinkError>(
404 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
405 " can not read implicit addend for aarch32 edge kind " +
406 G.getEdgeKindName(K: Kind));
407 }
408}
409
410Expected<int64_t> readAddendArm(LinkGraph &G, Block &B, Edge::OffsetT Offset,
411 Edge::Kind Kind) {
412 ArmRelocation R(B.getContent().data() + Offset);
413 if (Error Err = checkOpcode(G, R, Kind))
414 return std::move(Err);
415
416 switch (Kind) {
417 case Arm_Call:
418 case Arm_Jump24:
419 return decodeImmBA1BlA1BlxA2(Value: R.Wd);
420
421 case Arm_MovtAbs:
422 case Arm_MovwAbsNC:
423 return decodeImmMovtA1MovwA2(Value: R.Wd);
424
425 default:
426 return make_error<JITLinkError>(
427 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
428 " can not read implicit addend for aarch32 edge kind " +
429 G.getEdgeKindName(K: Kind));
430 }
431}
432
433Expected<int64_t> readAddendThumb(LinkGraph &G, Block &B, Edge::OffsetT Offset,
434 Edge::Kind Kind, const ArmConfig &ArmCfg) {
435 ThumbRelocation R(B.getContent().data() + Offset);
436 if (Error Err = checkOpcode(G, R, Kind))
437 return std::move(Err);
438
439 switch (Kind) {
440 case Thumb_Call:
441 case Thumb_Jump24:
442 return LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)
443 ? decodeImmBT4BlT1BlxT2_J1J2(Hi: R.Hi, Lo: R.Lo)
444 : decodeImmBT4BlT1BlxT2(Hi: R.Hi, Lo: R.Lo);
445
446 case Thumb_MovwAbsNC:
447 case Thumb_MovwPrelNC:
448 // Initial addend is interpreted as a signed value
449 return SignExtend64<16>(x: decodeImmMovtT1MovwT3(Hi: R.Hi, Lo: R.Lo));
450
451 case Thumb_MovtAbs:
452 case Thumb_MovtPrel:
453 // Initial addend is interpreted as a signed value
454 return SignExtend64<16>(x: decodeImmMovtT1MovwT3(Hi: R.Hi, Lo: R.Lo));
455
456 default:
457 return make_error<JITLinkError>(
458 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
459 " can not read implicit addend for aarch32 edge kind " +
460 G.getEdgeKindName(K: Kind));
461 }
462}
463
464Error applyFixupData(LinkGraph &G, Block &B, const Edge &E) {
465 using namespace support;
466
467 char *BlockWorkingMem = B.getAlreadyMutableContent().data();
468 char *FixupPtr = BlockWorkingMem + E.getOffset();
469
470 Edge::Kind Kind = E.getKind();
471 uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
472 int64_t Addend = E.getAddend();
473 Symbol &TargetSymbol = E.getTarget();
474 uint64_t TargetAddress = TargetSymbol.getAddress().getValue();
475
476 // Data relocations have alignment 1, size 4 (except R_ARM_ABS8 and
477 // R_ARM_ABS16) and write the full 32-bit result (except R_ARM_PREL31).
478 switch (Kind) {
479 case Data_Delta32: {
480 int64_t Value = TargetAddress - FixupAddress + Addend;
481 if (!isInt<32>(x: Value))
482 return makeTargetOutOfRangeError(G, B, E);
483 if (LLVM_LIKELY(G.getEndianness() == endianness::little))
484 endian::write32le(P: FixupPtr, V: Value);
485 else
486 endian::write32be(P: FixupPtr, V: Value);
487 return Error::success();
488 }
489 case Data_Pointer32: {
490 int64_t Value = TargetAddress + Addend;
491 if (!isUInt<32>(x: Value))
492 return makeTargetOutOfRangeError(G, B, E);
493 if (LLVM_LIKELY(G.getEndianness() == endianness::little))
494 endian::write32le(P: FixupPtr, V: Value);
495 else
496 endian::write32be(P: FixupPtr, V: Value);
497 return Error::success();
498 }
499 case Data_PRel31: {
500 int64_t Value = TargetAddress - FixupAddress + Addend;
501 if (!isInt<31>(x: Value))
502 return makeTargetOutOfRangeError(G, B, E);
503 if (LLVM_LIKELY(G.getEndianness() == endianness::little)) {
504 uint32_t MSB = endian::read32le(P: FixupPtr) & 0x80000000;
505 endian::write32le(P: FixupPtr, V: MSB | (Value & ~0x80000000));
506 } else {
507 uint32_t MSB = endian::read32be(P: FixupPtr) & 0x80000000;
508 endian::write32be(P: FixupPtr, V: MSB | (Value & ~0x80000000));
509 }
510 return Error::success();
511 }
512 case Data_RequestGOTAndTransformToDelta32:
513 llvm_unreachable("Should be transformed");
514 default:
515 return make_error<JITLinkError>(
516 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
517 " encountered unfixable aarch32 edge kind " +
518 G.getEdgeKindName(K: E.getKind()));
519 }
520}
521
522Error applyFixupArm(LinkGraph &G, Block &B, const Edge &E) {
523 WritableArmRelocation R(B.getAlreadyMutableContent().data() + E.getOffset());
524 Edge::Kind Kind = E.getKind();
525 if (Error Err = checkOpcode(G, R, Kind))
526 return Err;
527
528 uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
529 int64_t Addend = E.getAddend();
530 Symbol &TargetSymbol = E.getTarget();
531 uint64_t TargetAddress = TargetSymbol.getAddress().getValue();
532
533 switch (Kind) {
534 case Arm_Jump24: {
535 if (hasTargetFlags(Sym&: TargetSymbol, Flags: ThumbSymbol))
536 return make_error<JITLinkError>(Args: "Branch relocation needs interworking "
537 "stub when bridging to Thumb: " +
538 StringRef(G.getEdgeKindName(K: Kind)));
539
540 int64_t Value = TargetAddress - FixupAddress + Addend;
541
542 if (!isInt<26>(x: Value))
543 return makeTargetOutOfRangeError(G, B, E);
544 writeImmediate<Arm_Jump24>(R, Imm: encodeImmBA1BlA1BlxA2(Value));
545
546 return Error::success();
547 }
548 case Arm_Call: {
549 if ((R.Wd & FixupInfo<Arm_Call>::CondMask) !=
550 FixupInfo<Arm_Call>::Unconditional)
551 return make_error<JITLinkError>(Args: "Relocation expects an unconditional "
552 "BL/BLX branch instruction: " +
553 StringRef(G.getEdgeKindName(K: Kind)));
554
555 int64_t Value = TargetAddress - FixupAddress + Addend;
556
557 // The call instruction itself is Arm. The call destination can either be
558 // Thumb or Arm. We use BL to stay in Arm and BLX to change to Thumb.
559 bool TargetIsThumb = hasTargetFlags(Sym&: TargetSymbol, Flags: ThumbSymbol);
560 bool InstrIsBlx = (~R.Wd & FixupInfo<Arm_Call>::BitBlx) == 0;
561 if (TargetIsThumb != InstrIsBlx) {
562 if (LLVM_LIKELY(TargetIsThumb)) {
563 // Change opcode BL -> BLX
564 R.Wd = R.Wd | FixupInfo<Arm_Call>::BitBlx;
565 R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitH;
566 } else {
567 // Change opcode BLX -> BL
568 R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitBlx;
569 }
570 }
571
572 if (!isInt<26>(x: Value))
573 return makeTargetOutOfRangeError(G, B, E);
574 writeImmediate<Arm_Call>(R, Imm: encodeImmBA1BlA1BlxA2(Value));
575
576 return Error::success();
577 }
578 case Arm_MovwAbsNC: {
579 uint16_t Value = (TargetAddress + Addend) & 0xffff;
580 writeImmediate<Arm_MovwAbsNC>(R, Imm: encodeImmMovtA1MovwA2(Value));
581 return Error::success();
582 }
583 case Arm_MovtAbs: {
584 uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff;
585 writeImmediate<Arm_MovtAbs>(R, Imm: encodeImmMovtA1MovwA2(Value));
586 return Error::success();
587 }
588 default:
589 return make_error<JITLinkError>(
590 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
591 " encountered unfixable aarch32 edge kind " +
592 G.getEdgeKindName(K: E.getKind()));
593 }
594}
595
596Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E,
597 const ArmConfig &ArmCfg) {
598 WritableThumbRelocation R(B.getAlreadyMutableContent().data() +
599 E.getOffset());
600 Edge::Kind Kind = E.getKind();
601 if (Error Err = checkOpcode(G, R, Kind))
602 return Err;
603
604 uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
605 int64_t Addend = E.getAddend();
606 Symbol &TargetSymbol = E.getTarget();
607 uint64_t TargetAddress = TargetSymbol.getAddress().getValue();
608
609 switch (Kind) {
610 case Thumb_Jump24: {
611 if (!hasTargetFlags(Sym&: TargetSymbol, Flags: ThumbSymbol))
612 return make_error<JITLinkError>(Args: "Branch relocation needs interworking "
613 "stub when bridging to ARM: " +
614 StringRef(G.getEdgeKindName(K: Kind)));
615
616 int64_t Value = TargetAddress - FixupAddress + Addend;
617 if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) {
618 if (!isInt<25>(x: Value))
619 return makeTargetOutOfRangeError(G, B, E);
620 writeImmediate<Thumb_Jump24>(R, Imm: encodeImmBT4BlT1BlxT2_J1J2(Value));
621 } else {
622 if (!isInt<22>(x: Value))
623 return makeTargetOutOfRangeError(G, B, E);
624 writeImmediate<Thumb_Jump24>(R, Imm: encodeImmBT4BlT1BlxT2(Value));
625 }
626
627 return Error::success();
628 }
629
630 case Thumb_Call: {
631 int64_t Value = TargetAddress - FixupAddress + Addend;
632
633 // The call instruction itself is Thumb. The call destination can either be
634 // Thumb or Arm. We use BL to stay in Thumb and BLX to change to Arm.
635 bool TargetIsArm = !hasTargetFlags(Sym&: TargetSymbol, Flags: ThumbSymbol);
636 bool InstrIsBlx = (R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) == 0;
637 if (TargetIsArm != InstrIsBlx) {
638 if (LLVM_LIKELY(TargetIsArm)) {
639 // Change opcode BL -> BLX and fix range value: account for 4-byte
640 // aligned destination while instruction may only be 2-byte aligned
641 R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx;
642 R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitH;
643 Value = alignTo(Value, Align: 4);
644 } else {
645 // Change opcode BLX -> BL
646 R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx;
647 }
648 }
649
650 if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) {
651 if (!isInt<25>(x: Value))
652 return makeTargetOutOfRangeError(G, B, E);
653 writeImmediate<Thumb_Call>(R, Imm: encodeImmBT4BlT1BlxT2_J1J2(Value));
654 } else {
655 if (!isInt<22>(x: Value))
656 return makeTargetOutOfRangeError(G, B, E);
657 writeImmediate<Thumb_Call>(R, Imm: encodeImmBT4BlT1BlxT2(Value));
658 }
659
660 assert(((R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) ||
661 (R.Lo & FixupInfo<Thumb_Call>::LoBitH) == 0) &&
662 "Opcode BLX implies H bit is clear (avoid UB in BLX T2)");
663 return Error::success();
664 }
665
666 case Thumb_MovwAbsNC: {
667 uint16_t Value = (TargetAddress + Addend) & 0xffff;
668 writeImmediate<Thumb_MovwAbsNC>(R, Imm: encodeImmMovtT1MovwT3(Value));
669 return Error::success();
670 }
671 case Thumb_MovtAbs: {
672 uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff;
673 writeImmediate<Thumb_MovtAbs>(R, Imm: encodeImmMovtT1MovwT3(Value));
674 return Error::success();
675 }
676 case Thumb_MovwPrelNC: {
677 uint16_t Value = ((TargetAddress + Addend - FixupAddress) & 0xffff);
678 writeImmediate<Thumb_MovwPrelNC>(R, Imm: encodeImmMovtT1MovwT3(Value));
679 return Error::success();
680 }
681 case Thumb_MovtPrel: {
682 uint16_t Value = (((TargetAddress + Addend - FixupAddress) >> 16) & 0xffff);
683 writeImmediate<Thumb_MovtPrel>(R, Imm: encodeImmMovtT1MovwT3(Value));
684 return Error::success();
685 }
686
687 default:
688 return make_error<JITLinkError>(
689 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
690 " encountered unfixable aarch32 edge kind " +
691 G.getEdgeKindName(K: E.getKind()));
692 }
693}
694
695const uint8_t GOTEntryInit[] = {
696 0x00,
697 0x00,
698 0x00,
699 0x00,
700};
701
702/// Create a new node in the link-graph for the given pointer value.
703template <size_t Size>
704static Block &allocPointer(LinkGraph &G, Section &S,
705 const uint8_t (&Content)[Size]) {
706 static_assert(Size == 4, "Pointers are 32-bit");
707 constexpr uint64_t Alignment = 4;
708 ArrayRef<char> Init(reinterpret_cast<const char *>(Content), Size);
709 return G.createContentBlock(Parent&: S, Content: Init, Address: orc::ExecutorAddr(), Alignment, AlignmentOffset: 0);
710}
711
712Symbol &GOTBuilder::createEntry(LinkGraph &G, Symbol &Target) {
713 if (!GOTSection)
714 GOTSection = &G.createSection(Name: getSectionName(), Prot: orc::MemProt::Read);
715 Block &B = allocPointer(G, S&: *GOTSection, Content: GOTEntryInit);
716 constexpr int64_t GOTEntryAddend = 0;
717 B.addEdge(K: Data_Pointer32, Offset: 0, Target, Addend: GOTEntryAddend);
718 return G.addAnonymousSymbol(Content&: B, Offset: 0, Size: B.getSize(), IsCallable: false, IsLive: false);
719}
720
721bool GOTBuilder::visitEdge(LinkGraph &G, Block *B, Edge &E) {
722 Edge::Kind KindToSet = Edge::Invalid;
723 switch (E.getKind()) {
724 case aarch32::Data_RequestGOTAndTransformToDelta32: {
725 KindToSet = aarch32::Data_Delta32;
726 break;
727 }
728 default:
729 return false;
730 }
731 LLVM_DEBUG(dbgs() << " Transforming " << G.getEdgeKindName(E.getKind())
732 << " edge at " << B->getFixupAddress(E) << " ("
733 << B->getAddress() << " + "
734 << formatv("{0:x}", E.getOffset()) << ") into "
735 << G.getEdgeKindName(KindToSet) << "\n");
736 E.setKind(KindToSet);
737 E.setTarget(getEntryForTarget(G, Target&: E.getTarget()));
738 return true;
739}
740
741const uint8_t ArmThumbv5LdrPc[] = {
742 0x78, 0x47, // bx pc
743 0xfd, 0xe7, // b #-6 ; Arm recommended sequence to follow bx pc
744 0x04, 0xf0, 0x1f, 0xe5, // ldr pc, [pc,#-4] ; L1
745 0x00, 0x00, 0x00, 0x00, // L1: .word S
746};
747
748const uint8_t Armv7ABS[] = {
749 0x00, 0xc0, 0x00, 0xe3, // movw r12, #0x0000 ; lower 16-bit
750 0x00, 0xc0, 0x40, 0xe3, // movt r12, #0x0000 ; upper 16-bit
751 0x1c, 0xff, 0x2f, 0xe1 // bx r12
752};
753
754const uint8_t Thumbv7ABS[] = {
755 0x40, 0xf2, 0x00, 0x0c, // movw r12, #0x0000 ; lower 16-bit
756 0xc0, 0xf2, 0x00, 0x0c, // movt r12, #0x0000 ; upper 16-bit
757 0x60, 0x47 // bx r12
758};
759
760/// Create a new node in the link-graph for the given stub template.
761template <size_t Size>
762static Block &allocStub(LinkGraph &G, Section &S, const uint8_t (&Code)[Size]) {
763 constexpr uint64_t Alignment = 4;
764 ArrayRef<char> Template(reinterpret_cast<const char *>(Code), Size);
765 return G.createContentBlock(Parent&: S, Content: Template, Address: orc::ExecutorAddr(), Alignment, AlignmentOffset: 0);
766}
767
768static Block &createStubPrev7(LinkGraph &G, Section &S, Symbol &Target) {
769 Block &B = allocStub(G, S, Code: ArmThumbv5LdrPc);
770 B.addEdge(K: Data_Pointer32, Offset: 8, Target, Addend: 0);
771 return B;
772}
773
774static Block &createStubThumbv7(LinkGraph &G, Section &S, Symbol &Target) {
775 Block &B = allocStub(G, S, Code: Thumbv7ABS);
776 B.addEdge(K: Thumb_MovwAbsNC, Offset: 0, Target, Addend: 0);
777 B.addEdge(K: Thumb_MovtAbs, Offset: 4, Target, Addend: 0);
778
779 [[maybe_unused]] const char *StubPtr = B.getContent().data();
780 [[maybe_unused]] HalfWords Reg12 = encodeRegMovtT1MovwT3(Value: 12);
781 assert(checkRegister<Thumb_MovwAbsNC>(StubPtr, Reg12) &&
782 checkRegister<Thumb_MovtAbs>(StubPtr + 4, Reg12) &&
783 "Linker generated stubs may only corrupt register r12 (IP)");
784 return B;
785}
786
787static Block &createStubArmv7(LinkGraph &G, Section &S, Symbol &Target) {
788 Block &B = allocStub(G, S, Code: Armv7ABS);
789 B.addEdge(K: Arm_MovwAbsNC, Offset: 0, Target, Addend: 0);
790 B.addEdge(K: Arm_MovtAbs, Offset: 4, Target, Addend: 0);
791
792 [[maybe_unused]] const char *StubPtr = B.getContent().data();
793 [[maybe_unused]] uint32_t Reg12 = encodeRegMovtA1MovwA2(Value: 12);
794 assert(checkRegister<Arm_MovwAbsNC>(StubPtr, Reg12) &&
795 checkRegister<Arm_MovtAbs>(StubPtr + 4, Reg12) &&
796 "Linker generated stubs may only corrupt register r12 (IP)");
797 return B;
798}
799
800static bool needsStub(const Edge &E) {
801 Symbol &Target = E.getTarget();
802
803 // Create stubs for external branch targets.
804 if (!Target.isDefined()) {
805 switch (E.getKind()) {
806 case Arm_Call:
807 case Arm_Jump24:
808 case Thumb_Call:
809 case Thumb_Jump24:
810 return true;
811 default:
812 return false;
813 }
814 }
815
816 // For local targets, create interworking stubs if we switch Arm/Thumb with an
817 // instruction that cannot switch the instruction set state natively.
818 bool TargetIsThumb = Target.getTargetFlags() & ThumbSymbol;
819 switch (E.getKind()) {
820 case Arm_Jump24:
821 return TargetIsThumb; // Branch to Thumb needs interworking stub
822 case Thumb_Jump24:
823 return !TargetIsThumb; // Branch to Arm needs interworking stub
824 default:
825 break;
826 }
827
828 return false;
829}
830
831// The ArmThumbv5LdrPc stub has 2 entrypoints: Thumb at offset 0 is taken only
832// for Thumb B instructions. Thumb BL is rewritten to BLX and takes the Arm
833// entrypoint at offset 4. Arm branches always use that one.
834Symbol *StubsManager_prev7::getOrCreateSlotEntrypoint(LinkGraph &G,
835 StubMapEntry &Slot,
836 bool Thumb) {
837 constexpr orc::ExecutorAddrDiff ThumbEntrypointOffset = 0;
838 constexpr orc::ExecutorAddrDiff ArmEntrypointOffset = 4;
839 if (Thumb && !Slot.ThumbEntry) {
840 Slot.ThumbEntry =
841 &G.addAnonymousSymbol(Content&: *Slot.B, Offset: ThumbEntrypointOffset, Size: 4, IsCallable: true, IsLive: false);
842 Slot.ThumbEntry->setTargetFlags(ThumbSymbol);
843 }
844 if (!Thumb && !Slot.ArmEntry)
845 Slot.ArmEntry =
846 &G.addAnonymousSymbol(Content&: *Slot.B, Offset: ArmEntrypointOffset, Size: 8, IsCallable: true, IsLive: false);
847 return Thumb ? Slot.ThumbEntry : Slot.ArmEntry;
848}
849
850bool StubsManager_prev7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
851 if (!needsStub(E))
852 return false;
853
854 Symbol &Target = E.getTarget();
855 assert(Target.hasName() && "Edge cannot point to anonymous target");
856 auto [Slot, NewStub] = getStubMapSlot(Name: *Target.getName());
857
858 if (NewStub) {
859 if (!StubsSection)
860 StubsSection = &G.createSection(Name: getSectionName(),
861 Prot: orc::MemProt::Read | orc::MemProt::Exec);
862 LLVM_DEBUG({
863 dbgs() << " Created stub entry for " << Target.getName() << " in "
864 << StubsSection->getName() << "\n";
865 });
866 Slot->B = &createStubPrev7(G, S&: *StubsSection, Target);
867 }
868
869 // The ArmThumbv5LdrPc stub has 2 entrypoints: Thumb at offset 0 is taken only
870 // for Thumb B instructions. Thumb BL is rewritten to BLX and takes the Arm
871 // entrypoint at offset 4. Arm branches always use that one.
872 bool UseThumb = E.getKind() == Thumb_Jump24;
873 Symbol *StubEntrypoint = getOrCreateSlotEntrypoint(G, Slot&: *Slot, Thumb: UseThumb);
874
875 LLVM_DEBUG({
876 dbgs() << " Using " << (UseThumb ? "Thumb" : "Arm") << " entrypoint "
877 << *StubEntrypoint << " in "
878 << StubEntrypoint->getSection().getName() << "\n";
879 });
880
881 E.setTarget(*StubEntrypoint);
882 return true;
883}
884
885bool StubsManager_v7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
886 if (!needsStub(E))
887 return false;
888
889 // Stub Arm/Thumb follows instruction set state at relocation site.
890 // TODO: We may reduce them at relaxation time and reuse freed slots.
891 bool MakeThumb = (E.getKind() > LastArmRelocation);
892 LLVM_DEBUG(dbgs() << " Preparing " << (MakeThumb ? "Thumb" : "Arm")
893 << " stub for " << G.getEdgeKindName(E.getKind())
894 << " edge at " << B->getFixupAddress(E) << " ("
895 << B->getAddress() << " + "
896 << formatv("{0:x}", E.getOffset()) << ")\n");
897
898 Symbol &Target = E.getTarget();
899 assert(Target.hasName() && "Edge cannot point to anonymous target");
900 Symbol *&StubSymbol = getStubSymbolSlot(Name: *Target.getName(), Thumb: MakeThumb);
901
902 if (!StubSymbol) {
903 if (!StubsSection)
904 StubsSection = &G.createSection(Name: getSectionName(),
905 Prot: orc::MemProt::Read | orc::MemProt::Exec);
906 Block &B = MakeThumb ? createStubThumbv7(G, S&: *StubsSection, Target)
907 : createStubArmv7(G, S&: *StubsSection, Target);
908 StubSymbol = &G.addAnonymousSymbol(Content&: B, Offset: 0, Size: B.getSize(), IsCallable: true, IsLive: false);
909 if (MakeThumb)
910 StubSymbol->setTargetFlags(ThumbSymbol);
911
912 LLVM_DEBUG({
913 dbgs() << " Created " << (MakeThumb ? "Thumb" : "Arm") << " entry for "
914 << Target.getName() << " in " << StubsSection->getName() << ": "
915 << *StubSymbol << "\n";
916 });
917 }
918
919 assert(MakeThumb == (StubSymbol->getTargetFlags() & ThumbSymbol) &&
920 "Instruction set states of stub and relocation site should be equal");
921 LLVM_DEBUG({
922 dbgs() << " Using " << (MakeThumb ? "Thumb" : "Arm") << " entry "
923 << *StubSymbol << " in " << StubSymbol->getSection().getName()
924 << "\n";
925 });
926
927 E.setTarget(*StubSymbol);
928 return true;
929}
930
931const char *getEdgeKindName(Edge::Kind K) {
932#define KIND_NAME_CASE(K) \
933 case K: \
934 return #K;
935
936 switch (K) {
937 KIND_NAME_CASE(Data_Delta32)
938 KIND_NAME_CASE(Data_Pointer32)
939 KIND_NAME_CASE(Data_PRel31)
940 KIND_NAME_CASE(Data_RequestGOTAndTransformToDelta32)
941 KIND_NAME_CASE(Arm_Call)
942 KIND_NAME_CASE(Arm_Jump24)
943 KIND_NAME_CASE(Arm_MovwAbsNC)
944 KIND_NAME_CASE(Arm_MovtAbs)
945 KIND_NAME_CASE(Thumb_Call)
946 KIND_NAME_CASE(Thumb_Jump24)
947 KIND_NAME_CASE(Thumb_MovwAbsNC)
948 KIND_NAME_CASE(Thumb_MovtAbs)
949 KIND_NAME_CASE(Thumb_MovwPrelNC)
950 KIND_NAME_CASE(Thumb_MovtPrel)
951 KIND_NAME_CASE(None)
952 default:
953 return getGenericEdgeKindName(K);
954 }
955#undef KIND_NAME_CASE
956}
957
958const char *getCPUArchName(ARMBuildAttrs::CPUArch K) {
959#define CPUARCH_NAME_CASE(K) \
960 case K: \
961 return #K;
962
963 using namespace ARMBuildAttrs;
964 switch (K) {
965 CPUARCH_NAME_CASE(Pre_v4)
966 CPUARCH_NAME_CASE(v4)
967 CPUARCH_NAME_CASE(v4T)
968 CPUARCH_NAME_CASE(v5T)
969 CPUARCH_NAME_CASE(v5TE)
970 CPUARCH_NAME_CASE(v5TEJ)
971 CPUARCH_NAME_CASE(v6)
972 CPUARCH_NAME_CASE(v6KZ)
973 CPUARCH_NAME_CASE(v6T2)
974 CPUARCH_NAME_CASE(v6K)
975 CPUARCH_NAME_CASE(v7)
976 CPUARCH_NAME_CASE(v6_M)
977 CPUARCH_NAME_CASE(v6S_M)
978 CPUARCH_NAME_CASE(v7E_M)
979 CPUARCH_NAME_CASE(v8_A)
980 CPUARCH_NAME_CASE(v8_R)
981 CPUARCH_NAME_CASE(v8_M_Base)
982 CPUARCH_NAME_CASE(v8_M_Main)
983 CPUARCH_NAME_CASE(v8_1_M_Main)
984 CPUARCH_NAME_CASE(v9_A)
985 }
986 llvm_unreachable("Missing CPUArch in switch?");
987#undef CPUARCH_NAME_CASE
988}
989
990} // namespace aarch32
991} // namespace jitlink
992} // namespace llvm
993