1//===--------- aarch32.cpp - Generic JITLink arm/thumb utilities ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Generic utilities for graphs representing arm/thumb objects.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ExecutionEngine/JITLink/aarch32.h"
14
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/ExecutionEngine/JITLink/JITLink.h"
17#include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
18#include "llvm/Support/Compiler.h"
19#include "llvm/Support/Endian.h"
20#include "llvm/Support/ManagedStatic.h"
21#include "llvm/Support/MathExtras.h"
22
23#define DEBUG_TYPE "jitlink"
24
25namespace llvm {
26namespace jitlink {
27namespace aarch32 {
28
29/// Check whether the given target flags are set for this Symbol.
30bool hasTargetFlags(Symbol &Sym, TargetFlagsType Flags) {
31 return Sym.getTargetFlags() & Flags;
32}
33
34/// Encode 22-bit immediate value for branch instructions without J1J2 range
35/// extension (formats B T4, BL T1 and BLX T2).
36///
37/// 00000:Imm11H:Imm11L:0 -> [ 00000:Imm11H, 00000:Imm11L ]
38/// J1^ ^J2 will always be 1
39///
40HalfWords encodeImmBT4BlT1BlxT2(int64_t Value) {
41 constexpr uint32_t J1J2 = 0x2800;
42 uint32_t Imm11H = (Value >> 12) & 0x07ff;
43 uint32_t Imm11L = (Value >> 1) & 0x07ff;
44 return HalfWords{Imm11H, Imm11L | J1J2};
45}
46
47/// Decode 22-bit immediate value for branch instructions without J1J2 range
48/// extension (formats B T4, BL T1 and BLX T2).
49///
50/// [ 00000:Imm11H, 00000:Imm11L ] -> 00000:Imm11H:Imm11L:0
51/// J1^ ^J2 will always be 1
52///
53int64_t decodeImmBT4BlT1BlxT2(uint32_t Hi, uint32_t Lo) {
54 uint32_t Imm11H = Hi & 0x07ff;
55 uint32_t Imm11L = Lo & 0x07ff;
56 return SignExtend64<22>(x: Imm11H << 12 | Imm11L << 1);
57}
58
59/// Encode 25-bit immediate value for branch instructions with J1J2 range
60/// extension (formats B T4, BL T1 and BLX T2).
61///
62/// S:I1:I2:Imm10:Imm11:0 -> [ 00000:S:Imm10, 00:J1:0:J2:Imm11 ]
63///
64LLVM_ABI HalfWords encodeImmBT4BlT1BlxT2_J1J2(int64_t Value) {
65 uint32_t S = (Value >> 14) & 0x0400;
66 uint32_t J1 = (((~(Value >> 10)) ^ (Value >> 11)) & 0x2000);
67 uint32_t J2 = (((~(Value >> 11)) ^ (Value >> 13)) & 0x0800);
68 uint32_t Imm10 = (Value >> 12) & 0x03ff;
69 uint32_t Imm11 = (Value >> 1) & 0x07ff;
70 return HalfWords{S | Imm10, J1 | J2 | Imm11};
71}
72
73/// Decode 25-bit immediate value for branch instructions with J1J2 range
74/// extension (formats B T4, BL T1 and BLX T2).
75///
76/// [ 00000:S:Imm10, 00:J1:0:J2:Imm11] -> S:I1:I2:Imm10:Imm11:0
77///
78LLVM_ABI int64_t decodeImmBT4BlT1BlxT2_J1J2(uint32_t Hi, uint32_t Lo) {
79 uint32_t S = Hi & 0x0400;
80 uint32_t I1 = ~((Lo ^ (Hi << 3)) << 10) & 0x00800000;
81 uint32_t I2 = ~((Lo ^ (Hi << 1)) << 11) & 0x00400000;
82 uint32_t Imm10 = Hi & 0x03ff;
83 uint32_t Imm11 = Lo & 0x07ff;
84 return SignExtend64<25>(x: S << 14 | I1 | I2 | Imm10 << 12 | Imm11 << 1);
85}
86
87/// Encode 26-bit immediate value for branch instructions
88/// (formats B A1, BL A1 and BLX A2).
89///
90/// Imm24:00 -> 00000000:Imm24
91///
92LLVM_ABI uint32_t encodeImmBA1BlA1BlxA2(int64_t Value) {
93 return (Value >> 2) & 0x00ffffff;
94}
95
96/// Decode 26-bit immediate value for branch instructions
97/// (formats B A1, BL A1 and BLX A2).
98///
99/// 00000000:Imm24 -> Imm24:00
100///
101LLVM_ABI int64_t decodeImmBA1BlA1BlxA2(int64_t Value) {
102 return SignExtend64<26>(x: (Value & 0x00ffffff) << 2);
103}
104
105/// Encode 16-bit immediate value for move instruction formats MOVT T1 and
106/// MOVW T3.
107///
108/// Imm4:Imm1:Imm3:Imm8 -> [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ]
109///
110LLVM_ABI HalfWords encodeImmMovtT1MovwT3(uint16_t Value) {
111 uint32_t Imm4 = (Value >> 12) & 0x0f;
112 uint32_t Imm1 = (Value >> 11) & 0x01;
113 uint32_t Imm3 = (Value >> 8) & 0x07;
114 uint32_t Imm8 = Value & 0xff;
115 return HalfWords{Imm1 << 10 | Imm4, Imm3 << 12 | Imm8};
116}
117
118/// Decode 16-bit immediate value from move instruction formats MOVT T1 and
119/// MOVW T3.
120///
121/// [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ] -> Imm4:Imm1:Imm3:Imm8
122///
123LLVM_ABI uint16_t decodeImmMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
124 uint32_t Imm4 = Hi & 0x0f;
125 uint32_t Imm1 = (Hi >> 10) & 0x01;
126 uint32_t Imm3 = (Lo >> 12) & 0x07;
127 uint32_t Imm8 = Lo & 0xff;
128 uint32_t Imm16 = Imm4 << 12 | Imm1 << 11 | Imm3 << 8 | Imm8;
129 assert(Imm16 <= 0xffff && "Decoded value out-of-range");
130 return Imm16;
131}
132
133/// Encode register ID for instruction formats MOVT T1 and MOVW T3.
134///
135/// Rd4 -> [0000000000000000, 0000:Rd4:00000000]
136///
137LLVM_ABI HalfWords encodeRegMovtT1MovwT3(int64_t Value) {
138 uint32_t Rd4 = (Value & 0x0f) << 8;
139 return HalfWords{0, Rd4};
140}
141
142/// Decode register ID from instruction formats MOVT T1 and MOVW T3.
143///
144/// [0000000000000000, 0000:Rd4:00000000] -> Rd4
145///
146LLVM_ABI int64_t decodeRegMovtT1MovwT3(uint32_t Hi, uint32_t Lo) {
147 uint32_t Rd4 = (Lo >> 8) & 0x0f;
148 return Rd4;
149}
150
151/// Encode 16-bit immediate value for move instruction formats MOVT A1 and
152/// MOVW A2.
153///
154/// Imm4:Imm12 -> 000000000000:Imm4:0000:Imm12
155///
156LLVM_ABI uint32_t encodeImmMovtA1MovwA2(uint16_t Value) {
157 uint32_t Imm4 = (Value >> 12) & 0x0f;
158 uint32_t Imm12 = Value & 0x0fff;
159 return (Imm4 << 16) | Imm12;
160}
161
162/// Decode 16-bit immediate value for move instruction formats MOVT A1 and
163/// MOVW A2.
164///
165/// 000000000000:Imm4:0000:Imm12 -> Imm4:Imm12
166///
167LLVM_ABI uint16_t decodeImmMovtA1MovwA2(uint64_t Value) {
168 uint32_t Imm4 = (Value >> 16) & 0x0f;
169 uint32_t Imm12 = Value & 0x0fff;
170 return (Imm4 << 12) | Imm12;
171}
172
173/// Encode register ID for instruction formats MOVT A1 and
174/// MOVW A2.
175///
176/// Rd4 -> 0000000000000000:Rd4:000000000000
177///
178LLVM_ABI uint32_t encodeRegMovtA1MovwA2(int64_t Value) {
179 uint32_t Rd4 = (Value & 0x00000f) << 12;
180 return Rd4;
181}
182
183/// Decode register ID for instruction formats MOVT A1 and
184/// MOVW A2.
185///
186/// 0000000000000000:Rd4:000000000000 -> Rd4
187///
188LLVM_ABI int64_t decodeRegMovtA1MovwA2(uint64_t Value) {
189 uint32_t Rd4 = (Value >> 12) & 0x00000f;
190 return Rd4;
191}
192
193namespace {
194
195/// 32-bit Thumb instructions are stored as two little-endian halfwords.
196/// An instruction at address A encodes bytes A+1, A in the first halfword (Hi),
197/// followed by bytes A+3, A+2 in the second halfword (Lo).
198struct WritableThumbRelocation {
199 /// Create a writable reference to a Thumb32 fixup.
200 WritableThumbRelocation(char *FixupPtr)
201 : Hi{*reinterpret_cast<support::ulittle16_t *>(FixupPtr)},
202 Lo{*reinterpret_cast<support::ulittle16_t *>(FixupPtr + 2)} {}
203
204 support::ulittle16_t &Hi; // First halfword
205 support::ulittle16_t &Lo; // Second halfword
206};
207
208struct ThumbRelocation {
209 /// Create a read-only reference to a Thumb32 fixup.
210 ThumbRelocation(const char *FixupPtr)
211 : Hi{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr)},
212 Lo{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr + 2)} {}
213
214 /// Create a read-only Thumb32 fixup from a writeable one.
215 ThumbRelocation(WritableThumbRelocation &Writable)
216 : Hi{Writable.Hi}, Lo(Writable.Lo) {}
217
218 const support::ulittle16_t &Hi; // First halfword
219 const support::ulittle16_t &Lo; // Second halfword
220};
221
222struct WritableArmRelocation {
223 WritableArmRelocation(char *FixupPtr)
224 : Wd{*reinterpret_cast<support::ulittle32_t *>(FixupPtr)} {}
225
226 support::ulittle32_t &Wd;
227};
228
229struct ArmRelocation {
230 ArmRelocation(const char *FixupPtr)
231 : Wd{*reinterpret_cast<const support::ulittle32_t *>(FixupPtr)} {}
232
233 ArmRelocation(WritableArmRelocation &Writable) : Wd{Writable.Wd} {}
234
235 const support::ulittle32_t &Wd;
236};
237
238Error makeUnexpectedOpcodeError(const LinkGraph &G, const ThumbRelocation &R,
239 Edge::Kind Kind) {
240 return make_error<JITLinkError>(
241 Args: formatv(Fmt: "Invalid opcode [ {0:x4}, {1:x4} ] for relocation: {2}",
242 Vals: R.Hi.value(), Vals: R.Lo.value(), Vals: G.getEdgeKindName(K: Kind)));
243}
244
245Error makeUnexpectedOpcodeError(const LinkGraph &G, const ArmRelocation &R,
246 Edge::Kind Kind) {
247 return make_error<JITLinkError>(
248 Args: formatv(Fmt: "Invalid opcode {0:x8} for relocation: {1}", Vals: R.Wd.value(),
249 Vals: G.getEdgeKindName(K: Kind)));
250}
251
252template <EdgeKind_aarch32 K> constexpr bool isArm() {
253 return FirstArmRelocation <= K && K <= LastArmRelocation;
254}
255template <EdgeKind_aarch32 K> constexpr bool isThumb() {
256 return FirstThumbRelocation <= K && K <= LastThumbRelocation;
257}
258
259template <EdgeKind_aarch32 K> static bool checkOpcodeArm(uint32_t Wd) {
260 return (Wd & FixupInfo<K>::OpcodeMask) == FixupInfo<K>::Opcode;
261}
262
263template <EdgeKind_aarch32 K>
264static bool checkOpcodeThumb(uint16_t Hi, uint16_t Lo) {
265 return (Hi & FixupInfo<K>::OpcodeMask.Hi) == FixupInfo<K>::Opcode.Hi &&
266 (Lo & FixupInfo<K>::OpcodeMask.Lo) == FixupInfo<K>::Opcode.Lo;
267}
268
269class FixupInfoTable {
270 static constexpr size_t Items = LastRelocation + 1;
271
272public:
273 FixupInfoTable() {
274 populateEntries<FirstArmRelocation, LastArmRelocation>();
275 populateEntries<FirstThumbRelocation, LastThumbRelocation>();
276 }
277
278 const FixupInfoBase *getEntry(Edge::Kind K) {
279 assert(K < Data.size() && "Index out of bounds");
280 return Data.at(n: K).get();
281 }
282
283private:
284 template <EdgeKind_aarch32 K, EdgeKind_aarch32 LastK> void populateEntries() {
285 assert(K < Data.size() && "Index out of range");
286 assert(Data.at(K) == nullptr && "Initialized entries are immutable");
287 Data[K] = initEntry<K>();
288 if constexpr (K < LastK) {
289 constexpr auto Next = static_cast<EdgeKind_aarch32>(K + 1);
290 populateEntries<Next, LastK>();
291 }
292 }
293
294 template <EdgeKind_aarch32 K>
295 static std::unique_ptr<FixupInfoBase> initEntry() {
296 auto Entry = std::make_unique<FixupInfo<K>>();
297 static_assert(isArm<K>() != isThumb<K>(), "Classes are mutually exclusive");
298 if constexpr (isArm<K>())
299 Entry->checkOpcode = checkOpcodeArm<K>;
300 if constexpr (isThumb<K>())
301 Entry->checkOpcode = checkOpcodeThumb<K>;
302 return Entry;
303 }
304
305private:
306 std::array<std::unique_ptr<FixupInfoBase>, Items> Data;
307};
308
309ManagedStatic<FixupInfoTable> DynFixupInfos;
310
311} // namespace
312
313static Error checkOpcode(LinkGraph &G, const ArmRelocation &R,
314 Edge::Kind Kind) {
315 assert(Kind >= FirstArmRelocation && Kind <= LastArmRelocation &&
316 "Edge kind must be Arm relocation");
317 const FixupInfoBase *Entry = DynFixupInfos->getEntry(K: Kind);
318 const FixupInfoArm &Info = *static_cast<const FixupInfoArm *>(Entry);
319 assert(Info.checkOpcode && "Opcode check is mandatory for Arm edges");
320 if (!Info.checkOpcode(R.Wd))
321 return makeUnexpectedOpcodeError(G, R, Kind);
322
323 return Error::success();
324}
325
326static Error checkOpcode(LinkGraph &G, const ThumbRelocation &R,
327 Edge::Kind Kind) {
328 assert(Kind >= FirstThumbRelocation && Kind <= LastThumbRelocation &&
329 "Edge kind must be Thumb relocation");
330 const FixupInfoBase *Entry = DynFixupInfos->getEntry(K: Kind);
331 const FixupInfoThumb &Info = *static_cast<const FixupInfoThumb *>(Entry);
332 assert(Info.checkOpcode && "Opcode check is mandatory for Thumb edges");
333 if (!Info.checkOpcode(R.Hi, R.Lo))
334 return makeUnexpectedOpcodeError(G, R, Kind);
335
336 return Error::success();
337}
338
339const FixupInfoBase *FixupInfoBase::getDynFixupInfo(Edge::Kind K) {
340 return DynFixupInfos->getEntry(K);
341}
342
343template <EdgeKind_aarch32 Kind>
344[[maybe_unused]] bool checkRegister(const ThumbRelocation &R, HalfWords Reg) {
345 uint16_t Hi = R.Hi & FixupInfo<Kind>::RegMask.Hi;
346 uint16_t Lo = R.Lo & FixupInfo<Kind>::RegMask.Lo;
347 return Hi == Reg.Hi && Lo == Reg.Lo;
348}
349
350template <EdgeKind_aarch32 Kind>
351[[maybe_unused]] bool checkRegister(const ArmRelocation &R, uint32_t Reg) {
352 uint32_t Wd = R.Wd & FixupInfo<Kind>::RegMask;
353 return Wd == Reg;
354}
355
356template <EdgeKind_aarch32 Kind>
357void writeImmediate(WritableThumbRelocation &R, HalfWords Imm) {
358 static constexpr HalfWords Mask = FixupInfo<Kind>::ImmMask;
359 assert((Mask.Hi & Imm.Hi) == Imm.Hi && (Mask.Lo & Imm.Lo) == Imm.Lo &&
360 "Value bits exceed bit range of given mask");
361 R.Hi = (R.Hi & ~Mask.Hi) | Imm.Hi;
362 R.Lo = (R.Lo & ~Mask.Lo) | Imm.Lo;
363}
364
365template <EdgeKind_aarch32 Kind>
366void writeImmediate(WritableArmRelocation &R, uint32_t Imm) {
367 static constexpr uint32_t Mask = FixupInfo<Kind>::ImmMask;
368 assert((Mask & Imm) == Imm && "Value bits exceed bit range of given mask");
369 R.Wd = (R.Wd & ~Mask) | Imm;
370}
371
372Expected<int64_t> readAddendData(LinkGraph &G, Block &B, Edge::OffsetT Offset,
373 Edge::Kind Kind) {
374 endianness Endian = G.getEndianness();
375 const char *BlockWorkingMem = B.getContent().data();
376 const char *FixupPtr = BlockWorkingMem + Offset;
377
378 switch (Kind) {
379 case Data_Delta32:
380 case Data_Pointer32:
381 case Data_RequestGOTAndTransformToDelta32:
382 return SignExtend64<32>(x: support::endian::read32(P: FixupPtr, E: Endian));
383 case Data_PRel31:
384 return SignExtend64<31>(x: support::endian::read32(P: FixupPtr, E: Endian));
385 default:
386 return make_error<JITLinkError>(
387 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
388 " can not read implicit addend for aarch32 edge kind " +
389 G.getEdgeKindName(K: Kind));
390 }
391}
392
393Expected<int64_t> readAddendArm(LinkGraph &G, Block &B, Edge::OffsetT Offset,
394 Edge::Kind Kind) {
395 ArmRelocation R(B.getContent().data() + Offset);
396 if (Error Err = checkOpcode(G, R, Kind))
397 return std::move(Err);
398
399 switch (Kind) {
400 case Arm_Call:
401 case Arm_Jump24:
402 return decodeImmBA1BlA1BlxA2(Value: R.Wd);
403
404 case Arm_MovtAbs:
405 case Arm_MovwAbsNC:
406 return decodeImmMovtA1MovwA2(Value: R.Wd);
407
408 default:
409 return make_error<JITLinkError>(
410 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
411 " can not read implicit addend for aarch32 edge kind " +
412 G.getEdgeKindName(K: Kind));
413 }
414}
415
416Expected<int64_t> readAddendThumb(LinkGraph &G, Block &B, Edge::OffsetT Offset,
417 Edge::Kind Kind, const ArmConfig &ArmCfg) {
418 ThumbRelocation R(B.getContent().data() + Offset);
419 if (Error Err = checkOpcode(G, R, Kind))
420 return std::move(Err);
421
422 switch (Kind) {
423 case Thumb_Call:
424 case Thumb_Jump24:
425 return LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)
426 ? decodeImmBT4BlT1BlxT2_J1J2(Hi: R.Hi, Lo: R.Lo)
427 : decodeImmBT4BlT1BlxT2(Hi: R.Hi, Lo: R.Lo);
428
429 case Thumb_MovwAbsNC:
430 case Thumb_MovwPrelNC:
431 // Initial addend is interpreted as a signed value
432 return SignExtend64<16>(x: decodeImmMovtT1MovwT3(Hi: R.Hi, Lo: R.Lo));
433
434 case Thumb_MovtAbs:
435 case Thumb_MovtPrel:
436 // Initial addend is interpreted as a signed value
437 return SignExtend64<16>(x: decodeImmMovtT1MovwT3(Hi: R.Hi, Lo: R.Lo));
438
439 default:
440 return make_error<JITLinkError>(
441 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
442 " can not read implicit addend for aarch32 edge kind " +
443 G.getEdgeKindName(K: Kind));
444 }
445}
446
447Error applyFixupData(LinkGraph &G, Block &B, const Edge &E) {
448 using namespace support;
449
450 char *BlockWorkingMem = B.getAlreadyMutableContent().data();
451 char *FixupPtr = BlockWorkingMem + E.getOffset();
452
453 Edge::Kind Kind = E.getKind();
454 uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
455 int64_t Addend = E.getAddend();
456 Symbol &TargetSymbol = E.getTarget();
457 uint64_t TargetAddress = TargetSymbol.getAddress().getValue();
458
459 // Data relocations have alignment 1, size 4 (except R_ARM_ABS8 and
460 // R_ARM_ABS16) and write the full 32-bit result (except R_ARM_PREL31).
461 switch (Kind) {
462 case Data_Delta32: {
463 int64_t Value = TargetAddress - FixupAddress + Addend;
464 if (!isInt<32>(x: Value))
465 return makeTargetOutOfRangeError(G, B, E);
466 if (LLVM_LIKELY(G.getEndianness() == endianness::little))
467 endian::write32le(P: FixupPtr, V: Value);
468 else
469 endian::write32be(P: FixupPtr, V: Value);
470 return Error::success();
471 }
472 case Data_Pointer32: {
473 int64_t Value = TargetAddress + Addend;
474 if (!isUInt<32>(x: Value))
475 return makeTargetOutOfRangeError(G, B, E);
476 if (LLVM_LIKELY(G.getEndianness() == endianness::little))
477 endian::write32le(P: FixupPtr, V: Value);
478 else
479 endian::write32be(P: FixupPtr, V: Value);
480 return Error::success();
481 }
482 case Data_PRel31: {
483 int64_t Value = TargetAddress - FixupAddress + Addend;
484 if (!isInt<31>(x: Value))
485 return makeTargetOutOfRangeError(G, B, E);
486 if (LLVM_LIKELY(G.getEndianness() == endianness::little)) {
487 uint32_t MSB = endian::read32le(P: FixupPtr) & 0x80000000;
488 endian::write32le(P: FixupPtr, V: MSB | (Value & ~0x80000000));
489 } else {
490 uint32_t MSB = endian::read32be(P: FixupPtr) & 0x80000000;
491 endian::write32be(P: FixupPtr, V: MSB | (Value & ~0x80000000));
492 }
493 return Error::success();
494 }
495 case Data_RequestGOTAndTransformToDelta32:
496 llvm_unreachable("Should be transformed");
497 default:
498 return make_error<JITLinkError>(
499 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
500 " encountered unfixable aarch32 edge kind " +
501 G.getEdgeKindName(K: E.getKind()));
502 }
503}
504
505Error applyFixupArm(LinkGraph &G, Block &B, const Edge &E) {
506 WritableArmRelocation R(B.getAlreadyMutableContent().data() + E.getOffset());
507 Edge::Kind Kind = E.getKind();
508 if (Error Err = checkOpcode(G, R, Kind))
509 return Err;
510
511 uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
512 int64_t Addend = E.getAddend();
513 Symbol &TargetSymbol = E.getTarget();
514 uint64_t TargetAddress = TargetSymbol.getAddress().getValue();
515
516 switch (Kind) {
517 case Arm_Jump24: {
518 if (hasTargetFlags(Sym&: TargetSymbol, Flags: ThumbSymbol))
519 return make_error<JITLinkError>(Args: "Branch relocation needs interworking "
520 "stub when bridging to Thumb: " +
521 StringRef(G.getEdgeKindName(K: Kind)));
522
523 int64_t Value = TargetAddress - FixupAddress + Addend;
524
525 if (!isInt<26>(x: Value))
526 return makeTargetOutOfRangeError(G, B, E);
527 writeImmediate<Arm_Jump24>(R, Imm: encodeImmBA1BlA1BlxA2(Value));
528
529 return Error::success();
530 }
531 case Arm_Call: {
532 if ((R.Wd & FixupInfo<Arm_Call>::CondMask) !=
533 FixupInfo<Arm_Call>::Unconditional)
534 return make_error<JITLinkError>(Args: "Relocation expects an unconditional "
535 "BL/BLX branch instruction: " +
536 StringRef(G.getEdgeKindName(K: Kind)));
537
538 int64_t Value = TargetAddress - FixupAddress + Addend;
539
540 // The call instruction itself is Arm. The call destination can either be
541 // Thumb or Arm. We use BL to stay in Arm and BLX to change to Thumb.
542 bool TargetIsThumb = hasTargetFlags(Sym&: TargetSymbol, Flags: ThumbSymbol);
543 bool InstrIsBlx = (~R.Wd & FixupInfo<Arm_Call>::BitBlx) == 0;
544 if (TargetIsThumb != InstrIsBlx) {
545 if (LLVM_LIKELY(TargetIsThumb)) {
546 // Change opcode BL -> BLX
547 R.Wd = R.Wd | FixupInfo<Arm_Call>::BitBlx;
548 R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitH;
549 } else {
550 // Change opcode BLX -> BL
551 R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitBlx;
552 }
553 }
554
555 if (!isInt<26>(x: Value))
556 return makeTargetOutOfRangeError(G, B, E);
557 writeImmediate<Arm_Call>(R, Imm: encodeImmBA1BlA1BlxA2(Value));
558
559 return Error::success();
560 }
561 case Arm_MovwAbsNC: {
562 uint16_t Value = (TargetAddress + Addend) & 0xffff;
563 writeImmediate<Arm_MovwAbsNC>(R, Imm: encodeImmMovtA1MovwA2(Value));
564 return Error::success();
565 }
566 case Arm_MovtAbs: {
567 uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff;
568 writeImmediate<Arm_MovtAbs>(R, Imm: encodeImmMovtA1MovwA2(Value));
569 return Error::success();
570 }
571 default:
572 return make_error<JITLinkError>(
573 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
574 " encountered unfixable aarch32 edge kind " +
575 G.getEdgeKindName(K: E.getKind()));
576 }
577}
578
579Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E,
580 const ArmConfig &ArmCfg) {
581 WritableThumbRelocation R(B.getAlreadyMutableContent().data() +
582 E.getOffset());
583 Edge::Kind Kind = E.getKind();
584 if (Error Err = checkOpcode(G, R, Kind))
585 return Err;
586
587 uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue();
588 int64_t Addend = E.getAddend();
589 Symbol &TargetSymbol = E.getTarget();
590 uint64_t TargetAddress = TargetSymbol.getAddress().getValue();
591
592 switch (Kind) {
593 case Thumb_Jump24: {
594 if (!hasTargetFlags(Sym&: TargetSymbol, Flags: ThumbSymbol))
595 return make_error<JITLinkError>(Args: "Branch relocation needs interworking "
596 "stub when bridging to ARM: " +
597 StringRef(G.getEdgeKindName(K: Kind)));
598
599 int64_t Value = TargetAddress - FixupAddress + Addend;
600 if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) {
601 if (!isInt<25>(x: Value))
602 return makeTargetOutOfRangeError(G, B, E);
603 writeImmediate<Thumb_Jump24>(R, Imm: encodeImmBT4BlT1BlxT2_J1J2(Value));
604 } else {
605 if (!isInt<22>(x: Value))
606 return makeTargetOutOfRangeError(G, B, E);
607 writeImmediate<Thumb_Jump24>(R, Imm: encodeImmBT4BlT1BlxT2(Value));
608 }
609
610 return Error::success();
611 }
612
613 case Thumb_Call: {
614 int64_t Value = TargetAddress - FixupAddress + Addend;
615
616 // The call instruction itself is Thumb. The call destination can either be
617 // Thumb or Arm. We use BL to stay in Thumb and BLX to change to Arm.
618 bool TargetIsArm = !hasTargetFlags(Sym&: TargetSymbol, Flags: ThumbSymbol);
619 bool InstrIsBlx = (R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) == 0;
620 if (TargetIsArm != InstrIsBlx) {
621 if (LLVM_LIKELY(TargetIsArm)) {
622 // Change opcode BL -> BLX and fix range value: account for 4-byte
623 // aligned destination while instruction may only be 2-byte aligned
624 R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx;
625 R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitH;
626 Value = alignTo(Value, Align: 4);
627 } else {
628 // Change opcode BLX -> BL
629 R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx;
630 }
631 }
632
633 if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) {
634 if (!isInt<25>(x: Value))
635 return makeTargetOutOfRangeError(G, B, E);
636 writeImmediate<Thumb_Call>(R, Imm: encodeImmBT4BlT1BlxT2_J1J2(Value));
637 } else {
638 if (!isInt<22>(x: Value))
639 return makeTargetOutOfRangeError(G, B, E);
640 writeImmediate<Thumb_Call>(R, Imm: encodeImmBT4BlT1BlxT2(Value));
641 }
642
643 assert(((R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) ||
644 (R.Lo & FixupInfo<Thumb_Call>::LoBitH) == 0) &&
645 "Opcode BLX implies H bit is clear (avoid UB in BLX T2)");
646 return Error::success();
647 }
648
649 case Thumb_MovwAbsNC: {
650 uint16_t Value = (TargetAddress + Addend) & 0xffff;
651 writeImmediate<Thumb_MovwAbsNC>(R, Imm: encodeImmMovtT1MovwT3(Value));
652 return Error::success();
653 }
654 case Thumb_MovtAbs: {
655 uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff;
656 writeImmediate<Thumb_MovtAbs>(R, Imm: encodeImmMovtT1MovwT3(Value));
657 return Error::success();
658 }
659 case Thumb_MovwPrelNC: {
660 uint16_t Value = ((TargetAddress + Addend - FixupAddress) & 0xffff);
661 writeImmediate<Thumb_MovwPrelNC>(R, Imm: encodeImmMovtT1MovwT3(Value));
662 return Error::success();
663 }
664 case Thumb_MovtPrel: {
665 uint16_t Value = (((TargetAddress + Addend - FixupAddress) >> 16) & 0xffff);
666 writeImmediate<Thumb_MovtPrel>(R, Imm: encodeImmMovtT1MovwT3(Value));
667 return Error::success();
668 }
669
670 default:
671 return make_error<JITLinkError>(
672 Args: "In graph " + G.getName() + ", section " + B.getSection().getName() +
673 " encountered unfixable aarch32 edge kind " +
674 G.getEdgeKindName(K: E.getKind()));
675 }
676}
677
678const uint8_t GOTEntryInit[] = {
679 0x00,
680 0x00,
681 0x00,
682 0x00,
683};
684
685/// Create a new node in the link-graph for the given pointer value.
686template <size_t Size>
687static Block &allocPointer(LinkGraph &G, Section &S,
688 const uint8_t (&Content)[Size]) {
689 static_assert(Size == 4, "Pointers are 32-bit");
690 constexpr uint64_t Alignment = 4;
691 ArrayRef<char> Init(reinterpret_cast<const char *>(Content), Size);
692 return G.createContentBlock(Parent&: S, Content: Init, Address: orc::ExecutorAddr(), Alignment, AlignmentOffset: 0);
693}
694
695Symbol &GOTBuilder::createEntry(LinkGraph &G, Symbol &Target) {
696 if (!GOTSection)
697 GOTSection = &G.createSection(Name: getSectionName(), Prot: orc::MemProt::Read);
698 Block &B = allocPointer(G, S&: *GOTSection, Content: GOTEntryInit);
699 constexpr int64_t GOTEntryAddend = 0;
700 B.addEdge(K: Data_Pointer32, Offset: 0, Target, Addend: GOTEntryAddend);
701 return G.addAnonymousSymbol(Content&: B, Offset: 0, Size: B.getSize(), IsCallable: false, IsLive: false);
702}
703
704bool GOTBuilder::visitEdge(LinkGraph &G, Block *B, Edge &E) {
705 Edge::Kind KindToSet = Edge::Invalid;
706 switch (E.getKind()) {
707 case aarch32::Data_RequestGOTAndTransformToDelta32: {
708 KindToSet = aarch32::Data_Delta32;
709 break;
710 }
711 default:
712 return false;
713 }
714 LLVM_DEBUG(dbgs() << " Transforming " << G.getEdgeKindName(E.getKind())
715 << " edge at " << B->getFixupAddress(E) << " ("
716 << B->getAddress() << " + "
717 << formatv("{0:x}", E.getOffset()) << ") into "
718 << G.getEdgeKindName(KindToSet) << "\n");
719 E.setKind(KindToSet);
720 E.setTarget(getEntryForTarget(G, Target&: E.getTarget()));
721 return true;
722}
723
724const uint8_t ArmThumbv5LdrPc[] = {
725 0x78, 0x47, // bx pc
726 0xfd, 0xe7, // b #-6 ; Arm recommended sequence to follow bx pc
727 0x04, 0xf0, 0x1f, 0xe5, // ldr pc, [pc,#-4] ; L1
728 0x00, 0x00, 0x00, 0x00, // L1: .word S
729};
730
731const uint8_t Armv7ABS[] = {
732 0x00, 0xc0, 0x00, 0xe3, // movw r12, #0x0000 ; lower 16-bit
733 0x00, 0xc0, 0x40, 0xe3, // movt r12, #0x0000 ; upper 16-bit
734 0x1c, 0xff, 0x2f, 0xe1 // bx r12
735};
736
737const uint8_t Thumbv7ABS[] = {
738 0x40, 0xf2, 0x00, 0x0c, // movw r12, #0x0000 ; lower 16-bit
739 0xc0, 0xf2, 0x00, 0x0c, // movt r12, #0x0000 ; upper 16-bit
740 0x60, 0x47 // bx r12
741};
742
743/// Create a new node in the link-graph for the given stub template.
744template <size_t Size>
745static Block &allocStub(LinkGraph &G, Section &S, const uint8_t (&Code)[Size]) {
746 constexpr uint64_t Alignment = 4;
747 ArrayRef<char> Template(reinterpret_cast<const char *>(Code), Size);
748 return G.createContentBlock(Parent&: S, Content: Template, Address: orc::ExecutorAddr(), Alignment, AlignmentOffset: 0);
749}
750
751static Block &createStubPrev7(LinkGraph &G, Section &S, Symbol &Target) {
752 Block &B = allocStub(G, S, Code: ArmThumbv5LdrPc);
753 B.addEdge(K: Data_Pointer32, Offset: 8, Target, Addend: 0);
754 return B;
755}
756
757static Block &createStubThumbv7(LinkGraph &G, Section &S, Symbol &Target) {
758 Block &B = allocStub(G, S, Code: Thumbv7ABS);
759 B.addEdge(K: Thumb_MovwAbsNC, Offset: 0, Target, Addend: 0);
760 B.addEdge(K: Thumb_MovtAbs, Offset: 4, Target, Addend: 0);
761
762 [[maybe_unused]] const char *StubPtr = B.getContent().data();
763 [[maybe_unused]] HalfWords Reg12 = encodeRegMovtT1MovwT3(Value: 12);
764 assert(checkRegister<Thumb_MovwAbsNC>(StubPtr, Reg12) &&
765 checkRegister<Thumb_MovtAbs>(StubPtr + 4, Reg12) &&
766 "Linker generated stubs may only corrupt register r12 (IP)");
767 return B;
768}
769
770static Block &createStubArmv7(LinkGraph &G, Section &S, Symbol &Target) {
771 Block &B = allocStub(G, S, Code: Armv7ABS);
772 B.addEdge(K: Arm_MovwAbsNC, Offset: 0, Target, Addend: 0);
773 B.addEdge(K: Arm_MovtAbs, Offset: 4, Target, Addend: 0);
774
775 [[maybe_unused]] const char *StubPtr = B.getContent().data();
776 [[maybe_unused]] uint32_t Reg12 = encodeRegMovtA1MovwA2(Value: 12);
777 assert(checkRegister<Arm_MovwAbsNC>(StubPtr, Reg12) &&
778 checkRegister<Arm_MovtAbs>(StubPtr + 4, Reg12) &&
779 "Linker generated stubs may only corrupt register r12 (IP)");
780 return B;
781}
782
783static bool needsStub(const Edge &E) {
784 Symbol &Target = E.getTarget();
785
786 // Create stubs for external branch targets.
787 if (!Target.isDefined()) {
788 switch (E.getKind()) {
789 case Arm_Call:
790 case Arm_Jump24:
791 case Thumb_Call:
792 case Thumb_Jump24:
793 return true;
794 default:
795 return false;
796 }
797 }
798
799 // For local targets, create interworking stubs if we switch Arm/Thumb with an
800 // instruction that cannot switch the instruction set state natively.
801 bool TargetIsThumb = Target.getTargetFlags() & ThumbSymbol;
802 switch (E.getKind()) {
803 case Arm_Jump24:
804 return TargetIsThumb; // Branch to Thumb needs interworking stub
805 case Thumb_Jump24:
806 return !TargetIsThumb; // Branch to Arm needs interworking stub
807 default:
808 break;
809 }
810
811 return false;
812}
813
814// The ArmThumbv5LdrPc stub has 2 entrypoints: Thumb at offset 0 is taken only
815// for Thumb B instructions. Thumb BL is rewritten to BLX and takes the Arm
816// entrypoint at offset 4. Arm branches always use that one.
817Symbol *StubsManager_prev7::getOrCreateSlotEntrypoint(LinkGraph &G,
818 StubMapEntry &Slot,
819 bool Thumb) {
820 constexpr orc::ExecutorAddrDiff ThumbEntrypointOffset = 0;
821 constexpr orc::ExecutorAddrDiff ArmEntrypointOffset = 4;
822 if (Thumb && !Slot.ThumbEntry) {
823 Slot.ThumbEntry =
824 &G.addAnonymousSymbol(Content&: *Slot.B, Offset: ThumbEntrypointOffset, Size: 4, IsCallable: true, IsLive: false);
825 Slot.ThumbEntry->setTargetFlags(ThumbSymbol);
826 }
827 if (!Thumb && !Slot.ArmEntry)
828 Slot.ArmEntry =
829 &G.addAnonymousSymbol(Content&: *Slot.B, Offset: ArmEntrypointOffset, Size: 8, IsCallable: true, IsLive: false);
830 return Thumb ? Slot.ThumbEntry : Slot.ArmEntry;
831}
832
833bool StubsManager_prev7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
834 if (!needsStub(E))
835 return false;
836
837 Symbol &Target = E.getTarget();
838 assert(Target.hasName() && "Edge cannot point to anonymous target");
839 auto [Slot, NewStub] = getStubMapSlot(Name: *Target.getName());
840
841 if (NewStub) {
842 if (!StubsSection)
843 StubsSection = &G.createSection(Name: getSectionName(),
844 Prot: orc::MemProt::Read | orc::MemProt::Exec);
845 LLVM_DEBUG({
846 dbgs() << " Created stub entry for " << Target.getName() << " in "
847 << StubsSection->getName() << "\n";
848 });
849 Slot->B = &createStubPrev7(G, S&: *StubsSection, Target);
850 }
851
852 // The ArmThumbv5LdrPc stub has 2 entrypoints: Thumb at offset 0 is taken only
853 // for Thumb B instructions. Thumb BL is rewritten to BLX and takes the Arm
854 // entrypoint at offset 4. Arm branches always use that one.
855 bool UseThumb = E.getKind() == Thumb_Jump24;
856 Symbol *StubEntrypoint = getOrCreateSlotEntrypoint(G, Slot&: *Slot, Thumb: UseThumb);
857
858 LLVM_DEBUG({
859 dbgs() << " Using " << (UseThumb ? "Thumb" : "Arm") << " entrypoint "
860 << *StubEntrypoint << " in "
861 << StubEntrypoint->getSection().getName() << "\n";
862 });
863
864 E.setTarget(*StubEntrypoint);
865 return true;
866}
867
868bool StubsManager_v7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
869 if (!needsStub(E))
870 return false;
871
872 // Stub Arm/Thumb follows instruction set state at relocation site.
873 // TODO: We may reduce them at relaxation time and reuse freed slots.
874 bool MakeThumb = (E.getKind() > LastArmRelocation);
875 LLVM_DEBUG(dbgs() << " Preparing " << (MakeThumb ? "Thumb" : "Arm")
876 << " stub for " << G.getEdgeKindName(E.getKind())
877 << " edge at " << B->getFixupAddress(E) << " ("
878 << B->getAddress() << " + "
879 << formatv("{0:x}", E.getOffset()) << ")\n");
880
881 Symbol &Target = E.getTarget();
882 assert(Target.hasName() && "Edge cannot point to anonymous target");
883 Symbol *&StubSymbol = getStubSymbolSlot(Name: *Target.getName(), Thumb: MakeThumb);
884
885 if (!StubSymbol) {
886 if (!StubsSection)
887 StubsSection = &G.createSection(Name: getSectionName(),
888 Prot: orc::MemProt::Read | orc::MemProt::Exec);
889 Block &B = MakeThumb ? createStubThumbv7(G, S&: *StubsSection, Target)
890 : createStubArmv7(G, S&: *StubsSection, Target);
891 StubSymbol = &G.addAnonymousSymbol(Content&: B, Offset: 0, Size: B.getSize(), IsCallable: true, IsLive: false);
892 if (MakeThumb)
893 StubSymbol->setTargetFlags(ThumbSymbol);
894
895 LLVM_DEBUG({
896 dbgs() << " Created " << (MakeThumb ? "Thumb" : "Arm") << " entry for "
897 << Target.getName() << " in " << StubsSection->getName() << ": "
898 << *StubSymbol << "\n";
899 });
900 }
901
902 assert(MakeThumb == (StubSymbol->getTargetFlags() & ThumbSymbol) &&
903 "Instruction set states of stub and relocation site should be equal");
904 LLVM_DEBUG({
905 dbgs() << " Using " << (MakeThumb ? "Thumb" : "Arm") << " entry "
906 << *StubSymbol << " in " << StubSymbol->getSection().getName()
907 << "\n";
908 });
909
910 E.setTarget(*StubSymbol);
911 return true;
912}
913
914const char *getEdgeKindName(Edge::Kind K) {
915#define KIND_NAME_CASE(K) \
916 case K: \
917 return #K;
918
919 switch (K) {
920 KIND_NAME_CASE(Data_Delta32)
921 KIND_NAME_CASE(Data_Pointer32)
922 KIND_NAME_CASE(Data_PRel31)
923 KIND_NAME_CASE(Data_RequestGOTAndTransformToDelta32)
924 KIND_NAME_CASE(Arm_Call)
925 KIND_NAME_CASE(Arm_Jump24)
926 KIND_NAME_CASE(Arm_MovwAbsNC)
927 KIND_NAME_CASE(Arm_MovtAbs)
928 KIND_NAME_CASE(Thumb_Call)
929 KIND_NAME_CASE(Thumb_Jump24)
930 KIND_NAME_CASE(Thumb_MovwAbsNC)
931 KIND_NAME_CASE(Thumb_MovtAbs)
932 KIND_NAME_CASE(Thumb_MovwPrelNC)
933 KIND_NAME_CASE(Thumb_MovtPrel)
934 KIND_NAME_CASE(None)
935 default:
936 return getGenericEdgeKindName(K);
937 }
938#undef KIND_NAME_CASE
939}
940
941const char *getCPUArchName(ARMBuildAttrs::CPUArch K) {
942#define CPUARCH_NAME_CASE(K) \
943 case K: \
944 return #K;
945
946 using namespace ARMBuildAttrs;
947 switch (K) {
948 CPUARCH_NAME_CASE(Pre_v4)
949 CPUARCH_NAME_CASE(v4)
950 CPUARCH_NAME_CASE(v4T)
951 CPUARCH_NAME_CASE(v5T)
952 CPUARCH_NAME_CASE(v5TE)
953 CPUARCH_NAME_CASE(v5TEJ)
954 CPUARCH_NAME_CASE(v6)
955 CPUARCH_NAME_CASE(v6KZ)
956 CPUARCH_NAME_CASE(v6T2)
957 CPUARCH_NAME_CASE(v6K)
958 CPUARCH_NAME_CASE(v7)
959 CPUARCH_NAME_CASE(v6_M)
960 CPUARCH_NAME_CASE(v6S_M)
961 CPUARCH_NAME_CASE(v7E_M)
962 CPUARCH_NAME_CASE(v8_A)
963 CPUARCH_NAME_CASE(v8_R)
964 CPUARCH_NAME_CASE(v8_M_Base)
965 CPUARCH_NAME_CASE(v8_M_Main)
966 CPUARCH_NAME_CASE(v8_1_M_Main)
967 CPUARCH_NAME_CASE(v9_A)
968 }
969 llvm_unreachable("Missing CPUArch in switch?");
970#undef CPUARCH_NAME_CASE
971}
972
973} // namespace aarch32
974} // namespace jitlink
975} // namespace llvm
976