1//===---- aarch64.cpp - Generic JITLink aarch64 edge kinds, utilities -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Generic utilities for graphs representing aarch64 objects.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ExecutionEngine/JITLink/aarch64.h"
14
15#include "llvm/Support/BinaryStreamWriter.h"
16
17#define DEBUG_TYPE "jitlink"
18
19namespace llvm {
20namespace jitlink {
21namespace aarch64 {
22
23const char NullPointerContent[8] = {0x00, 0x00, 0x00, 0x00,
24 0x00, 0x00, 0x00, 0x00};
25
26const char PointerJumpStubContent[12] = {
27 0x10, 0x00, 0x00, (char)0x90u, // ADRP x16, <imm>@page21
28 0x10, 0x02, 0x40, (char)0xf9u, // LDR x16, [x16, <imm>@pageoff12]
29 0x00, 0x02, 0x1f, (char)0xd6u // BR x16
30};
31
32const char ReentryTrampolineContent[8] = {
33 (char)0xfd, 0x7b, (char)0xbf, (char)0xa9, // STP x30, [sp, #-8]
34 0x00, 0x00, 0x00, (char)0x94 // BL
35};
36
37const char *getEdgeKindName(Edge::Kind R) {
38 switch (R) {
39 case Pointer64:
40 return "Pointer64";
41 case Pointer64Authenticated:
42 return "Pointer64Authenticated";
43 case Pointer32:
44 return "Pointer32";
45 case Delta64:
46 return "Delta64";
47 case Delta32:
48 return "Delta32";
49 case NegDelta64:
50 return "NegDelta64";
51 case NegDelta32:
52 return "NegDelta32";
53 case Branch26PCRel:
54 return "Branch26PCRel";
55 case MoveWide16:
56 return "MoveWide16";
57 case LDRLiteral19:
58 return "LDRLiteral19";
59 case TestAndBranch14PCRel:
60 return "TestAndBranch14PCRel";
61 case CondBranch19PCRel:
62 return "CondBranch19PCRel";
63 case ADRLiteral21:
64 return "ADRLiteral21";
65 case Page21:
66 return "Page21";
67 case PageOffset12:
68 return "PageOffset12";
69 case GotPageOffset15:
70 return "GotPageOffset15";
71 case RequestGOTAndTransformToPage21:
72 return "RequestGOTAndTransformToPage21";
73 case RequestGOTAndTransformToPageOffset12:
74 return "RequestGOTAndTransformToPageOffset12";
75 case RequestGOTAndTransformToPageOffset15:
76 return "RequestGOTAndTransformToPageOffset15";
77 case RequestGOTAndTransformToDelta32:
78 return "RequestGOTAndTransformToDelta32";
79 case RequestTLVPAndTransformToPage21:
80 return "RequestTLVPAndTransformToPage21";
81 case RequestTLVPAndTransformToPageOffset12:
82 return "RequestTLVPAndTransformToPageOffset12";
83 case RequestTLSDescEntryAndTransformToPage21:
84 return "RequestTLSDescEntryAndTransformToPage21";
85 case RequestTLSDescEntryAndTransformToPageOffset12:
86 return "RequestTLSDescEntryAndTransformToPageOffset12";
87 default:
88 return getGenericEdgeKindName(K: R);
89 }
90}
91
92// Write a 64-bit GPR -> GPR move.
93template <typename AppendFtor>
94static Error writeMovRegRegSeq(AppendFtor &Append, uint64_t DstReg,
95 uint64_t SrcReg) {
96 assert(DstReg < 32 && "Dst reg out of range");
97 assert(SrcReg < 32 && "Src reg out of range");
98
99 if (DstReg == SrcReg)
100 return Error::success();
101
102 constexpr uint32_t MOVGPR64Template = 0xaa0003e0;
103 constexpr uint32_t DstRegIndex = 0;
104 constexpr uint32_t SrcRegIndex = 16;
105 uint32_t Instr = MOVGPR64Template;
106 Instr |= DstReg << DstRegIndex;
107 Instr |= SrcReg << SrcRegIndex;
108 return Append(Instr);
109}
110
111// Generate a sequence of imm writes to assign the given value.
112template <typename AppendFtor>
113static Error writeMovRegImm64Seq(AppendFtor &Append, uint64_t Reg,
114 uint64_t Imm) {
115 assert(Reg < 32 && "Invalid register number");
116
117 constexpr uint32_t MovRegImm64Template = 0xd2800000;
118 constexpr unsigned PreserveBitIndex = 29;
119 constexpr unsigned ShiftBitsIndex = 21;
120 constexpr unsigned ImmBitsIndex = 5;
121
122 bool PreserveRegValue = false;
123 for (unsigned I = 0; I != 4; ++I) {
124 uint32_t ImmBits = Imm & 0xffff;
125 Imm >>= 16;
126
127 // Skip any all-zero immediates after the first one.
128 if (PreserveRegValue && !ImmBits)
129 continue;
130
131 uint32_t Instr = MovRegImm64Template;
132 Instr |= PreserveRegValue << PreserveBitIndex;
133 Instr |= (I << ShiftBitsIndex);
134 Instr |= ImmBits << ImmBitsIndex;
135 Instr |= Reg;
136 if (auto Err = Append(Instr))
137 return Err;
138 PreserveRegValue = true;
139 }
140
141 return Error::success();
142}
143
144template <typename AppendFtor>
145static Error
146writePACSignSeq(AppendFtor &Append, unsigned DstReg, orc::ExecutorAddr RawAddr,
147 unsigned RawAddrReg, unsigned DiscriminatorReg, unsigned Key,
148 uint64_t EncodedDiscriminator, bool AddressDiversify) {
149 assert(DstReg < 32 && "DstReg out of range");
150 assert(RawAddrReg < 32 && "AddrReg out of range");
151 assert(DiscriminatorReg < 32 && "DiscriminatorReg out of range");
152 assert(EncodedDiscriminator < 0x10000 && "EncodedDiscriminator out of range");
153
154 if (AddressDiversify) {
155 // Move the address into the discriminator register.
156 if (auto Err = writeMovRegRegSeq(Append, DiscriminatorReg, RawAddrReg))
157 return Err;
158 // Blend encoded discriminator if there is one.
159 if (EncodedDiscriminator) {
160 constexpr uint32_t MOVKTemplate = 0xf2e00000;
161 constexpr unsigned ImmIndex = 5;
162 uint32_t BlendInstr = MOVKTemplate;
163 BlendInstr |= EncodedDiscriminator << ImmIndex;
164 BlendInstr |= DiscriminatorReg;
165 if (auto Err = Append(BlendInstr))
166 return Err;
167 }
168 } else if (EncodedDiscriminator) {
169 // Move the encoded discriminator into the discriminator register.
170 if (auto Err =
171 writeMovRegImm64Seq(Append, DiscriminatorReg, EncodedDiscriminator))
172 return Err;
173 } else
174 DiscriminatorReg = 31; // WZR
175
176 constexpr uint32_t PACTemplate = 0xdac10000;
177 constexpr unsigned ZBitIndex = 13;
178 constexpr unsigned KeyIndex = 10;
179 constexpr unsigned DiscriminatorRegIndex = 5;
180
181 uint32_t Instr = PACTemplate;
182 Instr |= (DiscriminatorReg == 31) << ZBitIndex;
183 Instr |= Key << KeyIndex;
184 Instr |= DiscriminatorReg << DiscriminatorRegIndex;
185 Instr |= DstReg;
186
187 return Append(Instr);
188}
189
190template <typename AppendFtor>
191static Error writeStoreRegSeq(AppendFtor &Append, unsigned DstLocReg,
192 unsigned SrcReg) {
193 assert(DstLocReg < 32 && "DstLocReg out of range");
194 assert(SrcReg < 32 && "SrcReg out of range");
195
196 constexpr uint32_t STRTemplate = 0xf9000000;
197 constexpr unsigned DstLocRegIndex = 5;
198 constexpr unsigned SrcRegIndex = 0;
199
200 uint32_t Instr = STRTemplate;
201 Instr |= DstLocReg << DstLocRegIndex;
202 Instr |= SrcReg << SrcRegIndex;
203
204 return Append(Instr);
205}
206
207void GOTTableManager::registerExistingEntries() {
208 for (auto *EntrySym : GOTSection->symbols()) {
209 assert(EntrySym->getBlock().edges_size() == 1 &&
210 "GOT block edge count != 1");
211 registerPreExistingEntry(Target&: EntrySym->getBlock().edges().begin()->getTarget(),
212 Entry&: *EntrySym);
213 }
214}
215
216void PLTTableManager::registerExistingEntries() {
217 for (auto *EntrySym : StubsSection->symbols()) {
218 assert(EntrySym->getBlock().edges_size() == 2 &&
219 "PLT block edge count != 2");
220 auto &GOTSym = EntrySym->getBlock().edges().begin()->getTarget();
221 assert(GOTSym.getBlock().edges_size() == 1 && "GOT block edge count != 1");
222 registerPreExistingEntry(Target&: GOTSym.getBlock().edges().begin()->getTarget(),
223 Entry&: *EntrySym);
224 }
225}
226
227const char *getPointerSigningFunctionSectionName() { return "$__ptrauth_sign"; }
228
229/// Creates a pointer signing function section, block, and symbol to reserve
230/// space for a signing function for this LinkGraph. Clients should insert this
231/// pass in the post-prune phase, and add the paired
232/// lowerPointer64AuthEdgesToSigningFunction pass to the pre-fixup phase.
233Error createEmptyPointerSigningFunction(LinkGraph &G) {
234 LLVM_DEBUG({
235 dbgs() << "Creating empty pointer signing function for " << G.getName()
236 << "\n";
237 });
238
239 // FIXME: We could put a tighter bound on this if we inspected the ptrauth
240 // info encoded in the addend -- the only actually unknown quantity is the
241 // fixup location, and we can probably put constraints even on that.
242 size_t NumPtrAuthFixupLocations = 0;
243 for (auto &Sec : G.sections()) {
244
245 // No-alloc sections can't have ptrauth edges. We don't need to error out
246 // here: applyFixup will catch these edges if any make it to the fixup
247 // stage.
248 if (Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)
249 continue;
250
251 for (auto *B : Sec.blocks()) {
252 for (auto &E : B->edges())
253 NumPtrAuthFixupLocations +=
254 E.getKind() == aarch64::Pointer64Authenticated;
255 }
256 }
257
258 constexpr size_t MaxPtrSignSeqLength =
259 4 + // To materialize the value to sign.
260 4 + // To materialize the fixup location.
261 3 + // To copy, blend discriminator, and sign
262 1; // To store the result.
263
264 // The maximum number of signing instructions required is the maximum per
265 // location, times the number of locations, plus three instructions to
266 // materialize the return value and return.
267 size_t NumSigningInstrs = NumPtrAuthFixupLocations * MaxPtrSignSeqLength + 3;
268
269 // Create signing function section.
270 auto &SigningSection =
271 G.createSection(Name: getPointerSigningFunctionSectionName(),
272 Prot: orc::MemProt::Read | orc::MemProt::Exec);
273 SigningSection.setMemLifetime(orc::MemLifetime::Finalize);
274
275 size_t SigningFunctionSize = NumSigningInstrs * 4;
276 auto &SigningFunctionBlock = G.createMutableContentBlock(
277 Parent&: SigningSection, MutableContent: G.allocateBuffer(Size: SigningFunctionSize),
278 Address: orc::ExecutorAddr(), Alignment: 4, AlignmentOffset: 0);
279 G.addAnonymousSymbol(Content&: SigningFunctionBlock, Offset: 0, Size: SigningFunctionBlock.getSize(),
280 IsCallable: true, IsLive: true);
281
282 LLVM_DEBUG({
283 dbgs() << " " << NumPtrAuthFixupLocations << " location(s) to sign, up to "
284 << NumSigningInstrs << " instructions required ("
285 << formatv("{0:x}", SigningFunctionBlock.getSize()) << " bytes)\n";
286 });
287
288 return Error::success();
289}
290
291/// Given a LinkGraph containing Pointer64Auth edges, transform those edges to
292/// Pointer64 and add code to sign the pointers in the executor.
293///
294/// This function will add a $__ptrauth_sign section with finalization-lifetime
295/// containing an anonymous function that will sign all pointers in the graph.
296/// An allocation action will be added to run this function during finalization.
297Error lowerPointer64AuthEdgesToSigningFunction(LinkGraph &G) {
298 LLVM_DEBUG({
299 dbgs() << "Writing pointer signing function for " << G.getName() << "\n";
300 });
301
302 constexpr unsigned Reg1 = 8; // Holds pointer value to sign.
303 constexpr unsigned Reg2 = 9; // Holds fixup address.
304 constexpr unsigned Reg3 = 10; // Temporary for discriminator value if needed.
305
306 // Find the signing function.
307 auto *SigningSection =
308 G.findSectionByName(Name: getPointerSigningFunctionSectionName());
309 assert(SigningSection && "Siging section missing");
310 assert(SigningSection->blocks_size() == 1 &&
311 "Unexpected number of blocks in signing section");
312 assert(SigningSection->symbols_size() == 1 &&
313 "Unexpected number of symbols in signing section");
314
315 auto &SigningFunctionSym = **SigningSection->symbols().begin();
316 auto &SigningFunctionBlock = SigningFunctionSym.getBlock();
317 auto SigningFunctionBuf = SigningFunctionBlock.getAlreadyMutableContent();
318
319 // Write the instructions to the block content.
320 BinaryStreamWriter InstrWriter(
321 {reinterpret_cast<uint8_t *>(SigningFunctionBuf.data()),
322 SigningFunctionBuf.size()},
323 G.getEndianness());
324
325 auto AppendInstr = [&](uint32_t Instr) {
326 return InstrWriter.writeInteger(Value: Instr);
327 };
328
329 for (auto &Sec : G.sections()) {
330
331 if (Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)
332 continue;
333
334 for (auto *B : Sec.blocks()) {
335 for (auto &E : B->edges()) {
336 // We're only concerned with Pointer64Authenticated edges here.
337 if (E.getKind() != aarch64::Pointer64Authenticated)
338 continue;
339
340 uint64_t EncodedInfo = E.getAddend();
341 int32_t RealAddend = (uint32_t)(EncodedInfo & 0xffffffff);
342 auto ValueToSign = E.getTarget().getAddress() + RealAddend;
343 if (!ValueToSign) {
344 LLVM_DEBUG(dbgs() << " " << B->getFixupAddress(E) << " <- null\n");
345 E.setAddend(RealAddend);
346 E.setKind(aarch64::Pointer64);
347 continue;
348 }
349
350 uint32_t InitialDiscriminator = (EncodedInfo >> 32) & 0xffff;
351 bool AddressDiversify = (EncodedInfo >> 48) & 0x1;
352 uint32_t Key = (EncodedInfo >> 49) & 0x3;
353 uint32_t HighBits = EncodedInfo >> 51;
354
355 if (HighBits != 0x1000)
356 return make_error<JITLinkError>(
357 Args: "Pointer64Auth edge at " +
358 formatv(Fmt: "{0:x}", Vals: B->getFixupAddress(E).getValue()) +
359 " has invalid encoded addend " + formatv(Fmt: "{0:x}", Vals&: EncodedInfo));
360
361 LLVM_DEBUG({
362 const char *const KeyNames[] = {"IA", "IB", "DA", "DB"};
363 dbgs() << " " << B->getFixupAddress(E) << " <- " << ValueToSign
364 << " : key = " << KeyNames[Key] << ", discriminator = "
365 << formatv("{0:x4}", InitialDiscriminator)
366 << ", address diversified = "
367 << (AddressDiversify ? "yes" : "no") << "\n";
368 });
369
370 // Materialize pointer value.
371 cantFail(
372 Err: writeMovRegImm64Seq(Append&: AppendInstr, Reg: Reg1, Imm: ValueToSign.getValue()));
373
374 // Materialize fixup pointer.
375 cantFail(Err: writeMovRegImm64Seq(Append&: AppendInstr, Reg: Reg2,
376 Imm: B->getFixupAddress(E).getValue()));
377
378 // Write signing instruction(s).
379 cantFail(Err: writePACSignSeq(Append&: AppendInstr, DstReg: Reg1, RawAddr: ValueToSign, RawAddrReg: Reg2, DiscriminatorReg: Reg3,
380 Key, EncodedDiscriminator: InitialDiscriminator, AddressDiversify));
381
382 // Store signed pointer.
383 cantFail(Err: writeStoreRegSeq(Append&: AppendInstr, DstLocReg: Reg2, SrcReg: Reg1));
384
385 // Replace edge with a keep-alive to preserve dependence info.
386 E.setKind(Edge::KeepAlive);
387 }
388 }
389 }
390
391 // Write epilogue. x0 = 0, x1 = 1 is an SPS serialized Error::success value.
392 constexpr uint32_t RETInstr = 0xd65f03c0;
393 cantFail(Err: writeMovRegImm64Seq(Append&: AppendInstr, Reg: 0, Imm: 0)); // mov x0, #0
394 cantFail(Err: writeMovRegImm64Seq(Append&: AppendInstr, Reg: 1, Imm: 1)); // mov x1, #1
395 cantFail(Err: AppendInstr(RETInstr)); // ret
396
397 // Add an allocation action to call the signing function.
398 using namespace orc::shared;
399 G.allocActions().push_back(
400 x: {.Finalize: cantFail(ValOrErr: WrapperFunctionCall::Create<SPSArgList<>>(
401 FnAddr: SigningFunctionSym.getAddress())),
402 .Dealloc: {}});
403
404 return Error::success();
405}
406
407} // namespace aarch64
408} // namespace jitlink
409} // namespace llvm
410