1//===- lib/MC/MCWin64EH.cpp - MCWin64EH implementation --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/MC/MCWin64EH.h"
10#include "llvm/ADT/Twine.h"
11#include "llvm/MC/MCContext.h"
12#include "llvm/MC/MCExpr.h"
13#include "llvm/MC/MCObjectStreamer.h"
14#include "llvm/MC/MCStreamer.h"
15#include "llvm/MC/MCSymbol.h"
16#include "llvm/Support/Win64EH.h"
17namespace llvm {
18class MCSection;
19}
20
21using namespace llvm;
22
23// NOTE: All relocations generated here are 4-byte image-relative.
24
25static uint8_t CountOfUnwindCodes(std::vector<WinEH::Instruction> &Insns) {
26 uint8_t Count = 0;
27 for (const auto &I : Insns) {
28 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {
29 default:
30 llvm_unreachable("Unsupported unwind code");
31 case Win64EH::UOP_PushNonVol:
32 case Win64EH::UOP_AllocSmall:
33 case Win64EH::UOP_SetFPReg:
34 case Win64EH::UOP_PushMachFrame:
35 Count += 1;
36 break;
37 case Win64EH::UOP_SaveNonVol:
38 case Win64EH::UOP_SaveXMM128:
39 Count += 2;
40 break;
41 case Win64EH::UOP_SaveNonVolBig:
42 case Win64EH::UOP_SaveXMM128Big:
43 Count += 3;
44 break;
45 case Win64EH::UOP_AllocLarge:
46 Count += (I.Offset > 512 * 1024 - 8) ? 3 : 2;
47 break;
48 }
49 }
50 return Count;
51}
52
53static void EmitAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS,
54 const MCSymbol *RHS) {
55 MCContext &Context = Streamer.getContext();
56 const MCExpr *Diff =
57 MCBinaryExpr::createSub(LHS: MCSymbolRefExpr::create(Symbol: LHS, Ctx&: Context),
58 RHS: MCSymbolRefExpr::create(Symbol: RHS, Ctx&: Context), Ctx&: Context);
59 Streamer.emitValue(Value: Diff, Size: 1);
60}
61
62static void EmitUnwindCode(MCStreamer &streamer, const MCSymbol *begin,
63 WinEH::Instruction &inst) {
64 uint8_t b2;
65 uint16_t w;
66 b2 = (inst.Operation & 0x0F);
67 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {
68 default:
69 llvm_unreachable("Unsupported unwind code");
70 case Win64EH::UOP_PushNonVol:
71 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
72 b2 |= (inst.Register & 0x0F) << 4;
73 streamer.emitInt8(Value: b2);
74 break;
75 case Win64EH::UOP_AllocLarge:
76 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
77 if (inst.Offset > 512 * 1024 - 8) {
78 b2 |= 0x10;
79 streamer.emitInt8(Value: b2);
80 w = inst.Offset & 0xFFF8;
81 streamer.emitInt16(Value: w);
82 w = inst.Offset >> 16;
83 } else {
84 streamer.emitInt8(Value: b2);
85 w = inst.Offset >> 3;
86 }
87 streamer.emitInt16(Value: w);
88 break;
89 case Win64EH::UOP_AllocSmall:
90 b2 |= (((inst.Offset - 8) >> 3) & 0x0F) << 4;
91 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
92 streamer.emitInt8(Value: b2);
93 break;
94 case Win64EH::UOP_SetFPReg:
95 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
96 streamer.emitInt8(Value: b2);
97 break;
98 case Win64EH::UOP_SaveNonVol:
99 case Win64EH::UOP_SaveXMM128:
100 b2 |= (inst.Register & 0x0F) << 4;
101 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
102 streamer.emitInt8(Value: b2);
103 w = inst.Offset >> 3;
104 if (inst.Operation == Win64EH::UOP_SaveXMM128)
105 w >>= 1;
106 streamer.emitInt16(Value: w);
107 break;
108 case Win64EH::UOP_SaveNonVolBig:
109 case Win64EH::UOP_SaveXMM128Big:
110 b2 |= (inst.Register & 0x0F) << 4;
111 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
112 streamer.emitInt8(Value: b2);
113 if (inst.Operation == Win64EH::UOP_SaveXMM128Big)
114 w = inst.Offset & 0xFFF0;
115 else
116 w = inst.Offset & 0xFFF8;
117 streamer.emitInt16(Value: w);
118 w = inst.Offset >> 16;
119 streamer.emitInt16(Value: w);
120 break;
121 case Win64EH::UOP_PushMachFrame:
122 if (inst.Offset == 1)
123 b2 |= 0x10;
124 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
125 streamer.emitInt8(Value: b2);
126 break;
127 }
128}
129
130static void EmitSymbolRefWithOfs(MCStreamer &streamer,
131 const MCSymbol *Base,
132 int64_t Offset) {
133 MCContext &Context = streamer.getContext();
134 const MCConstantExpr *OffExpr = MCConstantExpr::create(Value: Offset, Ctx&: Context);
135 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Symbol: Base,
136 Kind: MCSymbolRefExpr::VK_COFF_IMGREL32,
137 Ctx&: Context);
138 streamer.emitValue(Value: MCBinaryExpr::createAdd(LHS: BaseRefRel, RHS: OffExpr, Ctx&: Context), Size: 4);
139}
140
141static void EmitSymbolRefWithOfs(MCStreamer &streamer,
142 const MCSymbol *Base,
143 const MCSymbol *Other) {
144 MCContext &Context = streamer.getContext();
145 const MCSymbolRefExpr *BaseRef = MCSymbolRefExpr::create(Symbol: Base, Ctx&: Context);
146 const MCSymbolRefExpr *OtherRef = MCSymbolRefExpr::create(Symbol: Other, Ctx&: Context);
147 const MCExpr *Ofs = MCBinaryExpr::createSub(LHS: OtherRef, RHS: BaseRef, Ctx&: Context);
148 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Symbol: Base,
149 Kind: MCSymbolRefExpr::VK_COFF_IMGREL32,
150 Ctx&: Context);
151 streamer.emitValue(Value: MCBinaryExpr::createAdd(LHS: BaseRefRel, RHS: Ofs, Ctx&: Context), Size: 4);
152}
153
154static void EmitRuntimeFunction(MCStreamer &streamer,
155 const WinEH::FrameInfo *info) {
156 MCContext &context = streamer.getContext();
157
158 streamer.emitValueToAlignment(Alignment: Align(4));
159 EmitSymbolRefWithOfs(streamer, Base: info->Begin, Other: info->Begin);
160 EmitSymbolRefWithOfs(streamer, Base: info->Begin, Other: info->End);
161 streamer.emitValue(Value: MCSymbolRefExpr::create(Symbol: info->Symbol,
162 Kind: MCSymbolRefExpr::VK_COFF_IMGREL32,
163 Ctx&: context), Size: 4);
164}
165
166static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {
167 // If this UNWIND_INFO already has a symbol, it's already been emitted.
168 if (info->Symbol)
169 return;
170
171 MCContext &context = streamer.getContext();
172 MCSymbol *Label = context.createTempSymbol();
173
174 streamer.emitValueToAlignment(Alignment: Align(4));
175 streamer.emitLabel(Symbol: Label);
176 info->Symbol = Label;
177
178 // Upper 3 bits are the version number (currently 1).
179 uint8_t flags = 0x01;
180 if (info->ChainedParent)
181 flags |= Win64EH::UNW_ChainInfo << 3;
182 else {
183 if (info->HandlesUnwind)
184 flags |= Win64EH::UNW_TerminateHandler << 3;
185 if (info->HandlesExceptions)
186 flags |= Win64EH::UNW_ExceptionHandler << 3;
187 }
188 streamer.emitInt8(Value: flags);
189
190 if (info->PrologEnd)
191 EmitAbsDifference(Streamer&: streamer, LHS: info->PrologEnd, RHS: info->Begin);
192 else
193 streamer.emitInt8(Value: 0);
194
195 uint8_t numCodes = CountOfUnwindCodes(Insns&: info->Instructions);
196 streamer.emitInt8(Value: numCodes);
197
198 uint8_t frame = 0;
199 if (info->LastFrameInst >= 0) {
200 WinEH::Instruction &frameInst = info->Instructions[info->LastFrameInst];
201 assert(frameInst.Operation == Win64EH::UOP_SetFPReg);
202 frame = (frameInst.Register & 0x0F) | (frameInst.Offset & 0xF0);
203 }
204 streamer.emitInt8(Value: frame);
205
206 // Emit unwind instructions (in reverse order).
207 uint8_t numInst = info->Instructions.size();
208 for (uint8_t c = 0; c < numInst; ++c) {
209 WinEH::Instruction inst = info->Instructions.back();
210 info->Instructions.pop_back();
211 EmitUnwindCode(streamer, begin: info->Begin, inst);
212 }
213
214 // For alignment purposes, the instruction array will always have an even
215 // number of entries, with the final entry potentially unused (in which case
216 // the array will be one longer than indicated by the count of unwind codes
217 // field).
218 if (numCodes & 1) {
219 streamer.emitInt16(Value: 0);
220 }
221
222 if (flags & (Win64EH::UNW_ChainInfo << 3))
223 EmitRuntimeFunction(streamer, info: info->ChainedParent);
224 else if (flags &
225 ((Win64EH::UNW_TerminateHandler|Win64EH::UNW_ExceptionHandler) << 3))
226 streamer.emitValue(Value: MCSymbolRefExpr::create(Symbol: info->ExceptionHandler,
227 Kind: MCSymbolRefExpr::VK_COFF_IMGREL32,
228 Ctx&: context), Size: 4);
229 else if (numCodes == 0) {
230 // The minimum size of an UNWIND_INFO struct is 8 bytes. If we're not
231 // a chained unwind info, if there is no handler, and if there are fewer
232 // than 2 slots used in the unwind code array, we have to pad to 8 bytes.
233 streamer.emitInt32(Value: 0);
234 }
235}
236
237void llvm::Win64EH::UnwindEmitter::Emit(MCStreamer &Streamer) const {
238 // Emit the unwind info structs first.
239 for (const auto &CFI : Streamer.getWinFrameInfos()) {
240 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: CFI->TextSection);
241 Streamer.switchSection(Section: XData);
242 ::EmitUnwindInfo(streamer&: Streamer, info: CFI.get());
243 }
244
245 // Now emit RUNTIME_FUNCTION entries.
246 for (const auto &CFI : Streamer.getWinFrameInfos()) {
247 MCSection *PData = Streamer.getAssociatedPDataSection(TextSec: CFI->TextSection);
248 Streamer.switchSection(Section: PData);
249 EmitRuntimeFunction(streamer&: Streamer, info: CFI.get());
250 }
251}
252
253void llvm::Win64EH::UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer,
254 WinEH::FrameInfo *info,
255 bool HandlerData) const {
256 // Switch sections (the static function above is meant to be called from
257 // here and from Emit().
258 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: info->TextSection);
259 Streamer.switchSection(Section: XData);
260
261 ::EmitUnwindInfo(streamer&: Streamer, info);
262}
263
264static const MCExpr *GetSubDivExpr(MCStreamer &Streamer, const MCSymbol *LHS,
265 const MCSymbol *RHS, int Div) {
266 MCContext &Context = Streamer.getContext();
267 const MCExpr *Expr =
268 MCBinaryExpr::createSub(LHS: MCSymbolRefExpr::create(Symbol: LHS, Ctx&: Context),
269 RHS: MCSymbolRefExpr::create(Symbol: RHS, Ctx&: Context), Ctx&: Context);
270 if (Div != 1)
271 Expr = MCBinaryExpr::createDiv(LHS: Expr, RHS: MCConstantExpr::create(Value: Div, Ctx&: Context),
272 Ctx&: Context);
273 return Expr;
274}
275
276static std::optional<int64_t> GetOptionalAbsDifference(MCStreamer &Streamer,
277 const MCSymbol *LHS,
278 const MCSymbol *RHS) {
279 MCContext &Context = Streamer.getContext();
280 const MCExpr *Diff =
281 MCBinaryExpr::createSub(LHS: MCSymbolRefExpr::create(Symbol: LHS, Ctx&: Context),
282 RHS: MCSymbolRefExpr::create(Symbol: RHS, Ctx&: Context), Ctx&: Context);
283 MCObjectStreamer *OS = (MCObjectStreamer *)(&Streamer);
284 // It should normally be possible to calculate the length of a function
285 // at this point, but it might not be possible in the presence of certain
286 // unusual constructs, like an inline asm with an alignment directive.
287 int64_t value;
288 if (!Diff->evaluateAsAbsolute(Res&: value, Asm: OS->getAssembler()))
289 return std::nullopt;
290 return value;
291}
292
293static int64_t GetAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS,
294 const MCSymbol *RHS) {
295 std::optional<int64_t> MaybeDiff =
296 GetOptionalAbsDifference(Streamer, LHS, RHS);
297 if (!MaybeDiff)
298 report_fatal_error(reason: "Failed to evaluate function length in SEH unwind info");
299 return *MaybeDiff;
300}
301
302static void checkARM64Instructions(MCStreamer &Streamer,
303 ArrayRef<WinEH::Instruction> Insns,
304 const MCSymbol *Begin, const MCSymbol *End,
305 StringRef Name, StringRef Type) {
306 if (!End)
307 return;
308 std::optional<int64_t> MaybeDistance =
309 GetOptionalAbsDifference(Streamer, LHS: End, RHS: Begin);
310 if (!MaybeDistance)
311 return;
312 uint32_t Distance = (uint32_t)*MaybeDistance;
313
314 for (const auto &I : Insns) {
315 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {
316 default:
317 break;
318 case Win64EH::UOP_TrapFrame:
319 case Win64EH::UOP_PushMachFrame:
320 case Win64EH::UOP_Context:
321 case Win64EH::UOP_ECContext:
322 case Win64EH::UOP_ClearUnwoundToCall:
323 // Can't reason about these opcodes and how they map to actual
324 // instructions.
325 return;
326 }
327 }
328 // Exclude the end opcode which doesn't map to an instruction.
329 uint32_t InstructionBytes = 4 * (Insns.size() - 1);
330 if (Distance != InstructionBytes) {
331 Streamer.getContext().reportError(
332 L: SMLoc(), Msg: "Incorrect size for " + Name + " " + Type + ": " +
333 Twine(Distance) +
334 " bytes of instructions in range, but .seh directives "
335 "corresponding to " +
336 Twine(InstructionBytes) + " bytes\n");
337 }
338}
339
340static uint32_t ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) {
341 uint32_t Count = 0;
342 for (const auto &I : Insns) {
343 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {
344 default:
345 llvm_unreachable("Unsupported ARM64 unwind code");
346 case Win64EH::UOP_AllocSmall:
347 Count += 1;
348 break;
349 case Win64EH::UOP_AllocMedium:
350 Count += 2;
351 break;
352 case Win64EH::UOP_AllocLarge:
353 Count += 4;
354 break;
355 case Win64EH::UOP_SaveR19R20X:
356 Count += 1;
357 break;
358 case Win64EH::UOP_SaveFPLRX:
359 Count += 1;
360 break;
361 case Win64EH::UOP_SaveFPLR:
362 Count += 1;
363 break;
364 case Win64EH::UOP_SaveReg:
365 Count += 2;
366 break;
367 case Win64EH::UOP_SaveRegP:
368 Count += 2;
369 break;
370 case Win64EH::UOP_SaveRegPX:
371 Count += 2;
372 break;
373 case Win64EH::UOP_SaveRegX:
374 Count += 2;
375 break;
376 case Win64EH::UOP_SaveLRPair:
377 Count += 2;
378 break;
379 case Win64EH::UOP_SaveFReg:
380 Count += 2;
381 break;
382 case Win64EH::UOP_SaveFRegP:
383 Count += 2;
384 break;
385 case Win64EH::UOP_SaveFRegX:
386 Count += 2;
387 break;
388 case Win64EH::UOP_SaveFRegPX:
389 Count += 2;
390 break;
391 case Win64EH::UOP_SetFP:
392 Count += 1;
393 break;
394 case Win64EH::UOP_AddFP:
395 Count += 2;
396 break;
397 case Win64EH::UOP_Nop:
398 Count += 1;
399 break;
400 case Win64EH::UOP_End:
401 Count += 1;
402 break;
403 case Win64EH::UOP_SaveNext:
404 Count += 1;
405 break;
406 case Win64EH::UOP_TrapFrame:
407 Count += 1;
408 break;
409 case Win64EH::UOP_PushMachFrame:
410 Count += 1;
411 break;
412 case Win64EH::UOP_Context:
413 Count += 1;
414 break;
415 case Win64EH::UOP_ECContext:
416 Count += 1;
417 break;
418 case Win64EH::UOP_ClearUnwoundToCall:
419 Count += 1;
420 break;
421 case Win64EH::UOP_PACSignLR:
422 Count += 1;
423 break;
424 case Win64EH::UOP_SaveAnyRegI:
425 case Win64EH::UOP_SaveAnyRegIP:
426 case Win64EH::UOP_SaveAnyRegD:
427 case Win64EH::UOP_SaveAnyRegDP:
428 case Win64EH::UOP_SaveAnyRegQ:
429 case Win64EH::UOP_SaveAnyRegQP:
430 case Win64EH::UOP_SaveAnyRegIX:
431 case Win64EH::UOP_SaveAnyRegIPX:
432 case Win64EH::UOP_SaveAnyRegDX:
433 case Win64EH::UOP_SaveAnyRegDPX:
434 case Win64EH::UOP_SaveAnyRegQX:
435 case Win64EH::UOP_SaveAnyRegQPX:
436 Count += 3;
437 break;
438 }
439 }
440 return Count;
441}
442
443// Unwind opcode encodings and restrictions are documented at
444// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
445static void ARM64EmitUnwindCode(MCStreamer &streamer,
446 const WinEH::Instruction &inst) {
447 uint8_t b, reg;
448 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {
449 default:
450 llvm_unreachable("Unsupported ARM64 unwind code");
451 case Win64EH::UOP_AllocSmall:
452 b = (inst.Offset >> 4) & 0x1F;
453 streamer.emitInt8(Value: b);
454 break;
455 case Win64EH::UOP_AllocMedium: {
456 uint16_t hw = (inst.Offset >> 4) & 0x7FF;
457 b = 0xC0;
458 b |= (hw >> 8);
459 streamer.emitInt8(Value: b);
460 b = hw & 0xFF;
461 streamer.emitInt8(Value: b);
462 break;
463 }
464 case Win64EH::UOP_AllocLarge: {
465 uint32_t w;
466 b = 0xE0;
467 streamer.emitInt8(Value: b);
468 w = inst.Offset >> 4;
469 b = (w & 0x00FF0000) >> 16;
470 streamer.emitInt8(Value: b);
471 b = (w & 0x0000FF00) >> 8;
472 streamer.emitInt8(Value: b);
473 b = w & 0x000000FF;
474 streamer.emitInt8(Value: b);
475 break;
476 }
477 case Win64EH::UOP_SetFP:
478 b = 0xE1;
479 streamer.emitInt8(Value: b);
480 break;
481 case Win64EH::UOP_AddFP:
482 b = 0xE2;
483 streamer.emitInt8(Value: b);
484 b = (inst.Offset >> 3);
485 streamer.emitInt8(Value: b);
486 break;
487 case Win64EH::UOP_Nop:
488 b = 0xE3;
489 streamer.emitInt8(Value: b);
490 break;
491 case Win64EH::UOP_SaveR19R20X:
492 b = 0x20;
493 b |= (inst.Offset >> 3) & 0x1F;
494 streamer.emitInt8(Value: b);
495 break;
496 case Win64EH::UOP_SaveFPLRX:
497 b = 0x80;
498 b |= ((inst.Offset - 1) >> 3) & 0x3F;
499 streamer.emitInt8(Value: b);
500 break;
501 case Win64EH::UOP_SaveFPLR:
502 b = 0x40;
503 b |= (inst.Offset >> 3) & 0x3F;
504 streamer.emitInt8(Value: b);
505 break;
506 case Win64EH::UOP_SaveReg:
507 assert(inst.Register >= 19 && "Saved reg must be >= 19");
508 reg = inst.Register - 19;
509 b = 0xD0 | ((reg & 0xC) >> 2);
510 streamer.emitInt8(Value: b);
511 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);
512 streamer.emitInt8(Value: b);
513 break;
514 case Win64EH::UOP_SaveRegX:
515 assert(inst.Register >= 19 && "Saved reg must be >= 19");
516 reg = inst.Register - 19;
517 b = 0xD4 | ((reg & 0x8) >> 3);
518 streamer.emitInt8(Value: b);
519 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1);
520 streamer.emitInt8(Value: b);
521 break;
522 case Win64EH::UOP_SaveRegP:
523 assert(inst.Register >= 19 && "Saved registers must be >= 19");
524 reg = inst.Register - 19;
525 b = 0xC8 | ((reg & 0xC) >> 2);
526 streamer.emitInt8(Value: b);
527 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);
528 streamer.emitInt8(Value: b);
529 break;
530 case Win64EH::UOP_SaveRegPX:
531 assert(inst.Register >= 19 && "Saved registers must be >= 19");
532 reg = inst.Register - 19;
533 b = 0xCC | ((reg & 0xC) >> 2);
534 streamer.emitInt8(Value: b);
535 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1);
536 streamer.emitInt8(Value: b);
537 break;
538 case Win64EH::UOP_SaveLRPair:
539 assert(inst.Register >= 19 && "Saved reg must be >= 19");
540 reg = inst.Register - 19;
541 assert((reg % 2) == 0 && "Saved reg must be 19+2*X");
542 reg /= 2;
543 b = 0xD6 | ((reg & 0x7) >> 2);
544 streamer.emitInt8(Value: b);
545 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);
546 streamer.emitInt8(Value: b);
547 break;
548 case Win64EH::UOP_SaveFReg:
549 assert(inst.Register >= 8 && "Saved dreg must be >= 8");
550 reg = inst.Register - 8;
551 b = 0xDC | ((reg & 0x4) >> 2);
552 streamer.emitInt8(Value: b);
553 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);
554 streamer.emitInt8(Value: b);
555 break;
556 case Win64EH::UOP_SaveFRegX:
557 assert(inst.Register >= 8 && "Saved dreg must be >= 8");
558 reg = inst.Register - 8;
559 b = 0xDE;
560 streamer.emitInt8(Value: b);
561 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1);
562 streamer.emitInt8(Value: b);
563 break;
564 case Win64EH::UOP_SaveFRegP:
565 assert(inst.Register >= 8 && "Saved dregs must be >= 8");
566 reg = inst.Register - 8;
567 b = 0xD8 | ((reg & 0x4) >> 2);
568 streamer.emitInt8(Value: b);
569 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);
570 streamer.emitInt8(Value: b);
571 break;
572 case Win64EH::UOP_SaveFRegPX:
573 assert(inst.Register >= 8 && "Saved dregs must be >= 8");
574 reg = inst.Register - 8;
575 b = 0xDA | ((reg & 0x4) >> 2);
576 streamer.emitInt8(Value: b);
577 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1);
578 streamer.emitInt8(Value: b);
579 break;
580 case Win64EH::UOP_End:
581 b = 0xE4;
582 streamer.emitInt8(Value: b);
583 break;
584 case Win64EH::UOP_SaveNext:
585 b = 0xE6;
586 streamer.emitInt8(Value: b);
587 break;
588 case Win64EH::UOP_TrapFrame:
589 b = 0xE8;
590 streamer.emitInt8(Value: b);
591 break;
592 case Win64EH::UOP_PushMachFrame:
593 b = 0xE9;
594 streamer.emitInt8(Value: b);
595 break;
596 case Win64EH::UOP_Context:
597 b = 0xEA;
598 streamer.emitInt8(Value: b);
599 break;
600 case Win64EH::UOP_ECContext:
601 b = 0xEB;
602 streamer.emitInt8(Value: b);
603 break;
604 case Win64EH::UOP_ClearUnwoundToCall:
605 b = 0xEC;
606 streamer.emitInt8(Value: b);
607 break;
608 case Win64EH::UOP_PACSignLR:
609 b = 0xFC;
610 streamer.emitInt8(Value: b);
611 break;
612 case Win64EH::UOP_SaveAnyRegI:
613 case Win64EH::UOP_SaveAnyRegIP:
614 case Win64EH::UOP_SaveAnyRegD:
615 case Win64EH::UOP_SaveAnyRegDP:
616 case Win64EH::UOP_SaveAnyRegQ:
617 case Win64EH::UOP_SaveAnyRegQP:
618 case Win64EH::UOP_SaveAnyRegIX:
619 case Win64EH::UOP_SaveAnyRegIPX:
620 case Win64EH::UOP_SaveAnyRegDX:
621 case Win64EH::UOP_SaveAnyRegDPX:
622 case Win64EH::UOP_SaveAnyRegQX:
623 case Win64EH::UOP_SaveAnyRegQPX: {
624 // This assumes the opcodes are listed in the enum in a particular order.
625 int Op = inst.Operation - Win64EH::UOP_SaveAnyRegI;
626 int Writeback = Op / 6;
627 int Paired = Op % 2;
628 int Mode = (Op / 2) % 3;
629 int Offset = inst.Offset >> 3;
630 if (Writeback || Paired || Mode == 2)
631 Offset >>= 1;
632 if (Writeback)
633 --Offset;
634 b = 0xE7;
635 streamer.emitInt8(Value: b);
636 assert(inst.Register < 32);
637 b = inst.Register | (Writeback << 5) | (Paired << 6);
638 streamer.emitInt8(Value: b);
639 b = Offset | (Mode << 6);
640 streamer.emitInt8(Value: b);
641 break;
642 }
643 }
644}
645
646// Returns the epilog symbol of an epilog with the exact same unwind code
647// sequence, if it exists. Otherwise, returns nullptr.
648// EpilogInstrs - Unwind codes for the current epilog.
649// Epilogs - Epilogs that potentialy match the current epilog.
650static MCSymbol*
651FindMatchingEpilog(const std::vector<WinEH::Instruction>& EpilogInstrs,
652 const std::vector<MCSymbol *>& Epilogs,
653 const WinEH::FrameInfo *info) {
654 for (auto *EpilogStart : Epilogs) {
655 auto InstrsIter = info->EpilogMap.find(Key: EpilogStart);
656 assert(InstrsIter != info->EpilogMap.end() &&
657 "Epilog not found in EpilogMap");
658 const auto &Instrs = InstrsIter->second.Instructions;
659
660 if (Instrs.size() != EpilogInstrs.size())
661 continue;
662
663 bool Match = true;
664 for (unsigned i = 0; i < Instrs.size(); ++i)
665 if (Instrs[i] != EpilogInstrs[i]) {
666 Match = false;
667 break;
668 }
669
670 if (Match)
671 return EpilogStart;
672 }
673 return nullptr;
674}
675
676static void simplifyARM64Opcodes(std::vector<WinEH::Instruction> &Instructions,
677 bool Reverse) {
678 unsigned PrevOffset = -1;
679 unsigned PrevRegister = -1;
680
681 auto VisitInstruction = [&](WinEH::Instruction &Inst) {
682 // Convert 2-byte opcodes into equivalent 1-byte ones.
683 if (Inst.Operation == Win64EH::UOP_SaveRegP && Inst.Register == 29) {
684 Inst.Operation = Win64EH::UOP_SaveFPLR;
685 Inst.Register = -1;
686 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX &&
687 Inst.Register == 29) {
688 Inst.Operation = Win64EH::UOP_SaveFPLRX;
689 Inst.Register = -1;
690 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX &&
691 Inst.Register == 19 && Inst.Offset <= 248) {
692 Inst.Operation = Win64EH::UOP_SaveR19R20X;
693 Inst.Register = -1;
694 } else if (Inst.Operation == Win64EH::UOP_AddFP && Inst.Offset == 0) {
695 Inst.Operation = Win64EH::UOP_SetFP;
696 } else if (Inst.Operation == Win64EH::UOP_SaveRegP &&
697 Inst.Register == PrevRegister + 2 &&
698 Inst.Offset == PrevOffset + 16) {
699 Inst.Operation = Win64EH::UOP_SaveNext;
700 Inst.Register = -1;
701 Inst.Offset = 0;
702 // Intentionally not creating UOP_SaveNext for float register pairs,
703 // as current versions of Windows (up to at least 20.04) is buggy
704 // regarding SaveNext for float pairs.
705 }
706 // Update info about the previous instruction, for detecting if
707 // the next one can be made a UOP_SaveNext
708 if (Inst.Operation == Win64EH::UOP_SaveR19R20X) {
709 PrevOffset = 0;
710 PrevRegister = 19;
711 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX) {
712 PrevOffset = 0;
713 PrevRegister = Inst.Register;
714 } else if (Inst.Operation == Win64EH::UOP_SaveRegP) {
715 PrevOffset = Inst.Offset;
716 PrevRegister = Inst.Register;
717 } else if (Inst.Operation == Win64EH::UOP_SaveNext) {
718 PrevRegister += 2;
719 PrevOffset += 16;
720 } else {
721 PrevRegister = -1;
722 PrevOffset = -1;
723 }
724 };
725
726 // Iterate over instructions in a forward order (for prologues),
727 // backwards for epilogues (i.e. always reverse compared to how the
728 // opcodes are stored).
729 if (Reverse) {
730 for (auto It = Instructions.rbegin(); It != Instructions.rend(); It++)
731 VisitInstruction(*It);
732 } else {
733 for (WinEH::Instruction &Inst : Instructions)
734 VisitInstruction(Inst);
735 }
736}
737
738// Check if an epilog exists as a subset of the end of a prolog (backwards).
739static int
740getARM64OffsetInProlog(const std::vector<WinEH::Instruction> &Prolog,
741 const std::vector<WinEH::Instruction> &Epilog) {
742 // Can't find an epilog as a subset if it is longer than the prolog.
743 if (Epilog.size() > Prolog.size())
744 return -1;
745
746 // Check that the epilog actually is a perfect match for the end (backwrds)
747 // of the prolog.
748 for (int I = Epilog.size() - 1; I >= 0; I--) {
749 if (Prolog[I] != Epilog[Epilog.size() - 1 - I])
750 return -1;
751 }
752
753 if (Epilog.size() == Prolog.size())
754 return 0;
755
756 // If the epilog was a subset of the prolog, find its offset.
757 return ARM64CountOfUnwindCodes(Insns: ArrayRef<WinEH::Instruction>(
758 &Prolog[Epilog.size()], Prolog.size() - Epilog.size()));
759}
760
761static int checkARM64PackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info,
762 WinEH::FrameInfo::Segment *Seg,
763 int PrologCodeBytes) {
764 // Can only pack if there's one single epilog
765 if (Seg->Epilogs.size() != 1)
766 return -1;
767
768 MCSymbol *Sym = Seg->Epilogs.begin()->first;
769 const std::vector<WinEH::Instruction> &Epilog =
770 info->EpilogMap[Sym].Instructions;
771
772 // Check that the epilog actually is at the very end of the function,
773 // otherwise it can't be packed.
774 uint32_t DistanceFromEnd =
775 (uint32_t)(Seg->Offset + Seg->Length - Seg->Epilogs.begin()->second);
776 if (DistanceFromEnd / 4 != Epilog.size())
777 return -1;
778
779 int RetVal = -1;
780 // Even if we don't end up sharing opcodes with the prolog, we can still
781 // write the offset as a packed offset, if the single epilog is located at
782 // the end of the function and the offset (pointing after the prolog) fits
783 // as a packed offset.
784 if (PrologCodeBytes <= 31 &&
785 PrologCodeBytes + ARM64CountOfUnwindCodes(Insns: Epilog) <= 124)
786 RetVal = PrologCodeBytes;
787
788 int Offset = getARM64OffsetInProlog(Prolog: info->Instructions, Epilog);
789 if (Offset < 0)
790 return RetVal;
791
792 // Check that the offset and prolog size fits in the first word; it's
793 // unclear whether the epilog count in the extension word can be taken
794 // as packed epilog offset.
795 if (Offset > 31 || PrologCodeBytes > 124)
796 return RetVal;
797
798 // As we choose to express the epilog as part of the prolog, remove the
799 // epilog from the map, so we don't try to emit its opcodes.
800 info->EpilogMap.erase(Key: Sym);
801 return Offset;
802}
803
804static bool tryARM64PackedUnwind(WinEH::FrameInfo *info, uint32_t FuncLength,
805 int PackedEpilogOffset) {
806 if (PackedEpilogOffset == 0) {
807 // Fully symmetric prolog and epilog, should be ok for packed format.
808 // For CR=3, the corresponding synthesized epilog actually lacks the
809 // SetFP opcode, but unwinding should work just fine despite that
810 // (if at the SetFP opcode, the unwinder considers it as part of the
811 // function body and just unwinds the full prolog instead).
812 } else if (PackedEpilogOffset == 1) {
813 // One single case of differences between prolog and epilog is allowed:
814 // The epilog can lack a single SetFP that is the last opcode in the
815 // prolog, for the CR=3 case.
816 if (info->Instructions.back().Operation != Win64EH::UOP_SetFP)
817 return false;
818 } else {
819 // Too much difference between prolog and epilog.
820 return false;
821 }
822 unsigned RegI = 0, RegF = 0;
823 int Predecrement = 0;
824 enum {
825 Start,
826 Start2,
827 Start3,
828 IntRegs,
829 FloatRegs,
830 InputArgs,
831 StackAdjust,
832 FrameRecord,
833 End
834 } Location = Start;
835 bool StandaloneLR = false, FPLRPair = false;
836 bool PAC = false;
837 int StackOffset = 0;
838 int Nops = 0;
839 // Iterate over the prolog and check that all opcodes exactly match
840 // the canonical order and form. A more lax check could verify that
841 // all saved registers are in the expected locations, but not enforce
842 // the order - that would work fine when unwinding from within
843 // functions, but not be exactly right if unwinding happens within
844 // prologs/epilogs.
845 for (const WinEH::Instruction &Inst : info->Instructions) {
846 switch (Inst.Operation) {
847 case Win64EH::UOP_End:
848 if (Location != Start)
849 return false;
850 Location = Start2;
851 break;
852 case Win64EH::UOP_PACSignLR:
853 if (Location != Start2)
854 return false;
855 PAC = true;
856 Location = Start3;
857 break;
858 case Win64EH::UOP_SaveR19R20X:
859 if (Location != Start2 && Location != Start3)
860 return false;
861 Predecrement = Inst.Offset;
862 RegI = 2;
863 Location = IntRegs;
864 break;
865 case Win64EH::UOP_SaveRegX:
866 if (Location != Start2 && Location != Start3)
867 return false;
868 Predecrement = Inst.Offset;
869 if (Inst.Register == 19)
870 RegI += 1;
871 else if (Inst.Register == 30)
872 StandaloneLR = true;
873 else
874 return false;
875 // Odd register; can't be any further int registers.
876 Location = FloatRegs;
877 break;
878 case Win64EH::UOP_SaveRegPX:
879 // Can't have this in a canonical prologue. Either this has been
880 // canonicalized into SaveR19R20X or SaveFPLRX, or it's an unsupported
881 // register pair.
882 // It can't be canonicalized into SaveR19R20X if the offset is
883 // larger than 248 bytes, but even with the maximum case with
884 // RegI=10/RegF=8/CR=1/H=1, we end up with SavSZ = 216, which should
885 // fit into SaveR19R20X.
886 // The unwinding opcodes can't describe the otherwise seemingly valid
887 // case for RegI=1 CR=1, that would start with a
888 // "stp x19, lr, [sp, #-...]!" as that fits neither SaveRegPX nor
889 // SaveLRPair.
890 return false;
891 case Win64EH::UOP_SaveRegP:
892 if (Location != IntRegs || Inst.Offset != 8 * RegI ||
893 Inst.Register != 19 + RegI)
894 return false;
895 RegI += 2;
896 break;
897 case Win64EH::UOP_SaveReg:
898 if (Location != IntRegs || Inst.Offset != 8 * RegI)
899 return false;
900 if (Inst.Register == 19 + RegI)
901 RegI += 1;
902 else if (Inst.Register == 30)
903 StandaloneLR = true;
904 else
905 return false;
906 // Odd register; can't be any further int registers.
907 Location = FloatRegs;
908 break;
909 case Win64EH::UOP_SaveLRPair:
910 if (Location != IntRegs || Inst.Offset != 8 * RegI ||
911 Inst.Register != 19 + RegI)
912 return false;
913 RegI += 1;
914 StandaloneLR = true;
915 Location = FloatRegs;
916 break;
917 case Win64EH::UOP_SaveFRegX:
918 // Packed unwind can't handle prologs that only save one single
919 // float register.
920 return false;
921 case Win64EH::UOP_SaveFReg:
922 if (Location != FloatRegs || RegF == 0 || Inst.Register != 8 + RegF ||
923 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF))
924 return false;
925 RegF += 1;
926 Location = InputArgs;
927 break;
928 case Win64EH::UOP_SaveFRegPX:
929 if ((Location != Start2 && Location != Start3) || Inst.Register != 8)
930 return false;
931 Predecrement = Inst.Offset;
932 RegF = 2;
933 Location = FloatRegs;
934 break;
935 case Win64EH::UOP_SaveFRegP:
936 if ((Location != IntRegs && Location != FloatRegs) ||
937 Inst.Register != 8 + RegF ||
938 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF))
939 return false;
940 RegF += 2;
941 Location = FloatRegs;
942 break;
943 case Win64EH::UOP_SaveNext:
944 if (Location == IntRegs)
945 RegI += 2;
946 else if (Location == FloatRegs)
947 RegF += 2;
948 else
949 return false;
950 break;
951 case Win64EH::UOP_Nop:
952 if (Location != IntRegs && Location != FloatRegs && Location != InputArgs)
953 return false;
954 Location = InputArgs;
955 Nops++;
956 break;
957 case Win64EH::UOP_AllocSmall:
958 case Win64EH::UOP_AllocMedium:
959 if (Location != Start2 && Location != Start3 && Location != IntRegs &&
960 Location != FloatRegs && Location != InputArgs &&
961 Location != StackAdjust)
962 return false;
963 // Can have either a single decrement, or a pair of decrements with
964 // 4080 and another decrement.
965 if (StackOffset == 0)
966 StackOffset = Inst.Offset;
967 else if (StackOffset != 4080)
968 return false;
969 else
970 StackOffset += Inst.Offset;
971 Location = StackAdjust;
972 break;
973 case Win64EH::UOP_SaveFPLRX:
974 // Not allowing FPLRX after StackAdjust; if a StackAdjust is used, it
975 // should be followed by a FPLR instead.
976 if (Location != Start2 && Location != Start3 && Location != IntRegs &&
977 Location != FloatRegs && Location != InputArgs)
978 return false;
979 StackOffset = Inst.Offset;
980 Location = FrameRecord;
981 FPLRPair = true;
982 break;
983 case Win64EH::UOP_SaveFPLR:
984 // This can only follow after a StackAdjust
985 if (Location != StackAdjust || Inst.Offset != 0)
986 return false;
987 Location = FrameRecord;
988 FPLRPair = true;
989 break;
990 case Win64EH::UOP_SetFP:
991 if (Location != FrameRecord)
992 return false;
993 Location = End;
994 break;
995 case Win64EH::UOP_SaveAnyRegI:
996 case Win64EH::UOP_SaveAnyRegIP:
997 case Win64EH::UOP_SaveAnyRegD:
998 case Win64EH::UOP_SaveAnyRegDP:
999 case Win64EH::UOP_SaveAnyRegQ:
1000 case Win64EH::UOP_SaveAnyRegQP:
1001 case Win64EH::UOP_SaveAnyRegIX:
1002 case Win64EH::UOP_SaveAnyRegIPX:
1003 case Win64EH::UOP_SaveAnyRegDX:
1004 case Win64EH::UOP_SaveAnyRegDPX:
1005 case Win64EH::UOP_SaveAnyRegQX:
1006 case Win64EH::UOP_SaveAnyRegQPX:
1007 // These are never canonical; they don't show up with the usual Arm64
1008 // calling convention.
1009 return false;
1010 case Win64EH::UOP_AllocLarge:
1011 // Allocations this large can't be represented in packed unwind (and
1012 // usually don't fit the canonical form anyway because we need to use
1013 // __chkstk to allocate the stack space).
1014 return false;
1015 case Win64EH::UOP_AddFP:
1016 // "add x29, sp, #N" doesn't show up in the canonical pattern (except for
1017 // N=0, which is UOP_SetFP).
1018 return false;
1019 case Win64EH::UOP_TrapFrame:
1020 case Win64EH::UOP_Context:
1021 case Win64EH::UOP_ECContext:
1022 case Win64EH::UOP_ClearUnwoundToCall:
1023 case Win64EH::UOP_PushMachFrame:
1024 // These are special opcodes that aren't normally generated.
1025 return false;
1026 default:
1027 report_fatal_error(reason: "Unknown Arm64 unwind opcode");
1028 }
1029 }
1030 if (RegI > 10 || RegF > 8)
1031 return false;
1032 if (StandaloneLR && FPLRPair)
1033 return false;
1034 if (FPLRPair && Location != End)
1035 return false;
1036 if (Nops != 0 && Nops != 4)
1037 return false;
1038 if (PAC && !FPLRPair)
1039 return false;
1040 int H = Nops == 4;
1041 // There's an inconsistency regarding packed unwind info with homed
1042 // parameters; according to the documentation, the epilog shouldn't have
1043 // the same corresponding nops (and thus, to set the H bit, we should
1044 // require an epilog which isn't exactly symmetrical - we shouldn't accept
1045 // an exact mirrored epilog for those cases), but in practice,
1046 // RtlVirtualUnwind behaves as if it does expect the epilogue to contain
1047 // the same nops. See https://github.com/llvm/llvm-project/issues/54879.
1048 // To play it safe, don't produce packed unwind info with homed parameters.
1049 if (H)
1050 return false;
1051 int IntSZ = 8 * RegI;
1052 if (StandaloneLR)
1053 IntSZ += 8;
1054 int FpSZ = 8 * RegF; // RegF not yet decremented
1055 int SavSZ = (IntSZ + FpSZ + 8 * 8 * H + 0xF) & ~0xF;
1056 if (Predecrement != SavSZ)
1057 return false;
1058 if (FPLRPair && StackOffset < 16)
1059 return false;
1060 if (StackOffset % 16)
1061 return false;
1062 uint32_t FrameSize = (StackOffset + SavSZ) / 16;
1063 if (FrameSize > 0x1FF)
1064 return false;
1065 assert(RegF != 1 && "One single float reg not allowed");
1066 if (RegF > 0)
1067 RegF--; // Convert from actual number of registers, to value stored
1068 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier");
1069 int Flag = 0x01; // Function segments not supported yet
1070 int CR = PAC ? 2 : FPLRPair ? 3 : StandaloneLR ? 1 : 0;
1071 info->PackedInfo |= Flag << 0;
1072 info->PackedInfo |= (FuncLength & 0x7FF) << 2;
1073 info->PackedInfo |= (RegF & 0x7) << 13;
1074 info->PackedInfo |= (RegI & 0xF) << 16;
1075 info->PackedInfo |= (H & 0x1) << 20;
1076 info->PackedInfo |= (CR & 0x3) << 21;
1077 info->PackedInfo |= (FrameSize & 0x1FF) << 23;
1078 return true;
1079}
1080
1081static void ARM64ProcessEpilogs(WinEH::FrameInfo *info,
1082 WinEH::FrameInfo::Segment *Seg,
1083 uint32_t &TotalCodeBytes,
1084 MapVector<MCSymbol *, uint32_t> &EpilogInfo) {
1085
1086 std::vector<MCSymbol *> EpilogStarts;
1087 for (auto &I : Seg->Epilogs)
1088 EpilogStarts.push_back(x: I.first);
1089
1090 // Epilogs processed so far.
1091 std::vector<MCSymbol *> AddedEpilogs;
1092 for (auto *S : EpilogStarts) {
1093 MCSymbol *EpilogStart = S;
1094 auto &EpilogInstrs = info->EpilogMap[S].Instructions;
1095 uint32_t CodeBytes = ARM64CountOfUnwindCodes(Insns: EpilogInstrs);
1096
1097 MCSymbol* MatchingEpilog =
1098 FindMatchingEpilog(EpilogInstrs, Epilogs: AddedEpilogs, info);
1099 int PrologOffset;
1100 if (MatchingEpilog) {
1101 assert(EpilogInfo.contains(MatchingEpilog) &&
1102 "Duplicate epilog not found");
1103 EpilogInfo[EpilogStart] = EpilogInfo.lookup(Key: MatchingEpilog);
1104 // Clear the unwind codes in the EpilogMap, so that they don't get output
1105 // in ARM64EmitUnwindInfoForSegment().
1106 EpilogInstrs.clear();
1107 } else if ((PrologOffset = getARM64OffsetInProlog(Prolog: info->Instructions,
1108 Epilog: EpilogInstrs)) >= 0) {
1109 EpilogInfo[EpilogStart] = PrologOffset;
1110 // If the segment doesn't have a prolog, an end_c will be emitted before
1111 // prolog opcodes. So epilog start index in opcodes array is moved by 1.
1112 if (!Seg->HasProlog)
1113 EpilogInfo[EpilogStart] += 1;
1114 // Clear the unwind codes in the EpilogMap, so that they don't get output
1115 // in ARM64EmitUnwindInfoForSegment().
1116 EpilogInstrs.clear();
1117 } else {
1118 EpilogInfo[EpilogStart] = TotalCodeBytes;
1119 TotalCodeBytes += CodeBytes;
1120 AddedEpilogs.push_back(x: EpilogStart);
1121 }
1122 }
1123}
1124
1125static void ARM64FindSegmentsInFunction(MCStreamer &streamer,
1126 WinEH::FrameInfo *info,
1127 int64_t RawFuncLength) {
1128 if (info->PrologEnd)
1129 checkARM64Instructions(Streamer&: streamer, Insns: info->Instructions, Begin: info->Begin,
1130 End: info->PrologEnd, Name: info->Function->getName(),
1131 Type: "prologue");
1132 struct EpilogStartEnd {
1133 MCSymbol *Start;
1134 int64_t Offset;
1135 int64_t End;
1136 };
1137 // Record Start and End of each epilog.
1138 SmallVector<struct EpilogStartEnd, 4> Epilogs;
1139 for (auto &I : info->EpilogMap) {
1140 MCSymbol *Start = I.first;
1141 auto &Instrs = I.second.Instructions;
1142 int64_t Offset = GetAbsDifference(Streamer&: streamer, LHS: Start, RHS: info->Begin);
1143 checkARM64Instructions(Streamer&: streamer, Insns: Instrs, Begin: Start, End: I.second.End,
1144 Name: info->Function->getName(), Type: "epilogue");
1145 assert((Epilogs.size() == 0 || Offset >= Epilogs.back().End) &&
1146 "Epilogs should be monotonically ordered");
1147 // Exclue the end opcode from Instrs.size() when calculating the end of the
1148 // epilog.
1149 Epilogs.push_back(Elt: {.Start: Start, .Offset: Offset, .End: Offset + (int64_t)(Instrs.size() - 1) * 4});
1150 }
1151
1152 unsigned E = 0;
1153 int64_t SegLimit = 0xFFFFC;
1154 int64_t SegOffset = 0;
1155
1156 if (RawFuncLength > SegLimit) {
1157
1158 int64_t RemainingLength = RawFuncLength;
1159
1160 while (RemainingLength > SegLimit) {
1161 // Try divide the function into segments, requirements:
1162 // 1. Segment length <= 0xFFFFC;
1163 // 2. Each Prologue or Epilogue must be fully within a segment.
1164 int64_t SegLength = SegLimit;
1165 int64_t SegEnd = SegOffset + SegLength;
1166 // Keep record on symbols and offsets of epilogs in this segment.
1167 MapVector<MCSymbol *, int64_t> EpilogsInSegment;
1168
1169 while (E < Epilogs.size() && Epilogs[E].End < SegEnd) {
1170 // Epilogs within current segment.
1171 EpilogsInSegment[Epilogs[E].Start] = Epilogs[E].Offset;
1172 ++E;
1173 }
1174
1175 // At this point, we have:
1176 // 1. Put all epilogs in segments already. No action needed here; or
1177 // 2. Found an epilog that will cross segments boundry. We need to
1178 // move back current segment's end boundry, so the epilog is entirely
1179 // in the next segment; or
1180 // 3. Left at least one epilog that is entirely after this segment.
1181 // It'll be handled by the next iteration, or the last segment.
1182 if (E < Epilogs.size() && Epilogs[E].Offset <= SegEnd)
1183 // Move back current Segment's end boundry.
1184 SegLength = Epilogs[E].Offset - SegOffset;
1185
1186 auto Seg = WinEH::FrameInfo::Segment(
1187 SegOffset, SegLength, /* HasProlog */!SegOffset);
1188 Seg.Epilogs = std::move(EpilogsInSegment);
1189 info->Segments.push_back(x: Seg);
1190
1191 SegOffset += SegLength;
1192 RemainingLength -= SegLength;
1193 }
1194 }
1195
1196 // Add the last segment when RawFuncLength > 0xFFFFC,
1197 // or the only segment otherwise.
1198 auto LastSeg =
1199 WinEH::FrameInfo::Segment(SegOffset, RawFuncLength - SegOffset,
1200 /* HasProlog */!SegOffset);
1201 for (; E < Epilogs.size(); ++E)
1202 LastSeg.Epilogs[Epilogs[E].Start] = Epilogs[E].Offset;
1203 info->Segments.push_back(x: LastSeg);
1204}
1205
1206static void ARM64EmitUnwindInfoForSegment(MCStreamer &streamer,
1207 WinEH::FrameInfo *info,
1208 WinEH::FrameInfo::Segment &Seg,
1209 bool TryPacked = true) {
1210 MCContext &context = streamer.getContext();
1211 MCSymbol *Label = context.createTempSymbol();
1212
1213 streamer.emitValueToAlignment(Alignment: Align(4));
1214 streamer.emitLabel(Symbol: Label);
1215 Seg.Symbol = Label;
1216 // Use the 1st segemnt's label as function's.
1217 if (Seg.Offset == 0)
1218 info->Symbol = Label;
1219
1220 bool HasProlog = Seg.HasProlog;
1221 bool HasEpilogs = (Seg.Epilogs.size() != 0);
1222
1223 uint32_t SegLength = (uint32_t)Seg.Length / 4;
1224 uint32_t PrologCodeBytes = info->PrologCodeBytes;
1225
1226 int PackedEpilogOffset = HasEpilogs ?
1227 checkARM64PackedEpilog(streamer, info, Seg: &Seg, PrologCodeBytes) : -1;
1228
1229 // TODO:
1230 // 1. Enable packed unwind info (.pdata only) for multi-segment functions.
1231 // 2. Emit packed unwind info (.pdata only) for segments that have neithor
1232 // prolog nor epilog.
1233 if (info->Segments.size() == 1 && PackedEpilogOffset >= 0 &&
1234 uint32_t(PackedEpilogOffset) < PrologCodeBytes &&
1235 !info->HandlesExceptions && SegLength <= 0x7ff && TryPacked) {
1236 // Matching prolog/epilog and no exception handlers; check if the
1237 // prolog matches the patterns that can be described by the packed
1238 // format.
1239
1240 // info->Symbol was already set even if we didn't actually write any
1241 // unwind info there. Keep using that as indicator that this unwind
1242 // info has been generated already.
1243 if (tryARM64PackedUnwind(info, FuncLength: SegLength, PackedEpilogOffset))
1244 return;
1245 }
1246
1247 // If the prolog is not in this segment, we need to emit an end_c, which takes
1248 // 1 byte, before prolog unwind ops.
1249 if (!HasProlog) {
1250 PrologCodeBytes += 1;
1251 if (PackedEpilogOffset >= 0)
1252 PackedEpilogOffset += 1;
1253 // If a segment has neither prolog nor epilog, "With full .xdata record,
1254 // Epilog Count = 1. Epilog Start Index points to end_c."
1255 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#function-fragments
1256 // TODO: We can remove this if testing shows zero epilog scope is ok with
1257 // MS unwinder.
1258 if (!HasEpilogs)
1259 // Pack the fake epilog into phantom prolog.
1260 PackedEpilogOffset = 0;
1261 }
1262
1263 uint32_t TotalCodeBytes = PrologCodeBytes;
1264
1265 // Process epilogs.
1266 MapVector<MCSymbol *, uint32_t> EpilogInfo;
1267 ARM64ProcessEpilogs(info, Seg: &Seg, TotalCodeBytes, EpilogInfo);
1268
1269 // Code Words, Epilog count, E, X, Vers, Function Length
1270 uint32_t row1 = 0x0;
1271 uint32_t CodeWords = TotalCodeBytes / 4;
1272 uint32_t CodeWordsMod = TotalCodeBytes % 4;
1273 if (CodeWordsMod)
1274 CodeWords++;
1275 uint32_t EpilogCount =
1276 PackedEpilogOffset >= 0 ? PackedEpilogOffset : Seg.Epilogs.size();
1277 bool ExtensionWord = EpilogCount > 31 || TotalCodeBytes > 124;
1278 if (!ExtensionWord) {
1279 row1 |= (EpilogCount & 0x1F) << 22;
1280 row1 |= (CodeWords & 0x1F) << 27;
1281 }
1282 if (info->HandlesExceptions) // X
1283 row1 |= 1 << 20;
1284 if (PackedEpilogOffset >= 0) // E
1285 row1 |= 1 << 21;
1286 row1 |= SegLength & 0x3FFFF;
1287 streamer.emitInt32(Value: row1);
1288
1289 // Extended Code Words, Extended Epilog Count
1290 if (ExtensionWord) {
1291 // FIXME: We should be able to split unwind info into multiple sections.
1292 if (CodeWords > 0xFF || EpilogCount > 0xFFFF)
1293 report_fatal_error(
1294 reason: "SEH unwind data splitting is only implemented for large functions, "
1295 "cases of too many code words or too many epilogs will be done "
1296 "later");
1297 uint32_t row2 = 0x0;
1298 row2 |= (CodeWords & 0xFF) << 16;
1299 row2 |= (EpilogCount & 0xFFFF);
1300 streamer.emitInt32(Value: row2);
1301 }
1302
1303 if (PackedEpilogOffset < 0) {
1304 // Epilog Start Index, Epilog Start Offset
1305 for (auto &I : EpilogInfo) {
1306 MCSymbol *EpilogStart = I.first;
1307 uint32_t EpilogIndex = I.second;
1308 // Epilog offset within the Segment.
1309 uint32_t EpilogOffset = (uint32_t)(Seg.Epilogs[EpilogStart] - Seg.Offset);
1310 if (EpilogOffset)
1311 EpilogOffset /= 4;
1312 uint32_t row3 = EpilogOffset;
1313 row3 |= (EpilogIndex & 0x3FF) << 22;
1314 streamer.emitInt32(Value: row3);
1315 }
1316 }
1317
1318 // Note that even for segments that have no prolog, we still need to emit
1319 // prolog unwinding opcodes so that the unwinder knows how to unwind from
1320 // such a segment.
1321 // The end_c opcode at the start indicates to the unwinder that the actual
1322 // prolog is outside of the current segment, and the unwinder shouldn't try
1323 // to check for unwinding from a partial prolog.
1324 if (!HasProlog)
1325 // Emit an end_c.
1326 streamer.emitInt8(Value: (uint8_t)0xE5);
1327
1328 // Emit prolog unwind instructions (in reverse order).
1329 for (auto Inst : llvm::reverse(C&: info->Instructions))
1330 ARM64EmitUnwindCode(streamer, inst: Inst);
1331
1332 // Emit epilog unwind instructions
1333 for (auto &I : Seg.Epilogs) {
1334 auto &EpilogInstrs = info->EpilogMap[I.first].Instructions;
1335 for (const WinEH::Instruction &inst : EpilogInstrs)
1336 ARM64EmitUnwindCode(streamer, inst);
1337 }
1338
1339 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes;
1340 assert(BytesMod >= 0);
1341 for (int i = 0; i < BytesMod; i++)
1342 streamer.emitInt8(Value: 0xE3);
1343
1344 if (info->HandlesExceptions)
1345 streamer.emitValue(
1346 Value: MCSymbolRefExpr::create(Symbol: info->ExceptionHandler,
1347 Kind: MCSymbolRefExpr::VK_COFF_IMGREL32, Ctx&: context),
1348 Size: 4);
1349}
1350
1351// Populate the .xdata section. The format of .xdata on ARM64 is documented at
1352// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
1353static void ARM64EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info,
1354 bool TryPacked = true) {
1355 // If this UNWIND_INFO already has a symbol, it's already been emitted.
1356 if (info->Symbol)
1357 return;
1358 // If there's no unwind info here (not even a terminating UOP_End), the
1359 // unwind info is considered bogus and skipped. If this was done in
1360 // response to an explicit .seh_handlerdata, the associated trailing
1361 // handler data is left orphaned in the xdata section.
1362 if (info->empty()) {
1363 info->EmitAttempted = true;
1364 return;
1365 }
1366 if (info->EmitAttempted) {
1367 // If we tried to emit unwind info before (due to an explicit
1368 // .seh_handlerdata directive), but skipped it (because there was no
1369 // valid information to emit at the time), and it later got valid unwind
1370 // opcodes, we can't emit it here, because the trailing handler data
1371 // was already emitted elsewhere in the xdata section.
1372 streamer.getContext().reportError(
1373 L: SMLoc(), Msg: "Earlier .seh_handlerdata for " + info->Function->getName() +
1374 " skipped due to no unwind info at the time "
1375 "(.seh_handlerdata too early?), but the function later "
1376 "did get unwind info that can't be emitted");
1377 return;
1378 }
1379
1380 simplifyARM64Opcodes(Instructions&: info->Instructions, Reverse: false);
1381 for (auto &I : info->EpilogMap)
1382 simplifyARM64Opcodes(Instructions&: I.second.Instructions, Reverse: true);
1383
1384 int64_t RawFuncLength;
1385 if (!info->FuncletOrFuncEnd) {
1386 report_fatal_error(reason: "FuncletOrFuncEnd not set");
1387 } else {
1388 // FIXME: GetAbsDifference tries to compute the length of the function
1389 // immediately, before the whole file is emitted, but in general
1390 // that's impossible: the size in bytes of certain assembler directives
1391 // like .align and .fill is not known until the whole file is parsed and
1392 // relaxations are applied. Currently, GetAbsDifference fails with a fatal
1393 // error in that case. (We mostly don't hit this because inline assembly
1394 // specifying those directives is rare, and we don't normally try to
1395 // align loops on AArch64.)
1396 //
1397 // There are two potential approaches to delaying the computation. One,
1398 // we could emit something like ".word (endfunc-beginfunc)/4+0x10800000",
1399 // as long as we have some conservative estimate we could use to prove
1400 // that we don't need to split the unwind data. Emitting the constant
1401 // is straightforward, but there's no existing code for estimating the
1402 // size of the function.
1403 //
1404 // The other approach would be to use a dedicated, relaxable fragment,
1405 // which could grow to accommodate splitting the unwind data if
1406 // necessary. This is more straightforward, since it automatically works
1407 // without any new infrastructure, and it's consistent with how we handle
1408 // relaxation in other contexts. But it would require some refactoring
1409 // to move parts of the pdata/xdata emission into the implementation of
1410 // a fragment. We could probably continue to encode the unwind codes
1411 // here, but we'd have to emit the pdata, the xdata header, and the
1412 // epilogue scopes later, since they depend on whether the we need to
1413 // split the unwind data.
1414 //
1415 // If this is fixed, remove code in AArch64ISelLowering.cpp that
1416 // disables loop alignment on Windows.
1417 RawFuncLength = GetAbsDifference(Streamer&: streamer, LHS: info->FuncletOrFuncEnd,
1418 RHS: info->Begin);
1419 }
1420
1421 ARM64FindSegmentsInFunction(streamer, info, RawFuncLength);
1422
1423 info->PrologCodeBytes = ARM64CountOfUnwindCodes(Insns: info->Instructions);
1424 for (auto &S : info->Segments)
1425 ARM64EmitUnwindInfoForSegment(streamer, info, Seg&: S, TryPacked);
1426
1427 // Clear prolog instructions after unwind info is emitted for all segments.
1428 info->Instructions.clear();
1429}
1430
1431static uint32_t ARMCountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) {
1432 uint32_t Count = 0;
1433 for (const auto &I : Insns) {
1434 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {
1435 default:
1436 llvm_unreachable("Unsupported ARM unwind code");
1437 case Win64EH::UOP_AllocSmall:
1438 Count += 1;
1439 break;
1440 case Win64EH::UOP_AllocLarge:
1441 Count += 3;
1442 break;
1443 case Win64EH::UOP_AllocHuge:
1444 Count += 4;
1445 break;
1446 case Win64EH::UOP_WideAllocMedium:
1447 Count += 2;
1448 break;
1449 case Win64EH::UOP_WideAllocLarge:
1450 Count += 3;
1451 break;
1452 case Win64EH::UOP_WideAllocHuge:
1453 Count += 4;
1454 break;
1455 case Win64EH::UOP_WideSaveRegMask:
1456 Count += 2;
1457 break;
1458 case Win64EH::UOP_SaveSP:
1459 Count += 1;
1460 break;
1461 case Win64EH::UOP_SaveRegsR4R7LR:
1462 Count += 1;
1463 break;
1464 case Win64EH::UOP_WideSaveRegsR4R11LR:
1465 Count += 1;
1466 break;
1467 case Win64EH::UOP_SaveFRegD8D15:
1468 Count += 1;
1469 break;
1470 case Win64EH::UOP_SaveRegMask:
1471 Count += 2;
1472 break;
1473 case Win64EH::UOP_SaveLR:
1474 Count += 2;
1475 break;
1476 case Win64EH::UOP_SaveFRegD0D15:
1477 Count += 2;
1478 break;
1479 case Win64EH::UOP_SaveFRegD16D31:
1480 Count += 2;
1481 break;
1482 case Win64EH::UOP_Nop:
1483 case Win64EH::UOP_WideNop:
1484 case Win64EH::UOP_End:
1485 case Win64EH::UOP_EndNop:
1486 case Win64EH::UOP_WideEndNop:
1487 Count += 1;
1488 break;
1489 case Win64EH::UOP_Custom: {
1490 int J;
1491 for (J = 3; J > 0; J--)
1492 if (I.Offset & (0xffu << (8 * J)))
1493 break;
1494 Count += J + 1;
1495 break;
1496 }
1497 }
1498 }
1499 return Count;
1500}
1501
1502static uint32_t ARMCountOfInstructionBytes(ArrayRef<WinEH::Instruction> Insns,
1503 bool *HasCustom = nullptr) {
1504 uint32_t Count = 0;
1505 for (const auto &I : Insns) {
1506 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {
1507 default:
1508 llvm_unreachable("Unsupported ARM unwind code");
1509 case Win64EH::UOP_AllocSmall:
1510 case Win64EH::UOP_AllocLarge:
1511 case Win64EH::UOP_AllocHuge:
1512 Count += 2;
1513 break;
1514 case Win64EH::UOP_WideAllocMedium:
1515 case Win64EH::UOP_WideAllocLarge:
1516 case Win64EH::UOP_WideAllocHuge:
1517 Count += 4;
1518 break;
1519 case Win64EH::UOP_WideSaveRegMask:
1520 case Win64EH::UOP_WideSaveRegsR4R11LR:
1521 Count += 4;
1522 break;
1523 case Win64EH::UOP_SaveSP:
1524 Count += 2;
1525 break;
1526 case Win64EH::UOP_SaveRegMask:
1527 case Win64EH::UOP_SaveRegsR4R7LR:
1528 Count += 2;
1529 break;
1530 case Win64EH::UOP_SaveFRegD8D15:
1531 case Win64EH::UOP_SaveFRegD0D15:
1532 case Win64EH::UOP_SaveFRegD16D31:
1533 Count += 4;
1534 break;
1535 case Win64EH::UOP_SaveLR:
1536 Count += 4;
1537 break;
1538 case Win64EH::UOP_Nop:
1539 case Win64EH::UOP_EndNop:
1540 Count += 2;
1541 break;
1542 case Win64EH::UOP_WideNop:
1543 case Win64EH::UOP_WideEndNop:
1544 Count += 4;
1545 break;
1546 case Win64EH::UOP_End:
1547 // This doesn't map to any instruction
1548 break;
1549 case Win64EH::UOP_Custom:
1550 // We can't reason about what instructions this maps to; return a
1551 // phony number to make sure we don't accidentally do epilog packing.
1552 Count += 1000;
1553 if (HasCustom)
1554 *HasCustom = true;
1555 break;
1556 }
1557 }
1558 return Count;
1559}
1560
1561static void checkARMInstructions(MCStreamer &Streamer,
1562 ArrayRef<WinEH::Instruction> Insns,
1563 const MCSymbol *Begin, const MCSymbol *End,
1564 StringRef Name, StringRef Type) {
1565 if (!End)
1566 return;
1567 std::optional<int64_t> MaybeDistance =
1568 GetOptionalAbsDifference(Streamer, LHS: End, RHS: Begin);
1569 if (!MaybeDistance)
1570 return;
1571 uint32_t Distance = (uint32_t)*MaybeDistance;
1572 bool HasCustom = false;
1573 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Insns, HasCustom: &HasCustom);
1574 if (HasCustom)
1575 return;
1576 if (Distance != InstructionBytes) {
1577 Streamer.getContext().reportError(
1578 L: SMLoc(), Msg: "Incorrect size for " + Name + " " + Type + ": " +
1579 Twine(Distance) +
1580 " bytes of instructions in range, but .seh directives "
1581 "corresponding to " +
1582 Twine(InstructionBytes) + " bytes\n");
1583 }
1584}
1585
1586static bool isARMTerminator(const WinEH::Instruction &inst) {
1587 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {
1588 case Win64EH::UOP_End:
1589 case Win64EH::UOP_EndNop:
1590 case Win64EH::UOP_WideEndNop:
1591 return true;
1592 default:
1593 return false;
1594 }
1595}
1596
1597// Unwind opcode encodings and restrictions are documented at
1598// https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling
1599static void ARMEmitUnwindCode(MCStreamer &streamer,
1600 const WinEH::Instruction &inst) {
1601 uint32_t w, lr;
1602 int i;
1603 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {
1604 default:
1605 llvm_unreachable("Unsupported ARM unwind code");
1606 case Win64EH::UOP_AllocSmall:
1607 assert((inst.Offset & 3) == 0);
1608 assert(inst.Offset / 4 <= 0x7f);
1609 streamer.emitInt8(Value: inst.Offset / 4);
1610 break;
1611 case Win64EH::UOP_WideSaveRegMask:
1612 assert((inst.Register & ~0x5fff) == 0);
1613 lr = (inst.Register >> 14) & 1;
1614 w = 0x8000 | (inst.Register & 0x1fff) | (lr << 13);
1615 streamer.emitInt8(Value: (w >> 8) & 0xff);
1616 streamer.emitInt8(Value: (w >> 0) & 0xff);
1617 break;
1618 case Win64EH::UOP_SaveSP:
1619 assert(inst.Register <= 0x0f);
1620 streamer.emitInt8(Value: 0xc0 | inst.Register);
1621 break;
1622 case Win64EH::UOP_SaveRegsR4R7LR:
1623 assert(inst.Register >= 4 && inst.Register <= 7);
1624 assert(inst.Offset <= 1);
1625 streamer.emitInt8(Value: 0xd0 | (inst.Register - 4) | (inst.Offset << 2));
1626 break;
1627 case Win64EH::UOP_WideSaveRegsR4R11LR:
1628 assert(inst.Register >= 8 && inst.Register <= 11);
1629 assert(inst.Offset <= 1);
1630 streamer.emitInt8(Value: 0xd8 | (inst.Register - 8) | (inst.Offset << 2));
1631 break;
1632 case Win64EH::UOP_SaveFRegD8D15:
1633 assert(inst.Register >= 8 && inst.Register <= 15);
1634 streamer.emitInt8(Value: 0xe0 | (inst.Register - 8));
1635 break;
1636 case Win64EH::UOP_WideAllocMedium:
1637 assert((inst.Offset & 3) == 0);
1638 assert(inst.Offset / 4 <= 0x3ff);
1639 w = 0xe800 | (inst.Offset / 4);
1640 streamer.emitInt8(Value: (w >> 8) & 0xff);
1641 streamer.emitInt8(Value: (w >> 0) & 0xff);
1642 break;
1643 case Win64EH::UOP_SaveRegMask:
1644 assert((inst.Register & ~0x40ff) == 0);
1645 lr = (inst.Register >> 14) & 1;
1646 w = 0xec00 | (inst.Register & 0x0ff) | (lr << 8);
1647 streamer.emitInt8(Value: (w >> 8) & 0xff);
1648 streamer.emitInt8(Value: (w >> 0) & 0xff);
1649 break;
1650 case Win64EH::UOP_SaveLR:
1651 assert((inst.Offset & 3) == 0);
1652 assert(inst.Offset / 4 <= 0x0f);
1653 streamer.emitInt8(Value: 0xef);
1654 streamer.emitInt8(Value: inst.Offset / 4);
1655 break;
1656 case Win64EH::UOP_SaveFRegD0D15:
1657 assert(inst.Register <= 15);
1658 assert(inst.Offset <= 15);
1659 assert(inst.Register <= inst.Offset);
1660 streamer.emitInt8(Value: 0xf5);
1661 streamer.emitInt8(Value: (inst.Register << 4) | inst.Offset);
1662 break;
1663 case Win64EH::UOP_SaveFRegD16D31:
1664 assert(inst.Register >= 16 && inst.Register <= 31);
1665 assert(inst.Offset >= 16 && inst.Offset <= 31);
1666 assert(inst.Register <= inst.Offset);
1667 streamer.emitInt8(Value: 0xf6);
1668 streamer.emitInt8(Value: ((inst.Register - 16) << 4) | (inst.Offset - 16));
1669 break;
1670 case Win64EH::UOP_AllocLarge:
1671 assert((inst.Offset & 3) == 0);
1672 assert(inst.Offset / 4 <= 0xffff);
1673 w = inst.Offset / 4;
1674 streamer.emitInt8(Value: 0xf7);
1675 streamer.emitInt8(Value: (w >> 8) & 0xff);
1676 streamer.emitInt8(Value: (w >> 0) & 0xff);
1677 break;
1678 case Win64EH::UOP_AllocHuge:
1679 assert((inst.Offset & 3) == 0);
1680 assert(inst.Offset / 4 <= 0xffffff);
1681 w = inst.Offset / 4;
1682 streamer.emitInt8(Value: 0xf8);
1683 streamer.emitInt8(Value: (w >> 16) & 0xff);
1684 streamer.emitInt8(Value: (w >> 8) & 0xff);
1685 streamer.emitInt8(Value: (w >> 0) & 0xff);
1686 break;
1687 case Win64EH::UOP_WideAllocLarge:
1688 assert((inst.Offset & 3) == 0);
1689 assert(inst.Offset / 4 <= 0xffff);
1690 w = inst.Offset / 4;
1691 streamer.emitInt8(Value: 0xf9);
1692 streamer.emitInt8(Value: (w >> 8) & 0xff);
1693 streamer.emitInt8(Value: (w >> 0) & 0xff);
1694 break;
1695 case Win64EH::UOP_WideAllocHuge:
1696 assert((inst.Offset & 3) == 0);
1697 assert(inst.Offset / 4 <= 0xffffff);
1698 w = inst.Offset / 4;
1699 streamer.emitInt8(Value: 0xfa);
1700 streamer.emitInt8(Value: (w >> 16) & 0xff);
1701 streamer.emitInt8(Value: (w >> 8) & 0xff);
1702 streamer.emitInt8(Value: (w >> 0) & 0xff);
1703 break;
1704 case Win64EH::UOP_Nop:
1705 streamer.emitInt8(Value: 0xfb);
1706 break;
1707 case Win64EH::UOP_WideNop:
1708 streamer.emitInt8(Value: 0xfc);
1709 break;
1710 case Win64EH::UOP_EndNop:
1711 streamer.emitInt8(Value: 0xfd);
1712 break;
1713 case Win64EH::UOP_WideEndNop:
1714 streamer.emitInt8(Value: 0xfe);
1715 break;
1716 case Win64EH::UOP_End:
1717 streamer.emitInt8(Value: 0xff);
1718 break;
1719 case Win64EH::UOP_Custom:
1720 for (i = 3; i > 0; i--)
1721 if (inst.Offset & (0xffu << (8 * i)))
1722 break;
1723 for (; i >= 0; i--)
1724 streamer.emitInt8(Value: (inst.Offset >> (8 * i)) & 0xff);
1725 break;
1726 }
1727}
1728
1729// Check if an epilog exists as a subset of the end of a prolog (backwards).
1730// An epilog may end with one out of three different end opcodes; if this
1731// is the first epilog that shares opcodes with the prolog, we can tolerate
1732// that this opcode differs (and the caller will update the prolog to use
1733// the same end opcode as the epilog). If another epilog already shares
1734// opcodes with the prolog, the ending opcode must be a strict match.
1735static int getARMOffsetInProlog(const std::vector<WinEH::Instruction> &Prolog,
1736 const std::vector<WinEH::Instruction> &Epilog,
1737 bool CanTweakProlog) {
1738 // Can't find an epilog as a subset if it is longer than the prolog.
1739 if (Epilog.size() > Prolog.size())
1740 return -1;
1741
1742 // Check that the epilog actually is a perfect match for the end (backwrds)
1743 // of the prolog.
1744 // If we can adjust the prolog afterwards, don't check that the end opcodes
1745 // match.
1746 int EndIdx = CanTweakProlog ? 1 : 0;
1747 for (int I = Epilog.size() - 1; I >= EndIdx; I--) {
1748 // TODO: Could also allow minor mismatches, e.g. "add sp, #16" vs
1749 // "push {r0-r3}".
1750 if (Prolog[I] != Epilog[Epilog.size() - 1 - I])
1751 return -1;
1752 }
1753
1754 if (CanTweakProlog) {
1755 // Check that both prolog and epilog end with an expected end opcode.
1756 if (Prolog.front().Operation != Win64EH::UOP_End)
1757 return -1;
1758 if (Epilog.back().Operation != Win64EH::UOP_End &&
1759 Epilog.back().Operation != Win64EH::UOP_EndNop &&
1760 Epilog.back().Operation != Win64EH::UOP_WideEndNop)
1761 return -1;
1762 }
1763
1764 // If the epilog was a subset of the prolog, find its offset.
1765 if (Epilog.size() == Prolog.size())
1766 return 0;
1767 return ARMCountOfUnwindCodes(Insns: ArrayRef<WinEH::Instruction>(
1768 &Prolog[Epilog.size()], Prolog.size() - Epilog.size()));
1769}
1770
1771static int checkARMPackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info,
1772 int PrologCodeBytes) {
1773 // Can only pack if there's one single epilog
1774 if (info->EpilogMap.size() != 1)
1775 return -1;
1776
1777 const WinEH::FrameInfo::Epilog &EpilogInfo = info->EpilogMap.begin()->second;
1778 // Can only pack if the epilog is unconditional
1779 if (EpilogInfo.Condition != 0xe) // ARMCC::AL
1780 return -1;
1781
1782 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions;
1783 // Make sure we have at least the trailing end opcode
1784 if (info->Instructions.empty() || Epilog.empty())
1785 return -1;
1786
1787 // Check that the epilog actually is at the very end of the function,
1788 // otherwise it can't be packed.
1789 std::optional<int64_t> MaybeDistance = GetOptionalAbsDifference(
1790 Streamer&: streamer, LHS: info->FuncletOrFuncEnd, RHS: info->EpilogMap.begin()->first);
1791 if (!MaybeDistance)
1792 return -1;
1793 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance;
1794 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Insns: Epilog);
1795 if (DistanceFromEnd != InstructionBytes)
1796 return -1;
1797
1798 int RetVal = -1;
1799 // Even if we don't end up sharing opcodes with the prolog, we can still
1800 // write the offset as a packed offset, if the single epilog is located at
1801 // the end of the function and the offset (pointing after the prolog) fits
1802 // as a packed offset.
1803 if (PrologCodeBytes <= 31 &&
1804 PrologCodeBytes + ARMCountOfUnwindCodes(Insns: Epilog) <= 63)
1805 RetVal = PrologCodeBytes;
1806
1807 int Offset =
1808 getARMOffsetInProlog(Prolog: info->Instructions, Epilog, /*CanTweakProlog=*/true);
1809 if (Offset < 0)
1810 return RetVal;
1811
1812 // Check that the offset and prolog size fits in the first word; it's
1813 // unclear whether the epilog count in the extension word can be taken
1814 // as packed epilog offset.
1815 if (Offset > 31 || PrologCodeBytes > 63)
1816 return RetVal;
1817
1818 // Replace the regular end opcode of the prolog with the one from the
1819 // epilog.
1820 info->Instructions.front() = Epilog.back();
1821
1822 // As we choose to express the epilog as part of the prolog, remove the
1823 // epilog from the map, so we don't try to emit its opcodes.
1824 info->EpilogMap.clear();
1825 return Offset;
1826}
1827
1828static bool parseRegMask(unsigned Mask, bool &HasLR, bool &HasR11,
1829 unsigned &Folded, int &IntRegs) {
1830 if (Mask & (1 << 14)) {
1831 HasLR = true;
1832 Mask &= ~(1 << 14);
1833 }
1834 if (Mask & (1 << 11)) {
1835 HasR11 = true;
1836 Mask &= ~(1 << 11);
1837 }
1838 Folded = 0;
1839 IntRegs = -1;
1840 if (!Mask)
1841 return true;
1842 int First = 0;
1843 // Shift right until we have the bits at the bottom
1844 while ((Mask & 1) == 0) {
1845 First++;
1846 Mask >>= 1;
1847 }
1848 if ((Mask & (Mask + 1)) != 0)
1849 return false; // Not a consecutive series of bits? Can't be packed.
1850 // Count the bits
1851 int N = 0;
1852 while (Mask & (1 << N))
1853 N++;
1854 if (First < 4) {
1855 if (First + N < 4)
1856 return false;
1857 Folded = 4 - First;
1858 N -= Folded;
1859 First = 4;
1860 }
1861 if (First > 4)
1862 return false; // Can't be packed
1863 if (N >= 1)
1864 IntRegs = N - 1;
1865 return true;
1866}
1867
1868static bool tryARMPackedUnwind(MCStreamer &streamer, WinEH::FrameInfo *info,
1869 uint32_t FuncLength) {
1870 int Step = 0;
1871 bool Homing = false;
1872 bool HasR11 = false;
1873 bool HasChain = false;
1874 bool HasLR = false;
1875 int IntRegs = -1; // r4 - r(4+N)
1876 int FloatRegs = -1; // d8 - d(8+N)
1877 unsigned PF = 0; // Number of extra pushed registers
1878 unsigned StackAdjust = 0;
1879 // Iterate over the prolog and check that all opcodes exactly match
1880 // the canonical order and form.
1881 for (const WinEH::Instruction &Inst : info->Instructions) {
1882 switch (Inst.Operation) {
1883 default:
1884 llvm_unreachable("Unsupported ARM unwind code");
1885 case Win64EH::UOP_Custom:
1886 case Win64EH::UOP_AllocLarge:
1887 case Win64EH::UOP_AllocHuge:
1888 case Win64EH::UOP_WideAllocLarge:
1889 case Win64EH::UOP_WideAllocHuge:
1890 case Win64EH::UOP_SaveFRegD0D15:
1891 case Win64EH::UOP_SaveFRegD16D31:
1892 // Can't be packed
1893 return false;
1894 case Win64EH::UOP_SaveSP:
1895 // Can't be packed; we can't rely on restoring sp from r11 when
1896 // unwinding a packed prologue.
1897 return false;
1898 case Win64EH::UOP_SaveLR:
1899 // Can't be present in a packed prologue
1900 return false;
1901
1902 case Win64EH::UOP_End:
1903 case Win64EH::UOP_EndNop:
1904 case Win64EH::UOP_WideEndNop:
1905 if (Step != 0)
1906 return false;
1907 Step = 1;
1908 break;
1909
1910 case Win64EH::UOP_SaveRegsR4R7LR:
1911 case Win64EH::UOP_WideSaveRegsR4R11LR:
1912 // push {r4-r11,lr}
1913 if (Step != 1 && Step != 2)
1914 return false;
1915 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX
1916 assert(Inst.Offset <= 1); // Lr
1917 IntRegs = Inst.Register - 4;
1918 if (Inst.Register == 11) {
1919 HasR11 = true;
1920 IntRegs--;
1921 }
1922 if (Inst.Offset)
1923 HasLR = true;
1924 Step = 3;
1925 break;
1926
1927 case Win64EH::UOP_SaveRegMask:
1928 if (Step == 1 && Inst.Register == 0x0f) {
1929 // push {r0-r3}
1930 Homing = true;
1931 Step = 2;
1932 break;
1933 }
1934 [[fallthrough]];
1935 case Win64EH::UOP_WideSaveRegMask:
1936 if (Step != 1 && Step != 2)
1937 return false;
1938 // push {r4-r9,r11,lr}
1939 // push {r11,lr}
1940 // push {r1-r5}
1941 if (!parseRegMask(Mask: Inst.Register, HasLR, HasR11, Folded&: PF, IntRegs))
1942 return false;
1943 Step = 3;
1944 break;
1945
1946 case Win64EH::UOP_Nop:
1947 // mov r11, sp
1948 if (Step != 3 || !HasR11 || IntRegs >= 0 || PF > 0)
1949 return false;
1950 HasChain = true;
1951 Step = 4;
1952 break;
1953 case Win64EH::UOP_WideNop:
1954 // add.w r11, sp, #xx
1955 if (Step != 3 || !HasR11 || (IntRegs < 0 && PF == 0))
1956 return false;
1957 HasChain = true;
1958 Step = 4;
1959 break;
1960
1961 case Win64EH::UOP_SaveFRegD8D15:
1962 if (Step != 1 && Step != 2 && Step != 3 && Step != 4)
1963 return false;
1964 assert(Inst.Register >= 8 && Inst.Register <= 15);
1965 if (Inst.Register == 15)
1966 return false; // Can't pack this case, R==7 means no IntRegs
1967 if (IntRegs >= 0)
1968 return false;
1969 FloatRegs = Inst.Register - 8;
1970 Step = 5;
1971 break;
1972
1973 case Win64EH::UOP_AllocSmall:
1974 case Win64EH::UOP_WideAllocMedium:
1975 if (Step != 1 && Step != 2 && Step != 3 && Step != 4 && Step != 5)
1976 return false;
1977 if (PF > 0) // Can't have both folded and explicit stack allocation
1978 return false;
1979 if (Inst.Offset / 4 >= 0x3f4)
1980 return false;
1981 StackAdjust = Inst.Offset / 4;
1982 Step = 6;
1983 break;
1984 }
1985 }
1986 if (HasR11 && !HasChain) {
1987 if (IntRegs + 4 == 10) {
1988 // r11 stored, but not chaining; can be packed if already saving r4-r10
1989 // and we can fit r11 into this range.
1990 IntRegs++;
1991 HasR11 = false;
1992 } else
1993 return false;
1994 }
1995 if (HasChain && !HasLR)
1996 return false;
1997
1998 // Packed uneind info can't express multiple epilogues.
1999 if (info->EpilogMap.size() > 1)
2000 return false;
2001
2002 unsigned EF = 0;
2003 int Ret = 0;
2004 if (info->EpilogMap.size() == 0) {
2005 Ret = 3; // No epilogue
2006 } else {
2007 // As the prologue and epilogue aren't exact mirrors of each other,
2008 // we have to check the epilogue too and see if it matches what we've
2009 // concluded from the prologue.
2010 const WinEH::FrameInfo::Epilog &EpilogInfo =
2011 info->EpilogMap.begin()->second;
2012 if (EpilogInfo.Condition != 0xe) // ARMCC::AL
2013 return false;
2014 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions;
2015 std::optional<int64_t> MaybeDistance = GetOptionalAbsDifference(
2016 Streamer&: streamer, LHS: info->FuncletOrFuncEnd, RHS: info->EpilogMap.begin()->first);
2017 if (!MaybeDistance)
2018 return false;
2019 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance;
2020 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Insns: Epilog);
2021 if (DistanceFromEnd != InstructionBytes)
2022 return false;
2023
2024 bool GotStackAdjust = false;
2025 bool GotFloatRegs = false;
2026 bool GotIntRegs = false;
2027 bool GotHomingRestore = false;
2028 bool GotLRRestore = false;
2029 bool NeedsReturn = false;
2030 bool GotReturn = false;
2031
2032 Step = 6;
2033 for (const WinEH::Instruction &Inst : Epilog) {
2034 switch (Inst.Operation) {
2035 default:
2036 llvm_unreachable("Unsupported ARM unwind code");
2037 case Win64EH::UOP_Custom:
2038 case Win64EH::UOP_AllocLarge:
2039 case Win64EH::UOP_AllocHuge:
2040 case Win64EH::UOP_WideAllocLarge:
2041 case Win64EH::UOP_WideAllocHuge:
2042 case Win64EH::UOP_SaveFRegD0D15:
2043 case Win64EH::UOP_SaveFRegD16D31:
2044 case Win64EH::UOP_SaveSP:
2045 case Win64EH::UOP_Nop:
2046 case Win64EH::UOP_WideNop:
2047 // Can't be packed in an epilogue
2048 return false;
2049
2050 case Win64EH::UOP_AllocSmall:
2051 case Win64EH::UOP_WideAllocMedium:
2052 if (Inst.Offset / 4 >= 0x3f4)
2053 return false;
2054 if (Step == 6) {
2055 if (Homing && FloatRegs < 0 && IntRegs < 0 && StackAdjust == 0 &&
2056 PF == 0 && Inst.Offset == 16) {
2057 GotHomingRestore = true;
2058 Step = 10;
2059 } else {
2060 if (StackAdjust > 0) {
2061 // Got stack adjust in prologue too; must match.
2062 if (StackAdjust != Inst.Offset / 4)
2063 return false;
2064 GotStackAdjust = true;
2065 } else if (PF == Inst.Offset / 4) {
2066 // Folded prologue, non-folded epilogue
2067 StackAdjust = Inst.Offset / 4;
2068 GotStackAdjust = true;
2069 } else {
2070 // StackAdjust == 0 in prologue, mismatch
2071 return false;
2072 }
2073 Step = 7;
2074 }
2075 } else if (Step == 7 || Step == 8 || Step == 9) {
2076 if (!Homing || Inst.Offset != 16)
2077 return false;
2078 GotHomingRestore = true;
2079 Step = 10;
2080 } else
2081 return false;
2082 break;
2083
2084 case Win64EH::UOP_SaveFRegD8D15:
2085 if (Step != 6 && Step != 7)
2086 return false;
2087 assert(Inst.Register >= 8 && Inst.Register <= 15);
2088 if (FloatRegs != (int)(Inst.Register - 8))
2089 return false;
2090 GotFloatRegs = true;
2091 Step = 8;
2092 break;
2093
2094 case Win64EH::UOP_SaveRegsR4R7LR:
2095 case Win64EH::UOP_WideSaveRegsR4R11LR: {
2096 // push {r4-r11,lr}
2097 if (Step != 6 && Step != 7 && Step != 8)
2098 return false;
2099 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX
2100 assert(Inst.Offset <= 1); // Lr
2101 if (Homing && HasLR) {
2102 // If homing and LR is backed up, we can either restore LR here
2103 // and return with Ret == 1 or 2, or return with SaveLR below
2104 if (Inst.Offset) {
2105 GotLRRestore = true;
2106 NeedsReturn = true;
2107 } else {
2108 // Expecting a separate SaveLR below
2109 }
2110 } else {
2111 if (HasLR != (Inst.Offset == 1))
2112 return false;
2113 }
2114 GotLRRestore = Inst.Offset == 1;
2115 if (IntRegs < 0) // This opcode must include r4
2116 return false;
2117 int Expected = IntRegs;
2118 if (HasChain) {
2119 // Can't express r11 here unless IntRegs describe r4-r10
2120 if (IntRegs != 6)
2121 return false;
2122 Expected++;
2123 }
2124 if (Expected != (int)(Inst.Register - 4))
2125 return false;
2126 GotIntRegs = true;
2127 Step = 9;
2128 break;
2129 }
2130
2131 case Win64EH::UOP_SaveRegMask:
2132 case Win64EH::UOP_WideSaveRegMask: {
2133 if (Step != 6 && Step != 7 && Step != 8)
2134 return false;
2135 // push {r4-r9,r11,lr}
2136 // push {r11,lr}
2137 // push {r1-r5}
2138 bool CurHasLR = false, CurHasR11 = false;
2139 int Regs;
2140 if (!parseRegMask(Mask: Inst.Register, HasLR&: CurHasLR, HasR11&: CurHasR11, Folded&: EF, IntRegs&: Regs))
2141 return false;
2142 if (EF > 0) {
2143 if (EF != PF && EF != StackAdjust)
2144 return false;
2145 }
2146 if (Homing && HasLR) {
2147 // If homing and LR is backed up, we can either restore LR here
2148 // and return with Ret == 1 or 2, or return with SaveLR below
2149 if (CurHasLR) {
2150 GotLRRestore = true;
2151 NeedsReturn = true;
2152 } else {
2153 // Expecting a separate SaveLR below
2154 }
2155 } else {
2156 if (CurHasLR != HasLR)
2157 return false;
2158 GotLRRestore = CurHasLR;
2159 }
2160 int Expected = IntRegs;
2161 if (HasChain) {
2162 // If we have chaining, the mask must have included r11.
2163 if (!CurHasR11)
2164 return false;
2165 } else if (Expected == 7) {
2166 // If we don't have chaining, the mask could still include r11,
2167 // expressed as part of IntRegs Instead.
2168 Expected--;
2169 if (!CurHasR11)
2170 return false;
2171 } else {
2172 // Neither HasChain nor r11 included in IntRegs, must not have r11
2173 // here either.
2174 if (CurHasR11)
2175 return false;
2176 }
2177 if (Expected != Regs)
2178 return false;
2179 GotIntRegs = true;
2180 Step = 9;
2181 break;
2182 }
2183
2184 case Win64EH::UOP_SaveLR:
2185 if (Step != 6 && Step != 7 && Step != 8 && Step != 9)
2186 return false;
2187 if (!Homing || Inst.Offset != 20 || GotLRRestore)
2188 return false;
2189 GotLRRestore = true;
2190 GotHomingRestore = true;
2191 Step = 10;
2192 break;
2193
2194 case Win64EH::UOP_EndNop:
2195 case Win64EH::UOP_WideEndNop:
2196 GotReturn = true;
2197 Ret = (Inst.Operation == Win64EH::UOP_EndNop) ? 1 : 2;
2198 [[fallthrough]];
2199 case Win64EH::UOP_End:
2200 if (Step != 6 && Step != 7 && Step != 8 && Step != 9 && Step != 10)
2201 return false;
2202 Step = 11;
2203 break;
2204 }
2205 }
2206
2207 if (Step != 11)
2208 return false;
2209 if (StackAdjust > 0 && !GotStackAdjust && EF == 0)
2210 return false;
2211 if (FloatRegs >= 0 && !GotFloatRegs)
2212 return false;
2213 if (IntRegs >= 0 && !GotIntRegs)
2214 return false;
2215 if (Homing && !GotHomingRestore)
2216 return false;
2217 if (HasLR && !GotLRRestore)
2218 return false;
2219 if (NeedsReturn && !GotReturn)
2220 return false;
2221 }
2222
2223 assert(PF == 0 || EF == 0 ||
2224 StackAdjust == 0); // Can't have adjust in all three
2225 if (PF > 0 || EF > 0) {
2226 StackAdjust = PF > 0 ? (PF - 1) : (EF - 1);
2227 assert(StackAdjust <= 3);
2228 StackAdjust |= 0x3f0;
2229 if (PF > 0)
2230 StackAdjust |= 1 << 2;
2231 if (EF > 0)
2232 StackAdjust |= 1 << 3;
2233 }
2234
2235 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier");
2236 int Flag = info->Fragment ? 0x02 : 0x01;
2237 int H = Homing ? 1 : 0;
2238 int L = HasLR ? 1 : 0;
2239 int C = HasChain ? 1 : 0;
2240 assert(IntRegs < 0 || FloatRegs < 0);
2241 unsigned Reg, R;
2242 if (IntRegs >= 0) {
2243 Reg = IntRegs;
2244 assert(Reg <= 7);
2245 R = 0;
2246 } else if (FloatRegs >= 0) {
2247 Reg = FloatRegs;
2248 assert(Reg < 7);
2249 R = 1;
2250 } else {
2251 // No int or float regs stored (except possibly R11,LR)
2252 Reg = 7;
2253 R = 1;
2254 }
2255 info->PackedInfo |= Flag << 0;
2256 info->PackedInfo |= (FuncLength & 0x7FF) << 2;
2257 info->PackedInfo |= (Ret & 0x3) << 13;
2258 info->PackedInfo |= H << 15;
2259 info->PackedInfo |= Reg << 16;
2260 info->PackedInfo |= R << 19;
2261 info->PackedInfo |= L << 20;
2262 info->PackedInfo |= C << 21;
2263 assert(StackAdjust <= 0x3ff);
2264 info->PackedInfo |= StackAdjust << 22;
2265 return true;
2266}
2267
2268// Populate the .xdata section. The format of .xdata on ARM is documented at
2269// https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling
2270static void ARMEmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info,
2271 bool TryPacked = true) {
2272 // If this UNWIND_INFO already has a symbol, it's already been emitted.
2273 if (info->Symbol)
2274 return;
2275 // If there's no unwind info here (not even a terminating UOP_End), the
2276 // unwind info is considered bogus and skipped. If this was done in
2277 // response to an explicit .seh_handlerdata, the associated trailing
2278 // handler data is left orphaned in the xdata section.
2279 if (info->empty()) {
2280 info->EmitAttempted = true;
2281 return;
2282 }
2283 if (info->EmitAttempted) {
2284 // If we tried to emit unwind info before (due to an explicit
2285 // .seh_handlerdata directive), but skipped it (because there was no
2286 // valid information to emit at the time), and it later got valid unwind
2287 // opcodes, we can't emit it here, because the trailing handler data
2288 // was already emitted elsewhere in the xdata section.
2289 streamer.getContext().reportError(
2290 L: SMLoc(), Msg: "Earlier .seh_handlerdata for " + info->Function->getName() +
2291 " skipped due to no unwind info at the time "
2292 "(.seh_handlerdata too early?), but the function later "
2293 "did get unwind info that can't be emitted");
2294 return;
2295 }
2296
2297 MCContext &context = streamer.getContext();
2298 MCSymbol *Label = context.createTempSymbol();
2299
2300 streamer.emitValueToAlignment(Alignment: Align(4));
2301 streamer.emitLabel(Symbol: Label);
2302 info->Symbol = Label;
2303
2304 if (!info->PrologEnd)
2305 streamer.getContext().reportError(L: SMLoc(), Msg: "Prologue in " +
2306 info->Function->getName() +
2307 " not correctly terminated");
2308
2309 if (info->PrologEnd && !info->Fragment)
2310 checkARMInstructions(Streamer&: streamer, Insns: info->Instructions, Begin: info->Begin,
2311 End: info->PrologEnd, Name: info->Function->getName(),
2312 Type: "prologue");
2313 for (auto &I : info->EpilogMap) {
2314 MCSymbol *EpilogStart = I.first;
2315 auto &Epilog = I.second;
2316 checkARMInstructions(Streamer&: streamer, Insns: Epilog.Instructions, Begin: EpilogStart, End: Epilog.End,
2317 Name: info->Function->getName(), Type: "epilogue");
2318 if (Epilog.Instructions.empty() ||
2319 !isARMTerminator(inst: Epilog.Instructions.back()))
2320 streamer.getContext().reportError(
2321 L: SMLoc(), Msg: "Epilogue in " + info->Function->getName() +
2322 " not correctly terminated");
2323 }
2324
2325 std::optional<int64_t> RawFuncLength;
2326 const MCExpr *FuncLengthExpr = nullptr;
2327 if (!info->FuncletOrFuncEnd) {
2328 report_fatal_error(reason: "FuncletOrFuncEnd not set");
2329 } else {
2330 // As the size of many thumb2 instructions isn't known until later,
2331 // we can't always rely on being able to calculate the absolute
2332 // length of the function here. If we can't calculate it, defer it
2333 // to a relocation.
2334 //
2335 // In such a case, we won't know if the function is too long so that
2336 // the unwind info would need to be split (but this isn't implemented
2337 // anyway).
2338 RawFuncLength =
2339 GetOptionalAbsDifference(Streamer&: streamer, LHS: info->FuncletOrFuncEnd, RHS: info->Begin);
2340 if (!RawFuncLength)
2341 FuncLengthExpr =
2342 GetSubDivExpr(Streamer&: streamer, LHS: info->FuncletOrFuncEnd, RHS: info->Begin, Div: 2);
2343 }
2344 uint32_t FuncLength = 0;
2345 if (RawFuncLength)
2346 FuncLength = (uint32_t)*RawFuncLength / 2;
2347 if (FuncLength > 0x3FFFF)
2348 report_fatal_error(reason: "SEH unwind data splitting not yet implemented");
2349 uint32_t PrologCodeBytes = ARMCountOfUnwindCodes(Insns: info->Instructions);
2350 uint32_t TotalCodeBytes = PrologCodeBytes;
2351
2352 if (!info->HandlesExceptions && RawFuncLength && FuncLength <= 0x7ff &&
2353 TryPacked) {
2354 // No exception handlers; check if the prolog and epilog matches the
2355 // patterns that can be described by the packed format. If we don't
2356 // know the exact function length yet, we can't do this.
2357
2358 // info->Symbol was already set even if we didn't actually write any
2359 // unwind info there. Keep using that as indicator that this unwind
2360 // info has been generated already.
2361
2362 if (tryARMPackedUnwind(streamer, info, FuncLength))
2363 return;
2364 }
2365
2366 int PackedEpilogOffset =
2367 checkARMPackedEpilog(streamer, info, PrologCodeBytes);
2368
2369 // Process epilogs.
2370 MapVector<MCSymbol *, uint32_t> EpilogInfo;
2371 // Epilogs processed so far.
2372 std::vector<MCSymbol *> AddedEpilogs;
2373
2374 bool CanTweakProlog = true;
2375 for (auto &I : info->EpilogMap) {
2376 MCSymbol *EpilogStart = I.first;
2377 auto &EpilogInstrs = I.second.Instructions;
2378 uint32_t CodeBytes = ARMCountOfUnwindCodes(Insns: EpilogInstrs);
2379
2380 MCSymbol *MatchingEpilog =
2381 FindMatchingEpilog(EpilogInstrs, Epilogs: AddedEpilogs, info);
2382 int PrologOffset;
2383 if (MatchingEpilog) {
2384 assert(EpilogInfo.contains(MatchingEpilog) &&
2385 "Duplicate epilog not found");
2386 EpilogInfo[EpilogStart] = EpilogInfo.lookup(Key: MatchingEpilog);
2387 // Clear the unwind codes in the EpilogMap, so that they don't get output
2388 // in the logic below.
2389 EpilogInstrs.clear();
2390 } else if ((PrologOffset = getARMOffsetInProlog(
2391 Prolog: info->Instructions, Epilog: EpilogInstrs, CanTweakProlog)) >= 0) {
2392 if (CanTweakProlog) {
2393 // Replace the regular end opcode of the prolog with the one from the
2394 // epilog.
2395 info->Instructions.front() = EpilogInstrs.back();
2396 // Later epilogs need a strict match for the end opcode.
2397 CanTweakProlog = false;
2398 }
2399 EpilogInfo[EpilogStart] = PrologOffset;
2400 // Clear the unwind codes in the EpilogMap, so that they don't get output
2401 // in the logic below.
2402 EpilogInstrs.clear();
2403 } else {
2404 EpilogInfo[EpilogStart] = TotalCodeBytes;
2405 TotalCodeBytes += CodeBytes;
2406 AddedEpilogs.push_back(x: EpilogStart);
2407 }
2408 }
2409
2410 // Code Words, Epilog count, F, E, X, Vers, Function Length
2411 uint32_t row1 = 0x0;
2412 uint32_t CodeWords = TotalCodeBytes / 4;
2413 uint32_t CodeWordsMod = TotalCodeBytes % 4;
2414 if (CodeWordsMod)
2415 CodeWords++;
2416 uint32_t EpilogCount =
2417 PackedEpilogOffset >= 0 ? PackedEpilogOffset : info->EpilogMap.size();
2418 bool ExtensionWord = EpilogCount > 31 || CodeWords > 15;
2419 if (!ExtensionWord) {
2420 row1 |= (EpilogCount & 0x1F) << 23;
2421 row1 |= (CodeWords & 0x0F) << 28;
2422 }
2423 if (info->HandlesExceptions) // X
2424 row1 |= 1 << 20;
2425 if (PackedEpilogOffset >= 0) // E
2426 row1 |= 1 << 21;
2427 if (info->Fragment) // F
2428 row1 |= 1 << 22;
2429 row1 |= FuncLength & 0x3FFFF;
2430 if (RawFuncLength)
2431 streamer.emitInt32(Value: row1);
2432 else
2433 streamer.emitValue(
2434 Value: MCBinaryExpr::createOr(LHS: FuncLengthExpr,
2435 RHS: MCConstantExpr::create(Value: row1, Ctx&: context), Ctx&: context),
2436 Size: 4);
2437
2438 // Extended Code Words, Extended Epilog Count
2439 if (ExtensionWord) {
2440 // FIXME: We should be able to split unwind info into multiple sections.
2441 if (CodeWords > 0xFF || EpilogCount > 0xFFFF)
2442 report_fatal_error(reason: "SEH unwind data splitting not yet implemented");
2443 uint32_t row2 = 0x0;
2444 row2 |= (CodeWords & 0xFF) << 16;
2445 row2 |= (EpilogCount & 0xFFFF);
2446 streamer.emitInt32(Value: row2);
2447 }
2448
2449 if (PackedEpilogOffset < 0) {
2450 // Epilog Start Index, Epilog Start Offset
2451 for (auto &I : EpilogInfo) {
2452 MCSymbol *EpilogStart = I.first;
2453 uint32_t EpilogIndex = I.second;
2454
2455 std::optional<int64_t> MaybeEpilogOffset =
2456 GetOptionalAbsDifference(Streamer&: streamer, LHS: EpilogStart, RHS: info->Begin);
2457 const MCExpr *OffsetExpr = nullptr;
2458 uint32_t EpilogOffset = 0;
2459 if (MaybeEpilogOffset)
2460 EpilogOffset = *MaybeEpilogOffset / 2;
2461 else
2462 OffsetExpr = GetSubDivExpr(Streamer&: streamer, LHS: EpilogStart, RHS: info->Begin, Div: 2);
2463
2464 assert(info->EpilogMap.contains(EpilogStart));
2465 unsigned Condition = info->EpilogMap[EpilogStart].Condition;
2466 assert(Condition <= 0xf);
2467
2468 uint32_t row3 = EpilogOffset;
2469 row3 |= Condition << 20;
2470 row3 |= (EpilogIndex & 0x3FF) << 24;
2471 if (MaybeEpilogOffset)
2472 streamer.emitInt32(Value: row3);
2473 else
2474 streamer.emitValue(
2475 Value: MCBinaryExpr::createOr(
2476 LHS: OffsetExpr, RHS: MCConstantExpr::create(Value: row3, Ctx&: context), Ctx&: context),
2477 Size: 4);
2478 }
2479 }
2480
2481 // Emit prolog unwind instructions (in reverse order).
2482 uint8_t numInst = info->Instructions.size();
2483 for (uint8_t c = 0; c < numInst; ++c) {
2484 WinEH::Instruction inst = info->Instructions.back();
2485 info->Instructions.pop_back();
2486 ARMEmitUnwindCode(streamer, inst);
2487 }
2488
2489 // Emit epilog unwind instructions
2490 for (auto &I : info->EpilogMap) {
2491 auto &EpilogInstrs = I.second.Instructions;
2492 for (const WinEH::Instruction &inst : EpilogInstrs)
2493 ARMEmitUnwindCode(streamer, inst);
2494 }
2495
2496 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes;
2497 assert(BytesMod >= 0);
2498 for (int i = 0; i < BytesMod; i++)
2499 streamer.emitInt8(Value: 0xFB);
2500
2501 if (info->HandlesExceptions)
2502 streamer.emitValue(
2503 Value: MCSymbolRefExpr::create(Symbol: info->ExceptionHandler,
2504 Kind: MCSymbolRefExpr::VK_COFF_IMGREL32, Ctx&: context),
2505 Size: 4);
2506}
2507
2508static void ARM64EmitRuntimeFunction(MCStreamer &streamer,
2509 const WinEH::FrameInfo *info) {
2510 MCContext &context = streamer.getContext();
2511
2512 streamer.emitValueToAlignment(Alignment: Align(4));
2513 for (const auto &S : info->Segments) {
2514 EmitSymbolRefWithOfs(streamer, Base: info->Begin, Offset: S.Offset);
2515 if (info->PackedInfo)
2516 streamer.emitInt32(Value: info->PackedInfo);
2517 else
2518 streamer.emitValue(
2519 Value: MCSymbolRefExpr::create(Symbol: S.Symbol, Kind: MCSymbolRefExpr::VK_COFF_IMGREL32,
2520 Ctx&: context),
2521 Size: 4);
2522 }
2523}
2524
2525
2526static void ARMEmitRuntimeFunction(MCStreamer &streamer,
2527 const WinEH::FrameInfo *info) {
2528 MCContext &context = streamer.getContext();
2529
2530 streamer.emitValueToAlignment(Alignment: Align(4));
2531 EmitSymbolRefWithOfs(streamer, Base: info->Begin, Other: info->Begin);
2532 if (info->PackedInfo)
2533 streamer.emitInt32(Value: info->PackedInfo);
2534 else
2535 streamer.emitValue(
2536 Value: MCSymbolRefExpr::create(Symbol: info->Symbol, Kind: MCSymbolRefExpr::VK_COFF_IMGREL32,
2537 Ctx&: context),
2538 Size: 4);
2539}
2540
2541void llvm::Win64EH::ARM64UnwindEmitter::Emit(MCStreamer &Streamer) const {
2542 // Emit the unwind info structs first.
2543 for (const auto &CFI : Streamer.getWinFrameInfos()) {
2544 WinEH::FrameInfo *Info = CFI.get();
2545 if (Info->empty())
2546 continue;
2547 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: CFI->TextSection);
2548 Streamer.switchSection(Section: XData);
2549 ARM64EmitUnwindInfo(streamer&: Streamer, info: Info);
2550 }
2551
2552 // Now emit RUNTIME_FUNCTION entries.
2553 for (const auto &CFI : Streamer.getWinFrameInfos()) {
2554 WinEH::FrameInfo *Info = CFI.get();
2555 // ARM64EmitUnwindInfo above clears the info struct, so we can't check
2556 // empty here. But if a Symbol is set, we should create the corresponding
2557 // pdata entry.
2558 if (!Info->Symbol)
2559 continue;
2560 MCSection *PData = Streamer.getAssociatedPDataSection(TextSec: CFI->TextSection);
2561 Streamer.switchSection(Section: PData);
2562 ARM64EmitRuntimeFunction(streamer&: Streamer, info: Info);
2563 }
2564}
2565
2566void llvm::Win64EH::ARM64UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer,
2567 WinEH::FrameInfo *info,
2568 bool HandlerData) const {
2569 // Called if there's an .seh_handlerdata directive before the end of the
2570 // function. This forces writing the xdata record already here - and
2571 // in this case, the function isn't actually ended already, but the xdata
2572 // record needs to know the function length. In these cases, if the funclet
2573 // end hasn't been marked yet, the xdata function length won't cover the
2574 // whole function, only up to this point.
2575 if (!info->FuncletOrFuncEnd) {
2576 Streamer.switchSection(Section: info->TextSection);
2577 info->FuncletOrFuncEnd = Streamer.emitCFILabel();
2578 }
2579 // Switch sections (the static function above is meant to be called from
2580 // here and from Emit().
2581 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: info->TextSection);
2582 Streamer.switchSection(Section: XData);
2583 ARM64EmitUnwindInfo(streamer&: Streamer, info, /* TryPacked = */ !HandlerData);
2584}
2585
2586void llvm::Win64EH::ARMUnwindEmitter::Emit(MCStreamer &Streamer) const {
2587 // Emit the unwind info structs first.
2588 for (const auto &CFI : Streamer.getWinFrameInfos()) {
2589 WinEH::FrameInfo *Info = CFI.get();
2590 if (Info->empty())
2591 continue;
2592 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: CFI->TextSection);
2593 Streamer.switchSection(Section: XData);
2594 ARMEmitUnwindInfo(streamer&: Streamer, info: Info);
2595 }
2596
2597 // Now emit RUNTIME_FUNCTION entries.
2598 for (const auto &CFI : Streamer.getWinFrameInfos()) {
2599 WinEH::FrameInfo *Info = CFI.get();
2600 // ARMEmitUnwindInfo above clears the info struct, so we can't check
2601 // empty here. But if a Symbol is set, we should create the corresponding
2602 // pdata entry.
2603 if (!Info->Symbol)
2604 continue;
2605 MCSection *PData = Streamer.getAssociatedPDataSection(TextSec: CFI->TextSection);
2606 Streamer.switchSection(Section: PData);
2607 ARMEmitRuntimeFunction(streamer&: Streamer, info: Info);
2608 }
2609}
2610
2611void llvm::Win64EH::ARMUnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer,
2612 WinEH::FrameInfo *info,
2613 bool HandlerData) const {
2614 // Called if there's an .seh_handlerdata directive before the end of the
2615 // function. This forces writing the xdata record already here - and
2616 // in this case, the function isn't actually ended already, but the xdata
2617 // record needs to know the function length. In these cases, if the funclet
2618 // end hasn't been marked yet, the xdata function length won't cover the
2619 // whole function, only up to this point.
2620 if (!info->FuncletOrFuncEnd) {
2621 Streamer.switchSection(Section: info->TextSection);
2622 info->FuncletOrFuncEnd = Streamer.emitCFILabel();
2623 }
2624 // Switch sections (the static function above is meant to be called from
2625 // here and from Emit().
2626 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: info->TextSection);
2627 Streamer.switchSection(Section: XData);
2628 ARMEmitUnwindInfo(streamer&: Streamer, info, /* TryPacked = */ !HandlerData);
2629}
2630