1//===- lib/MC/MCWin64EH.cpp - MCWin64EH implementation --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/MC/MCWin64EH.h"
10#include "llvm/ADT/Twine.h"
11#include "llvm/MC/MCAssembler.h"
12#include "llvm/MC/MCContext.h"
13#include "llvm/MC/MCExpr.h"
14#include "llvm/MC/MCObjectStreamer.h"
15#include "llvm/MC/MCStreamer.h"
16#include "llvm/MC/MCSymbol.h"
17#include "llvm/MC/MCValue.h"
18#include "llvm/Support/ErrorHandling.h"
19#include "llvm/Support/Win64EH.h"
20
21namespace llvm {
22class MCSection;
23}
24
25using namespace llvm;
26
27namespace {
28/// MCExpr that represents the epilog unwind code in an unwind table.
29class MCUnwindV2EpilogTargetExpr final : public MCTargetExpr {
30 const WinEH::FrameInfo &FrameInfo;
31 const MCSymbol *UnwindV2Start;
32 const MCSymbol *EpilogEnd;
33 uint8_t EpilogSize;
34 SMLoc Loc;
35
36 MCUnwindV2EpilogTargetExpr(const WinEH::FrameInfo &FrameInfo,
37 const WinEH::FrameInfo::Epilog &Epilog,
38 uint8_t EpilogSize_)
39 : FrameInfo(FrameInfo), UnwindV2Start(Epilog.UnwindV2Start),
40 EpilogEnd(Epilog.End), EpilogSize(EpilogSize_), Loc(Epilog.Loc) {
41 assert(UnwindV2Start && "Epilog must have a start");
42 assert(EpilogEnd && "Epilog must have an end");
43 }
44
45public:
46 static MCUnwindV2EpilogTargetExpr *
47 create(const WinEH::FrameInfo &FrameInfo,
48 const WinEH::FrameInfo::Epilog &Epilog, uint8_t EpilogSize_,
49 MCContext &Ctx) {
50 return new (Ctx) MCUnwindV2EpilogTargetExpr(FrameInfo, Epilog, EpilogSize_);
51 }
52
53 void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override {
54 OS << ":epilog:";
55 UnwindV2Start->print(OS, MAI);
56 }
57
58 bool evaluateAsRelocatableImpl(MCValue &Res,
59 const MCAssembler *Asm) const override;
60
61 void visitUsedExpr(MCStreamer &Streamer) const override {
62 // Contains no sub-expressions.
63 }
64
65 MCFragment *findAssociatedFragment() const override {
66 return UnwindV2Start->getFragment();
67 }
68};
69
70/// MCExpr representing a V3 epilog's tail-relative EpilogOffset field. The
71/// first epilog descriptor is encoded relative to the fragment end, and each
72/// subsequent descriptor relative to the previous epilog's start. Measuring
73/// from the tail keeps the magnitude small (epilogs sit near the end of the
74/// function), avoiding overflow of the signed 16-bit field for large
75/// functions. The fragment end may not have a symbol yet when the unwind info
76/// is emitted (e.g. via .seh_handlerdata), so the value is resolved lazily
77/// through the FrameInfo reference.
78class MCUnwindV3EpilogOffsetTargetExpr final : public MCTargetExpr {
79 const WinEH::FrameInfo &FrameInfo;
80 const MCSymbol *EpilogStart;
81 const MCSymbol *PrevEpilogStart;
82 SMLoc Loc;
83
84 MCUnwindV3EpilogOffsetTargetExpr(const WinEH::FrameInfo &FrameInfo,
85 const MCSymbol *EpilogStart,
86 const MCSymbol *PrevEpilogStart, SMLoc Loc)
87 : FrameInfo(FrameInfo), EpilogStart(EpilogStart),
88 PrevEpilogStart(PrevEpilogStart), Loc(Loc) {}
89
90public:
91 static MCUnwindV3EpilogOffsetTargetExpr *
92 create(const WinEH::FrameInfo &FrameInfo, const MCSymbol *EpilogStart,
93 const MCSymbol *PrevEpilogStart, SMLoc Loc, MCContext &Ctx) {
94 return new (Ctx) MCUnwindV3EpilogOffsetTargetExpr(FrameInfo, EpilogStart,
95 PrevEpilogStart, Loc);
96 }
97
98 void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override {
99 OS << ":epilogoffset:";
100 EpilogStart->print(OS, MAI);
101 }
102
103 bool evaluateAsRelocatableImpl(MCValue &Res,
104 const MCAssembler *Asm) const override;
105
106 void visitUsedExpr(MCStreamer &Streamer) const override {
107 // Contains no sub-expressions.
108 }
109
110 MCFragment *findAssociatedFragment() const override {
111 return EpilogStart->getFragment();
112 }
113};
114} // namespace
115
116// NOTE: All relocations generated here are 4-byte image-relative.
117
118static uint8_t CountOfUnwindCodes(std::vector<WinEH::Instruction> &Insns) {
119 uint8_t Count = 0;
120 for (const auto &I : Insns) {
121 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {
122 default:
123 llvm_unreachable("Unsupported unwind code");
124 case Win64EH::UOP_PushNonVol:
125 case Win64EH::UOP_AllocSmall:
126 case Win64EH::UOP_SetFPReg:
127 case Win64EH::UOP_PushMachFrame:
128 Count += 1;
129 break;
130 case Win64EH::UOP_SaveNonVol:
131 case Win64EH::UOP_SaveXMM128:
132 Count += 2;
133 break;
134 case Win64EH::UOP_SaveNonVolBig:
135 case Win64EH::UOP_SaveXMM128Big:
136 Count += 3;
137 break;
138 case Win64EH::UOP_AllocLarge:
139 Count += (I.Offset > 512 * 1024 - 8) ? 3 : 2;
140 break;
141 }
142 }
143 return Count;
144}
145
146static void EmitAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS,
147 const MCSymbol *RHS) {
148 MCContext &Context = Streamer.getContext();
149 const MCExpr *Diff =
150 MCBinaryExpr::createSub(LHS: MCSymbolRefExpr::create(Symbol: LHS, Ctx&: Context),
151 RHS: MCSymbolRefExpr::create(Symbol: RHS, Ctx&: Context), Ctx&: Context);
152 Streamer.emitValue(Value: Diff, Size: 1);
153}
154
155/// Emit a 16-bit (2-byte LE) label difference. If the difference is
156/// evaluatable at this point, validate that it fits in [0, UINT16_MAX]
157/// and emit it as a constant; otherwise emit a 16-bit fixup.
158static void EmitAbsDifference16(MCStreamer &Streamer, const MCSymbol *LHS,
159 const MCSymbol *RHS) {
160 MCContext &Context = Streamer.getContext();
161 const MCExpr *Diff =
162 MCBinaryExpr::createSub(LHS: MCSymbolRefExpr::create(Symbol: LHS, Ctx&: Context),
163 RHS: MCSymbolRefExpr::create(Symbol: RHS, Ctx&: Context), Ctx&: Context);
164 int64_t Value;
165 if (Diff->evaluateAsAbsolute(
166 Res&: Value, Asm: static_cast<MCObjectStreamer &>(Streamer).getAssembler())) {
167 if (Value < 0 || Value > UINT16_MAX)
168 Context.reportError(
169 L: SMLoc(),
170 Msg: "Label difference out of 16-bit unsigned range for V3 unwind info");
171 }
172 // Always emit a 2-byte value so subsequent emission stays in sync; if a
173 // diagnostic was reported, the object file will be discarded by the caller.
174 Streamer.emitValue(Value: Diff, Size: 2);
175}
176
177static void EmitUnwindCode(MCStreamer &streamer, const MCSymbol *begin,
178 WinEH::Instruction &inst) {
179 uint8_t b2;
180 uint16_t w;
181 b2 = (inst.Operation & 0x0F);
182 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {
183 default:
184 llvm_unreachable("Unsupported unwind code");
185 case Win64EH::UOP_Push2:
186 // Reachable from hand-written .s if a UOP_Push2 ends up in a V1/V2
187 // frame (e.g. via a per-function `.seh_unwindversion` downgrade after
188 // `.seh_push2regs`). Emit a recoverable diagnostic and skip the op so
189 // the assembler doesn't keep writing malformed bytes.
190 streamer.getContext().reportError(
191 L: SMLoc(), Msg: "UOP_Push2 (PUSH2 with two registers) requires V3 unwind "
192 "info. Use `.seh_unwindversion 3`.");
193 return;
194 case Win64EH::UOP_PushNonVol:
195 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
196 b2 |= (inst.Register & 0x0F) << 4;
197 streamer.emitInt8(Value: b2);
198 break;
199 case Win64EH::UOP_AllocLarge:
200 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
201 if (inst.Offset > 512 * 1024 - 8) {
202 b2 |= 0x10;
203 streamer.emitInt8(Value: b2);
204 w = inst.Offset & 0xFFF8;
205 streamer.emitInt16(Value: w);
206 w = inst.Offset >> 16;
207 } else {
208 streamer.emitInt8(Value: b2);
209 w = inst.Offset >> 3;
210 }
211 streamer.emitInt16(Value: w);
212 break;
213 case Win64EH::UOP_AllocSmall:
214 b2 |= (((inst.Offset - 8) >> 3) & 0x0F) << 4;
215 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
216 streamer.emitInt8(Value: b2);
217 break;
218 case Win64EH::UOP_SetFPReg:
219 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
220 streamer.emitInt8(Value: b2);
221 break;
222 case Win64EH::UOP_SaveNonVol:
223 case Win64EH::UOP_SaveXMM128:
224 b2 |= (inst.Register & 0x0F) << 4;
225 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
226 streamer.emitInt8(Value: b2);
227 w = inst.Offset >> 3;
228 if (inst.Operation == Win64EH::UOP_SaveXMM128)
229 w >>= 1;
230 streamer.emitInt16(Value: w);
231 break;
232 case Win64EH::UOP_SaveNonVolBig:
233 case Win64EH::UOP_SaveXMM128Big:
234 b2 |= (inst.Register & 0x0F) << 4;
235 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
236 streamer.emitInt8(Value: b2);
237 if (inst.Operation == Win64EH::UOP_SaveXMM128Big)
238 w = inst.Offset & 0xFFF0;
239 else
240 w = inst.Offset & 0xFFF8;
241 streamer.emitInt16(Value: w);
242 w = inst.Offset >> 16;
243 streamer.emitInt16(Value: w);
244 break;
245 case Win64EH::UOP_PushMachFrame:
246 if (inst.Offset == 1)
247 b2 |= 0x10;
248 EmitAbsDifference(Streamer&: streamer, LHS: inst.Label, RHS: begin);
249 streamer.emitInt8(Value: b2);
250 break;
251 }
252}
253
254static void EmitSymbolRefWithOfs(MCStreamer &streamer,
255 const MCSymbol *Base,
256 int64_t Offset) {
257 MCContext &Context = streamer.getContext();
258 const MCConstantExpr *OffExpr = MCConstantExpr::create(Value: Offset, Ctx&: Context);
259 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Symbol: Base,
260 specifier: MCSymbolRefExpr::VK_COFF_IMGREL32,
261 Ctx&: Context);
262 streamer.emitValue(Value: MCBinaryExpr::createAdd(LHS: BaseRefRel, RHS: OffExpr, Ctx&: Context), Size: 4);
263}
264
265static void EmitSymbolRefWithOfs(MCStreamer &streamer,
266 const MCSymbol *Base,
267 const MCSymbol *Other) {
268 MCContext &Context = streamer.getContext();
269 const MCSymbolRefExpr *BaseRef = MCSymbolRefExpr::create(Symbol: Base, Ctx&: Context);
270 const MCSymbolRefExpr *OtherRef = MCSymbolRefExpr::create(Symbol: Other, Ctx&: Context);
271 const MCExpr *Ofs = MCBinaryExpr::createSub(LHS: OtherRef, RHS: BaseRef, Ctx&: Context);
272 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Symbol: Base,
273 specifier: MCSymbolRefExpr::VK_COFF_IMGREL32,
274 Ctx&: Context);
275 streamer.emitValue(Value: MCBinaryExpr::createAdd(LHS: BaseRefRel, RHS: Ofs, Ctx&: Context), Size: 4);
276}
277
278static void EmitRuntimeFunction(MCStreamer &streamer,
279 const WinEH::FrameInfo *info) {
280 MCContext &context = streamer.getContext();
281
282 streamer.emitValueToAlignment(Alignment: Align(4));
283 EmitSymbolRefWithOfs(streamer, Base: info->Begin, Other: info->Begin);
284 EmitSymbolRefWithOfs(streamer, Base: info->Begin, Other: info->End);
285 streamer.emitValue(Value: MCSymbolRefExpr::create(Symbol: info->Symbol,
286 specifier: MCSymbolRefExpr::VK_COFF_IMGREL32,
287 Ctx&: context), Size: 4);
288}
289
290static std::optional<int64_t>
291GetOptionalAbsDifference(const MCAssembler &Assembler, const MCSymbol *LHS,
292 const MCSymbol *RHS) {
293 MCContext &Context = Assembler.getContext();
294 const MCExpr *Diff =
295 MCBinaryExpr::createSub(LHS: MCSymbolRefExpr::create(Symbol: LHS, Ctx&: Context),
296 RHS: MCSymbolRefExpr::create(Symbol: RHS, Ctx&: Context), Ctx&: Context);
297 // It should normally be possible to calculate the length of a function
298 // at this point, but it might not be possible in the presence of certain
299 // unusual constructs, like an inline asm with an alignment directive.
300 int64_t value;
301 if (!Diff->evaluateAsAbsolute(Res&: value, Asm: Assembler))
302 return std::nullopt;
303 return value;
304}
305
306//===----------------------------------------------------------------------===//
307// V3 UNWIND_INFO Emission
308// See https://learn.microsoft.com/en-us/cpp/build/x64-unwind-information-v3
309//===----------------------------------------------------------------------===//
310
311/// Encode a single WinEH::Instruction as V3 WOD bytes.
312/// Appends encoded bytes to Out.
313void Win64EH::EncodeWOD(const WinEH::Instruction &Inst,
314 SmallVectorImpl<uint8_t> &Out) {
315 switch (static_cast<Win64EH::UnwindOpcodes>(Inst.Operation)) {
316 case Win64EH::UOP_PushNonVol: {
317 // WOD_PUSH: 1 byte, bits [2:0] = 4, bits [7:3] = register (5-bit)
318 uint8_t Reg = Inst.Register & 0x1F;
319 Out.push_back(Elt: (Reg << 3) | Win64EH::WOD_PUSH);
320 break;
321 }
322 case Win64EH::UOP_AllocSmall: {
323 // WOD_ALLOC_SMALL: 1 byte, bits [3:0] = 8, bits [7:4] = (size/8 - 1)
324 // V1/V2 stores (size-8)/8 in OpInfo; actual size = Offset.
325 // Inst.Offset is the raw allocation size.
326 if (Inst.Offset < 8 || Inst.Offset > 128 || Inst.Offset % 8 != 0)
327 reportFatalInternalError(
328 reason: "UOP_AllocSmall outside expected range or alignment");
329 uint8_t Encoded = ((Inst.Offset / 8 - 1) & 0x0F);
330 Out.push_back(Elt: (Encoded << 4) | Win64EH::WOD_ALLOC_SMALL);
331 break;
332 }
333 case Win64EH::UOP_AllocLarge: {
334 if (Inst.Offset > 512 * 1024 - 8) {
335 // WOD_ALLOC_HUGE: 5 bytes, byte[0] = 1, bytes[1:4] = LE32(size)
336 Out.push_back(Elt: Win64EH::WOD_ALLOC_HUGE);
337 uint32_t Size = Inst.Offset;
338 Out.push_back(Elt: Size & 0xFF);
339 Out.push_back(Elt: (Size >> 8) & 0xFF);
340 Out.push_back(Elt: (Size >> 16) & 0xFF);
341 Out.push_back(Elt: (Size >> 24) & 0xFF);
342 } else {
343 // WOD_ALLOC_LARGE: 3 bytes, byte[0] = 2, bytes[1:2] = LE16(size/8)
344 Out.push_back(Elt: Win64EH::WOD_ALLOC_LARGE);
345 uint16_t Scaled = Inst.Offset / 8;
346 Out.push_back(Elt: Scaled & 0xFF);
347 Out.push_back(Elt: (Scaled >> 8) & 0xFF);
348 }
349 break;
350 }
351 case Win64EH::UOP_SetFPReg: {
352 // WOD_SET_FPREG: 2 bytes, byte[0] = 0, byte[1] = reg(4) | (offset/16)(4)
353 // The frame register field is only 4 bits, so EGPR (R16-R31) cannot be
354 // used as the frame pointer in V3 unwind info.
355 if (Inst.Register > 0x0F)
356 reportFatalInternalError(
357 reason: "SET_FPREG frame register does not fit in 4 bits");
358 Out.push_back(Elt: Win64EH::WOD_SET_FPREG);
359 uint8_t Reg = Inst.Register & 0x0F;
360 uint8_t Off = (Inst.Offset / 16) & 0x0F;
361 Out.push_back(Elt: Reg | (Off << 4));
362 break;
363 }
364 case Win64EH::UOP_SaveNonVol: {
365 // WOD_SAVE_NONVOL: 3 bytes, bits [2:0] = 6, bits [7:3] = reg,
366 // bytes[1:2] = LE16(displacement/8)
367 uint8_t Reg = Inst.Register & 0x1F;
368 Out.push_back(Elt: (Reg << 3) | Win64EH::WOD_SAVE_NONVOL);
369 uint16_t Disp = Inst.Offset / 8;
370 Out.push_back(Elt: Disp & 0xFF);
371 Out.push_back(Elt: (Disp >> 8) & 0xFF);
372 break;
373 }
374 case Win64EH::UOP_SaveNonVolBig: {
375 // WOD_SAVE_NONVOL_FAR: 5 bytes, bits [2:0] = 5, bits [7:3] = reg,
376 // bytes[1:4] = LE32(displacement)
377 uint8_t Reg = Inst.Register & 0x1F;
378 Out.push_back(Elt: (Reg << 3) | Win64EH::WOD_SAVE_NONVOL_FAR);
379 uint32_t Disp = Inst.Offset;
380 Out.push_back(Elt: Disp & 0xFF);
381 Out.push_back(Elt: (Disp >> 8) & 0xFF);
382 Out.push_back(Elt: (Disp >> 16) & 0xFF);
383 Out.push_back(Elt: (Disp >> 24) & 0xFF);
384 break;
385 }
386 case Win64EH::UOP_SaveXMM128: {
387 // WOD_SAVE_XMM128: 3 bytes, bits [3:0] = 10, bits [7:4] = reg,
388 // bytes[1:2] = LE16(displacement/16)
389 // The XMM register field is only 4 bits, so XMM16-XMM31 (AVX-512 EVEX)
390 // cannot be encoded. Such registers are caller-saved on Win64 and should
391 // never reach here from codegen.
392 if (Inst.Register > 0x0F)
393 reportFatalInternalError(
394 reason: "SAVE_XMM128 register does not fit in 4 bits (XMM16-31 unsupported)");
395 uint8_t Reg = Inst.Register & 0x0F;
396 Out.push_back(Elt: (Reg << 4) | Win64EH::WOD_SAVE_XMM128);
397 uint16_t Disp = Inst.Offset / 16;
398 Out.push_back(Elt: Disp & 0xFF);
399 Out.push_back(Elt: (Disp >> 8) & 0xFF);
400 break;
401 }
402 case Win64EH::UOP_SaveXMM128Big: {
403 // WOD_SAVE_XMM128_FAR: 5 bytes, bits [3:0] = 9, bits [7:4] = reg,
404 // bytes[1:4] = LE32(displacement)
405 if (Inst.Register > 0x0F)
406 reportFatalInternalError(reason: "SAVE_XMM128_FAR register does not fit in 4 "
407 "bits (XMM16-31 unsupported)");
408 uint8_t Reg = Inst.Register & 0x0F;
409 Out.push_back(Elt: (Reg << 4) | Win64EH::WOD_SAVE_XMM128_FAR);
410 uint32_t Disp = Inst.Offset;
411 Out.push_back(Elt: Disp & 0xFF);
412 Out.push_back(Elt: (Disp >> 8) & 0xFF);
413 Out.push_back(Elt: (Disp >> 16) & 0xFF);
414 Out.push_back(Elt: (Disp >> 24) & 0xFF);
415 break;
416 }
417 case Win64EH::UOP_PushMachFrame: {
418 // WOD_PUSH_CANONICAL_FRAME: 2 bytes, byte[0] = 3, byte[1] = type
419 Out.push_back(Elt: Win64EH::WOD_PUSH_CANONICAL_FRAME);
420 Out.push_back(Elt: Inst.Offset == 1 ? 1 : 0);
421 break;
422 }
423 case Win64EH::UOP_Push2: {
424 uint8_t Reg1 = Inst.Register & 0x1F;
425 uint8_t Reg2 = Inst.Register2 & 0x1F;
426 // Optimization: if registers are consecutive, use WOD_PUSH_CONSECUTIVE_2
427 // (opcode 7, 1 byte) instead of WOD_PUSH2 (opcode 32, 2 bytes).
428 if (Reg2 == Reg1 + 1) {
429 // WOD_PUSH_CONSECUTIVE_2: bits [2:0] = 111b, bits [7:3] = Register
430 Out.push_back(Elt: (Reg1 << 3) | Win64EH::WOD_PUSH_CONSECUTIVE_2);
431 } else {
432 // WOD_PUSH2: 2 bytes
433 // Byte 0: [5:0] = 100000b (opcode 32), [7:6] = Register1[1:0]
434 // Byte 1: [2:0] = Register1[4:2], [7:3] = Register2
435 Out.push_back(Elt: ((Reg1 & 0x03) << 6) | Win64EH::WOD_PUSH2);
436 Out.push_back(Elt: ((Reg2 & 0x1F) << 3) | ((Reg1 >> 2) & 0x07));
437 }
438 break;
439 }
440 default:
441 llvm_unreachable("Unsupported unwind operation for V3 encoding");
442 }
443}
444
445/// Try to find Needle as a contiguous subsequence within Haystack.
446/// Returns the byte offset if found, or std::nullopt.
447static std::optional<uint16_t> FindInPool(ArrayRef<uint8_t> Haystack,
448 ArrayRef<uint8_t> Needle) {
449 assert(!Needle.empty() && "FindInPool called with empty Needle");
450 auto It = std::search(first1: Haystack.begin(), last1: Haystack.end(), first2: Needle.begin(),
451 last2: Needle.end());
452 if (It == Haystack.end())
453 return std::nullopt;
454 return static_cast<uint16_t>(std::distance(first: Haystack.begin(), last: It));
455}
456
457/// Compare the relative IP offset arrays of two epilogs.
458static bool EpilogIpOffsetsMatch(const WinEH::FrameInfo::Epilog &A,
459 const WinEH::FrameInfo::Epilog &B,
460 const MCAssembler &Asm) {
461 if (A.Instructions.size() != B.Instructions.size())
462 return false;
463 for (unsigned I = 0; I < A.Instructions.size(); ++I) {
464 auto OffA = GetOptionalAbsDifference(Assembler: Asm, LHS: A.Instructions[I].Label, RHS: A.Start);
465 auto OffB = GetOptionalAbsDifference(Assembler: Asm, LHS: B.Instructions[I].Label, RHS: B.Start);
466 if (!OffA || !OffB || *OffA != *OffB)
467 return false;
468 }
469 return true;
470}
471
472/// Emit V3 UNWIND_INFO for a single frame.
473static void EmitUnwindInfoV3(MCStreamer &Streamer, WinEH::FrameInfo *Info) {
474 // Should have been checked by our caller.
475 assert(!Info->Symbol && "UNWIND_INFO already has a symbol");
476
477 MCContext &Context = Streamer.getContext();
478 MCObjectStreamer *OS = static_cast<MCObjectStreamer *>(&Streamer);
479 const MCAssembler &Asm = OS->getAssembler();
480
481 MCSymbol *Label = Context.createTempSymbol();
482 Streamer.emitValueToAlignment(Alignment: Align(4));
483 Streamer.emitLabel(Symbol: Label);
484 Info->Symbol = Label;
485
486 // ===================================================================
487 // Phase 1: Data preparation — compute all metadata before emitting.
488 // ===================================================================
489
490 // --- Build prolog WOD pool (body-to-entry order) ---
491 SmallVector<uint8_t, 64> WODPool;
492 for (auto It = Info->Instructions.rbegin(); It != Info->Instructions.rend();
493 ++It)
494 Win64EH::EncodeWOD(Inst: *It, Out&: WODPool);
495
496 // --- Build prolog IP offset label pairs (body-to-entry order) ---
497 SmallVector<std::pair<const MCSymbol *, const MCSymbol *>, 16> PrologIpLabels;
498 unsigned PrologOpCount = Info->Instructions.size();
499 if (PrologOpCount > 31) {
500 reportFatalUsageError(
501 reason: "Too many prolog unwind codes for V3 encoding. Maximum "
502 "is 31. This function has " +
503 Twine(PrologOpCount));
504 }
505 for (auto It = Info->Instructions.rbegin(); It != Info->Instructions.rend();
506 ++It)
507 PrologIpLabels.push_back(Elt: {It->Label, Info->Begin});
508
509 // --- Determine if UNW_FLAG_LARGE is needed for the prolog ---
510 // Conservative: if we KNOW a value exceeds 255 or can't measure, use LARGE.
511 // Reject evaluatable values that are negative or exceed the 16-bit unsigned
512 // range supported by LARGE, since those would be silently truncated.
513 bool NeedsLargeProlog = false;
514 if (Info->PrologEnd) {
515 auto MaybePrologSize =
516 GetOptionalAbsDifference(Assembler: Asm, LHS: Info->PrologEnd, RHS: Info->Begin);
517 if (MaybePrologSize) {
518 if (*MaybePrologSize < 0)
519 reportFatalUsageError(reason: "Negative SizeOfProlog in V3 unwind info");
520 if (*MaybePrologSize > UINT16_MAX)
521 reportFatalUsageError(
522 reason: "SizeOfProlog exceeds 16-bit range for V3 unwind info");
523 NeedsLargeProlog = (*MaybePrologSize > 255);
524 } else {
525 NeedsLargeProlog = true; // Can't measure -> be conservative
526 }
527 }
528 for (auto &[InstLabel, BeginLabel] : PrologIpLabels) {
529 if (NeedsLargeProlog)
530 break;
531 auto MaybeOffset = GetOptionalAbsDifference(Assembler: Asm, LHS: InstLabel, RHS: BeginLabel);
532 if (MaybeOffset) {
533 if (*MaybeOffset < 0)
534 reportFatalUsageError(reason: "Negative prolog IP offset in V3 unwind info");
535 if (*MaybeOffset > UINT16_MAX)
536 reportFatalUsageError(
537 reason: "Prolog IP offset exceeds 16-bit range for V3 unwind info");
538 NeedsLargeProlog = (*MaybeOffset > 255);
539 } else {
540 NeedsLargeProlog = true; // Can't measure -> be conservative
541 }
542 }
543
544 // --- Per-epilog data preparation ---
545 struct EpilogEmitInfo {
546 const WinEH::FrameInfo::Epilog *Epilog;
547 SmallVector<uint8_t, 32> WODBytes;
548 uint16_t FirstOp;
549 uint8_t NumberOfOps;
550 bool Inherited;
551 bool NeedsLarge; // EPILOG_INFO_LARGE needed for this epilog
552 };
553
554 SmallVector<EpilogEmitInfo, 8> EpilogInfos;
555 for (const auto &[EpilogSym, Epilog] : Info->EpilogMap) {
556 if (Epilog.Instructions.empty())
557 continue;
558
559 EpilogEmitInfo EI;
560 EI.Epilog = &Epilog;
561 EI.NumberOfOps = Epilog.Instructions.size();
562 if (EI.NumberOfOps > 31)
563 reportFatalUsageError(
564 reason: "Too many epilog unwind codes for V3 encoding. Maximum "
565 "is 31. This epilog has " +
566 Twine(EI.NumberOfOps));
567 EI.Inherited = false;
568 EI.NeedsLarge = false;
569
570 // Determine if EPILOG_INFO_LARGE is needed.
571 // Check IpOffsetOfLastInstruction (EpilogEnd - EpilogStart).
572 // Reject negative or out-of-range evaluatable values.
573 auto MaybeLastInstOfs =
574 GetOptionalAbsDifference(Assembler: Asm, LHS: Epilog.End, RHS: Epilog.Start);
575 if (MaybeLastInstOfs) {
576 if (*MaybeLastInstOfs < 0)
577 reportFatalUsageError(
578 reason: "Negative IpOffsetOfLastInstruction in V3 unwind info");
579 if (*MaybeLastInstOfs > UINT16_MAX)
580 reportFatalUsageError(
581 reason: "IpOffsetOfLastInstruction exceeds 16-bit range for "
582 "V3 unwind info");
583 EI.NeedsLarge = (*MaybeLastInstOfs > 255);
584 } else {
585 EI.NeedsLarge = true; // Can't measure -> be conservative
586 }
587
588 // Check each epilog IP offset.
589 for (const auto &EpiInst : Epilog.Instructions) {
590 if (EI.NeedsLarge)
591 break;
592 auto MaybeOffset =
593 GetOptionalAbsDifference(Assembler: Asm, LHS: EpiInst.Label, RHS: Epilog.Start);
594 if (MaybeOffset) {
595 if (*MaybeOffset < 0)
596 reportFatalUsageError(reason: "Negative epilog IP offset in V3 unwind info");
597 if (*MaybeOffset > UINT16_MAX)
598 reportFatalUsageError(
599 reason: "Epilog IP offset exceeds 16-bit range for V3 unwind info");
600 EI.NeedsLarge = (*MaybeOffset > 255);
601 } else {
602 EI.NeedsLarge = true; // Can't measure -> be conservative
603 }
604 }
605
606 // Encode this epilog's WODs (forward order: body-to-terminator).
607 for (const auto &Inst : Epilog.Instructions)
608 Win64EH::EncodeWOD(Inst, Out&: EI.WODBytes);
609
610 // Pool sharing: try to re-use existing bytes in the pool rather than
611 // appending.
612 if (auto Offset = FindInPool(Haystack: WODPool, Needle: EI.WODBytes)) {
613 EI.FirstOp = *Offset;
614 } else {
615 EI.FirstOp = WODPool.size();
616 WODPool.append(in_start: EI.WODBytes.begin(), in_end: EI.WODBytes.end());
617 }
618
619 EpilogInfos.push_back(Elt: std::move(EI));
620 }
621 if (EpilogInfos.size() > 7)
622 reportFatalUsageError(reason: "Too many epilogs for V3 encoding. Maximum is 7."
623 " This function has " +
624 Twine(EpilogInfos.size()));
625
626 // V3 epilog offsets are always encoded tail-relative: the first descriptor
627 // holds a negative byte offset from the fragment end, and each subsequent
628 // descriptor a (negative) delta from the previous epilog's start. The spec
629 // requires every descriptor in a fragment to use the same sign, so the
630 // descriptors must be listed in descending address order ("descending from
631 // tail"). EpilogMap is in ascending address order, so reverse it here before
632 // computing inheritance and emitting descriptors.
633 std::reverse(first: EpilogInfos.begin(), last: EpilogInfos.end());
634
635 // --- Inheritance decisions ---
636 // Per the V3 spec, an epilog descriptor with NumberOfOps == 0 inherits its
637 // effective NumberOfOps, FirstOp, IpOffsetOfLastInstruction, and IP offset
638 // array from the first preceding descriptor with NumberOfOps != 0 (the
639 // "base"), not necessarily the immediately preceding descriptor. An epilog
640 // can therefore use the inherited (3-byte) descriptor when FirstOp,
641 // NumberOfOps, and relative IP offsets all match that base. NeedsLarge is a
642 // deterministic function of those fields (it is set when an IP offset exceeds
643 // the 1-byte range), so a matching FirstOp/NumberOfOps and IP offsets imply a
644 // matching NeedsLarge; we assert that invariant rather than test it.
645 for (unsigned I = 1, Base = 0; I < EpilogInfos.size(); ++I) {
646 auto &BaseEI = EpilogInfos[Base];
647 auto &Curr = EpilogInfos[I];
648 if (Curr.FirstOp == BaseEI.FirstOp &&
649 Curr.NumberOfOps == BaseEI.NumberOfOps &&
650 EpilogIpOffsetsMatch(A: *Curr.Epilog, B: *BaseEI.Epilog,
651 Asm: OS->getAssembler())) {
652 assert(Curr.NeedsLarge == BaseEI.NeedsLarge &&
653 "NeedsLarge must follow from matching FirstOp, NumberOfOps, and "
654 "IP offsets");
655 Curr.Inherited = true;
656 } else
657 Base = I; // Curr is a full descriptor; it becomes the new base.
658 }
659
660 // --- Compute payload sizes ---
661 unsigned PrologIpEntrySize = NeedsLargeProlog ? 2 : 1;
662 unsigned EpilogDescBytes = 0;
663 for (const auto &EI : EpilogInfos) {
664 if (EI.Inherited) {
665 EpilogDescBytes += 3;
666 } else if (EI.NeedsLarge) {
667 // EPILOG_INFO_V3 (3) + EPILOG_INFO_LARGE_EX_V3 (4) + IP offsets (N*2)
668 EpilogDescBytes += 7 + EI.NumberOfOps * 2;
669 } else {
670 // EPILOG_INFO_V3 (3) + EPILOG_INFO_EX_V3 (3) + IP offsets (N*1)
671 EpilogDescBytes += 6 + EI.NumberOfOps;
672 }
673 }
674
675 unsigned PrologIpBytes = PrologOpCount * PrologIpEntrySize;
676 unsigned WODPoolBytes = WODPool.size();
677 // When UNW_FLAG_LARGE is set, the SizeOfPrologHighByte sits at the start
678 // of the payload (immediately after the 4-byte fixed header) and is
679 // counted in PayloadWords. See decodeUnwindInfoV3 / the V3 spec:
680 // handler_offset = ALIGN_UP(sizeof(UNWIND_INFO_V3) + PayloadWords * 2, 4)
681 unsigned LargeHeaderBytes = NeedsLargeProlog ? 1 : 0;
682 unsigned TotalPayloadBytes =
683 LargeHeaderBytes + PrologIpBytes + EpilogDescBytes + WODPoolBytes;
684 if (TotalPayloadBytes > 255 * 2) {
685 reportFatalUsageError(reason: "Too much unwind info for V3 encoding. Maximum is "
686 "510 bytes. This function has " +
687 Twine(TotalPayloadBytes));
688 }
689 uint8_t PayloadWords = (TotalPayloadBytes + 1) / 2;
690
691 // ===================================================================
692 // Phase 2: Emission — emit header, payload, and trailer.
693 // ===================================================================
694
695 // --- Emit header (4 bytes, or 5 when UNW_FLAG_LARGE) ---
696 uint8_t Flags = 0;
697 if (Info->ChainedParent)
698 Flags |= Win64EH::UNW_ChainInfo;
699 else {
700 if (Info->HandlesUnwind)
701 Flags |= Win64EH::UNW_TerminateHandler;
702 if (Info->HandlesExceptions)
703 Flags |= Win64EH::UNW_ExceptionHandler;
704 }
705 if (NeedsLargeProlog)
706 Flags |= Win64EH::UNW_FlagLarge;
707
708 // Byte 0: (Flags << 3) | Version(3)
709 Streamer.emitInt8(Value: (Flags << 3) | 3);
710
711 // Byte 1: SizeOfProlog (low byte, or full 8-bit value when not LARGE)
712 if (Info->PrologEnd) {
713 if (NeedsLargeProlog) {
714 // Emit low byte as a fixup; we'll emit the high byte after Byte 3.
715 // Use a 2-byte value at a temp symbol and extract bytes, OR just emit
716 // the known value if evaluable.
717 auto MaybePrologSize =
718 GetOptionalAbsDifference(Assembler: Asm, LHS: Info->PrologEnd, RHS: Info->Begin);
719 if (MaybePrologSize) {
720 Streamer.emitInt8(Value: *MaybePrologSize & 0xFF);
721 } else {
722 // Emit as 1-byte fixup for the low byte.
723 EmitAbsDifference(Streamer, LHS: Info->PrologEnd, RHS: Info->Begin);
724 }
725 } else {
726 EmitAbsDifference(Streamer, LHS: Info->PrologEnd, RHS: Info->Begin);
727 }
728 } else {
729 Streamer.emitInt8(Value: 0);
730 }
731
732 // Byte 2: PayloadWords
733 Streamer.emitInt8(Value: PayloadWords);
734
735 // Byte 3: (NumberOfEpilogs << 5) | NumberOfPrologOps
736 uint8_t NumberOfEpilogs = EpilogInfos.size();
737 Streamer.emitInt8(Value: (NumberOfEpilogs << 5) | (PrologOpCount & 0x1F));
738
739 // Byte 4 (LARGE only): SizeOfPrologHighByte
740 if (NeedsLargeProlog) {
741 if (Info->PrologEnd) {
742 auto MaybePrologSize =
743 GetOptionalAbsDifference(Assembler: Asm, LHS: Info->PrologEnd, RHS: Info->Begin);
744 if (MaybePrologSize) {
745 Streamer.emitInt8(Value: (*MaybePrologSize >> 8) & 0xFF);
746 } else {
747 // Can't evaluate at this point — emit a fixup that shifts the
748 // difference right by 8 to extract the high byte.
749 const MCExpr *Diff = MCBinaryExpr::createSub(
750 LHS: MCSymbolRefExpr::create(Symbol: Info->PrologEnd, Ctx&: Context),
751 RHS: MCSymbolRefExpr::create(Symbol: Info->Begin, Ctx&: Context), Ctx&: Context);
752 const MCExpr *HighByte = MCBinaryExpr::createLShr(
753 LHS: Diff, RHS: MCConstantExpr::create(Value: 8, Ctx&: Context), Ctx&: Context);
754 Streamer.emitValue(Value: HighByte, Size: 1);
755 }
756 } else {
757 Streamer.emitInt8(Value: 0);
758 }
759 }
760
761 // --- Emit prolog IP offsets (8-bit or 16-bit) ---
762 for (auto &[InstLabel, BeginLabel] : PrologIpLabels) {
763 if (NeedsLargeProlog)
764 EmitAbsDifference16(Streamer, LHS: InstLabel, RHS: BeginLabel);
765 else
766 EmitAbsDifference(Streamer, LHS: InstLabel, RHS: BeginLabel);
767 }
768
769 // --- Emit epilog descriptors ---
770 const MCSymbol *PrevEpilogStart = nullptr;
771 [[maybe_unused]] uint8_t BaseEpiFlags = 0;
772 for (const auto &EI : EpilogInfos) {
773 const auto &Epilog = *EI.Epilog;
774
775 // FlagsAndNumOps: bits [2:0] = flags, bits [7:3] = NumberOfOps.
776 // For inherited descriptors, NumberOfOps = 0.
777 //
778 // Per the V3 spec, Flags bits 0 and 1 are NOT inherited by a descriptor
779 // with NumberOfOps == 0; the producer must replicate them so they have the
780 // same value as the base descriptor's. Since inheritance requires a
781 // matching NeedsLarge value, EI.NeedsLarge already equals the base's, so we
782 // emit EPILOG_INFO_LARGE for inherited descriptors too.
783 uint8_t EpiFlags = 0;
784 if (EI.NeedsLarge)
785 EpiFlags |= Win64EH::EPILOG_INFO_LARGE;
786 // An inherited descriptor must replicate the base descriptor's flags bits
787 // 0 and 1; verify we are about to emit a byte consistent with the base.
788 assert((!EI.Inherited || (EpiFlags & 0x03) == (BaseEpiFlags & 0x03)) &&
789 "inherited epilog must replicate base descriptor's flags bits 0-1");
790 if (!EI.Inherited)
791 BaseEpiFlags = EpiFlags;
792 uint8_t EpiNumOps = EI.Inherited ? 0 : EI.NumberOfOps;
793 Streamer.emitInt8(Value: (EpiNumOps << 3) | EpiFlags);
794
795 // EpilogOffset: signed 16-bit, always tail-relative (see
796 // MCUnwindV3EpilogOffsetTargetExpr). The first descriptor holds the
797 // negative byte offset from the fragment end; each subsequent descriptor
798 // holds the delta from the previous epilog's start. Emitted in descending
799 // address order, every offset is non-positive, satisfying the V3
800 // "all epilogs use the same sign" rule.
801 {
802 const MCExpr *EpilogOffsetExpr = MCUnwindV3EpilogOffsetTargetExpr::create(
803 FrameInfo: *Info, EpilogStart: Epilog.Start, PrevEpilogStart, Loc: Epilog.Loc, Ctx&: Context);
804 OS->ensureHeadroom(Headroom: 2);
805 OS->addFixup(Value: EpilogOffsetExpr, Kind: FK_Data_2);
806 OS->appendContents(Num: 2, Elt: 0);
807 }
808 PrevEpilogStart = Epilog.Start;
809
810 // Full descriptor fields (only for non-inherited epilogs).
811 if (!EI.Inherited) {
812 // FirstOp: byte offset into WOD pool (2 bytes LE).
813 Streamer.emitInt8(Value: EI.FirstOp & 0xFF);
814 Streamer.emitInt8(Value: (EI.FirstOp >> 8) & 0xFF);
815
816 // IpOffsetOfLastInstruction: 8-bit or 16-bit depending on
817 // EPILOG_INFO_LARGE.
818 {
819 const MCExpr *LastInstOffsetExpr = MCBinaryExpr::createSub(
820 LHS: MCSymbolRefExpr::create(Symbol: Epilog.End, Ctx&: Context),
821 RHS: MCSymbolRefExpr::create(Symbol: Epilog.Start, Ctx&: Context), Ctx&: Context);
822 unsigned FixupSize = EI.NeedsLarge ? 2 : 1;
823 OS->ensureHeadroom(Headroom: FixupSize);
824 OS->addFixup(Value: LastInstOffsetExpr, Kind: EI.NeedsLarge ? FK_Data_2 : FK_Data_1);
825 OS->appendContents(Num: FixupSize, Elt: 0);
826 }
827
828 // Epilog IP offsets (forward order: body-to-terminator).
829 for (const auto &EpiInst : Epilog.Instructions) {
830 if (EI.NeedsLarge)
831 EmitAbsDifference16(Streamer, LHS: EpiInst.Label, RHS: Epilog.Start);
832 else
833 EmitAbsDifference(Streamer, LHS: EpiInst.Label, RHS: Epilog.Start);
834 }
835 }
836 }
837
838 // --- Emit WOD pool ---
839 for (uint8_t B : WODPool)
840 Streamer.emitInt8(Value: B);
841
842 // --- Pad to PayloadWords * 2 bytes ---
843 // PayloadWords = (TotalPayloadBytes + 1) / 2, so at most 1 byte of padding.
844 if (TotalPayloadBytes % 2 != 0)
845 Streamer.emitInt8(Value: 0);
846
847 // --- Pad to 4-byte boundary before handler/chain info ---
848 // Per the V3 spec, the handler RVA / chained RUNTIME_FUNCTION begins at
849 // handler_offset = ALIGN_UP(sizeof(UNWIND_INFO_V3) + PayloadWords * 2, 4)
850 // The unwind info structure itself is 4-byte aligned, so when PayloadWords
851 // is odd, the natural end of the payload sits at +2 mod 4 and requires 2
852 // additional zero bytes of padding before the handler/chain.
853 if (PayloadWords % 2 != 0)
854 Streamer.emitInt16(Value: 0);
855
856 // --- Emit handler/chained info (same position as V1/V2) ---
857 if (Flags & Win64EH::UNW_ChainInfo)
858 EmitRuntimeFunction(streamer&: Streamer, info: Info->ChainedParent);
859 else if (Flags &
860 (Win64EH::UNW_TerminateHandler | Win64EH::UNW_ExceptionHandler))
861 Streamer.emitValue(
862 Value: MCSymbolRefExpr::create(Symbol: Info->ExceptionHandler,
863 specifier: MCSymbolRefExpr::VK_COFF_IMGREL32, Ctx&: Context),
864 Size: 4);
865 else if (PayloadWords == 0) {
866 // Minimum size: pad to 8 bytes total.
867 Streamer.emitInt32(Value: 0);
868 }
869}
870
871static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {
872 // If this UNWIND_INFO already has a symbol, it's already been emitted.
873 if (info->Symbol)
874 return;
875
876 // V3 has a completely different binary layout; dispatch to separate emitter.
877 if (info->Version == 3) {
878 EmitUnwindInfoV3(Streamer&: streamer, Info: info);
879 return;
880 }
881
882 // UOP_Push2 is V3-only and cannot be encoded in V1/V2. Detect this early
883 // (before counting codes) so the error is reported cleanly. This is
884 // reachable from hand-written .s if `.seh_push2regs` is followed by a
885 // per-function `.seh_unwindversion 1` or `2` downgrade.
886 for (const auto &Inst : info->Instructions) {
887 if (Inst.Operation == Win64EH::UOP_Push2) {
888 streamer.getContext().reportError(
889 L: SMLoc(), Msg: "UOP_Push2 (PUSH2 with two registers) requires V3 unwind "
890 "info. Use `.seh_unwindversion 3`.");
891 // Mark the frame as emitted (with no UNWIND_INFO) and bail so we don't
892 // emit malformed bytes or hit a downstream assertion.
893 info->Symbol = streamer.getContext().createTempSymbol();
894 return;
895 }
896 }
897
898 MCContext &context = streamer.getContext();
899 MCObjectStreamer *OS = (MCObjectStreamer *)(&streamer);
900 MCSymbol *Label = context.createTempSymbol();
901
902 streamer.emitValueToAlignment(Alignment: Align(4));
903 streamer.emitLabel(Symbol: Label);
904 info->Symbol = Label;
905
906 uint8_t numCodes = CountOfUnwindCodes(Insns&: info->Instructions);
907 bool LastEpilogIsAtEnd = false;
908 bool AddPaddingEpilogCode = false;
909 uint8_t EpilogSize = 0;
910 bool EnableUnwindV2 = (info->Version >= 2) && !info->EpilogMap.empty();
911 if (EnableUnwindV2) {
912 auto &LastEpilog = info->EpilogMap.back().second;
913
914 // Calculate the size of the epilogs. Note that we +1 to the size so that
915 // the terminator instruction is also included in the epilog (the Windows
916 // unwinder does a simple range check versus the current instruction pointer
917 // so, although there are terminators that are large than 1 byte, the
918 // starting address of the terminator instruction will always be considered
919 // inside the epilog).
920 assert(
921 LastEpilog.UnwindV2Start &&
922 "If unwind v2 is enabled, epilog must have a unwind v2 start marker");
923 assert(LastEpilog.End && "Epilog must have an end");
924 auto MaybeSize = GetOptionalAbsDifference(
925 Assembler: OS->getAssembler(), LHS: LastEpilog.End, RHS: LastEpilog.UnwindV2Start);
926 if (!MaybeSize) {
927 context.reportError(L: LastEpilog.Loc,
928 Msg: "Failed to evaluate epilog size for Unwind v2 in " +
929 info->Function->getName());
930 return;
931 }
932 assert(*MaybeSize >= 0);
933 if (*MaybeSize >= (int64_t)UINT8_MAX) {
934 context.reportError(L: LastEpilog.Loc,
935 Msg: "Epilog size is too large for Unwind v2 in " +
936 info->Function->getName());
937 return;
938 }
939 EpilogSize = *MaybeSize + 1;
940
941 // If the last epilog is at the end of the function, we can use a special
942 // encoding for it. Because of our +1 trick for the size, this will only
943 // work where that final terminator instruction is 1 byte long.
944 // NOTE: At the point where the unwind info is emitted, the function may not
945 // have ended yet (e.g., if there is EH Handler Data), so assume that we
946 // aren't at the end (since we can't calculate it).
947 if (info->End) {
948 auto LastEpilogToFuncEnd = GetOptionalAbsDifference(
949 Assembler: OS->getAssembler(), LHS: info->End, RHS: LastEpilog.UnwindV2Start);
950 LastEpilogIsAtEnd = (LastEpilogToFuncEnd == EpilogSize);
951 }
952
953 // If we have an odd number of epilog codes, we need to add a padding code.
954 size_t numEpilogCodes =
955 info->EpilogMap.size() + (LastEpilogIsAtEnd ? 0 : 1);
956 if ((numEpilogCodes % 2) != 0) {
957 AddPaddingEpilogCode = true;
958 numEpilogCodes++;
959 }
960
961 // Too many epilogs to handle.
962 if ((size_t)numCodes + numEpilogCodes > UINT8_MAX) {
963 context.reportError(L: info->FunctionLoc,
964 Msg: "Too many unwind codes with Unwind v2 enabled in " +
965 info->Function->getName());
966 return;
967 }
968
969 numCodes += numEpilogCodes;
970 }
971
972 // Upper 3 bits are the version number.
973 uint8_t flags = info->Version;
974 if (info->ChainedParent)
975 flags |= Win64EH::UNW_ChainInfo << 3;
976 else {
977 if (info->HandlesUnwind)
978 flags |= Win64EH::UNW_TerminateHandler << 3;
979 if (info->HandlesExceptions)
980 flags |= Win64EH::UNW_ExceptionHandler << 3;
981 }
982 streamer.emitInt8(Value: flags);
983
984 if (info->PrologEnd)
985 EmitAbsDifference(Streamer&: streamer, LHS: info->PrologEnd, RHS: info->Begin);
986 else
987 streamer.emitInt8(Value: 0);
988
989 streamer.emitInt8(Value: numCodes);
990
991 uint8_t frame = 0;
992 if (info->LastFrameInst >= 0) {
993 WinEH::Instruction &frameInst = info->Instructions[info->LastFrameInst];
994 assert(frameInst.Operation == Win64EH::UOP_SetFPReg);
995 frame = (frameInst.Register & 0x0F) | (frameInst.Offset & 0xF0);
996 }
997 streamer.emitInt8(Value: frame);
998
999 // Emit the epilog instructions.
1000 if (EnableUnwindV2) {
1001 // Ensure the fixups and appended content apply to the same fragment.
1002 // size byte + flags byte + 2 per epilog (for the distance).
1003 OS->ensureHeadroom(Headroom: 2 + info->EpilogMap.size() * 2);
1004
1005 bool IsLast = true;
1006 for (const auto &Epilog : llvm::reverse(C&: info->EpilogMap)) {
1007 if (IsLast) {
1008 IsLast = false;
1009 uint8_t Flags = LastEpilogIsAtEnd ? 0x01 : 0;
1010 OS->emitInt8(Value: EpilogSize);
1011 OS->emitInt8(Value: (Flags << 4) | Win64EH::UOP_Epilog);
1012
1013 if (LastEpilogIsAtEnd)
1014 continue;
1015 }
1016
1017 // Each epilog is emitted as a fixup, since we can't measure the distance
1018 // between the start of the epilog and the end of the function until
1019 // layout has been completed.
1020 auto *MCE = MCUnwindV2EpilogTargetExpr::create(FrameInfo: *info, Epilog: Epilog.second,
1021 EpilogSize_: EpilogSize, Ctx&: context);
1022 OS->addFixup(Value: MCE, Kind: FK_Data_2);
1023 OS->appendContents(Num: 2, Elt: 0);
1024 }
1025 }
1026 if (AddPaddingEpilogCode)
1027 streamer.emitInt16(Value: Win64EH::UOP_Epilog << 8);
1028
1029 // Emit unwind instructions (in reverse order).
1030 uint8_t numInst = info->Instructions.size();
1031 for (uint8_t c = 0; c < numInst; ++c) {
1032 WinEH::Instruction inst = info->Instructions.back();
1033 info->Instructions.pop_back();
1034 EmitUnwindCode(streamer, begin: info->Begin, inst);
1035 }
1036
1037 // For alignment purposes, the instruction array will always have an even
1038 // number of entries, with the final entry potentially unused (in which case
1039 // the array will be one longer than indicated by the count of unwind codes
1040 // field).
1041 if (numCodes & 1) {
1042 streamer.emitInt16(Value: 0);
1043 }
1044
1045 if (flags & (Win64EH::UNW_ChainInfo << 3))
1046 EmitRuntimeFunction(streamer, info: info->ChainedParent);
1047 else if (flags &
1048 ((Win64EH::UNW_TerminateHandler|Win64EH::UNW_ExceptionHandler) << 3))
1049 streamer.emitValue(Value: MCSymbolRefExpr::create(Symbol: info->ExceptionHandler,
1050 specifier: MCSymbolRefExpr::VK_COFF_IMGREL32,
1051 Ctx&: context), Size: 4);
1052 else if (numCodes == 0) {
1053 // The minimum size of an UNWIND_INFO struct is 8 bytes. If we're not
1054 // a chained unwind info, if there is no handler, and if there are fewer
1055 // than 2 slots used in the unwind code array, we have to pad to 8 bytes.
1056 streamer.emitInt32(Value: 0);
1057 }
1058}
1059
1060bool MCUnwindV2EpilogTargetExpr::evaluateAsRelocatableImpl(
1061 MCValue &Res, const MCAssembler *Asm) const {
1062 // Calculate the offset to this epilog, and validate it's within the allowed
1063 // range.
1064 auto Offset = GetOptionalAbsDifference(Assembler: *Asm, LHS: FrameInfo.End, RHS: UnwindV2Start);
1065 if (!Offset) {
1066 Asm->getContext().reportError(
1067 L: Loc, Msg: "Failed to evaluate epilog offset for Unwind v2 in " +
1068 FrameInfo.Function->getName());
1069 return false;
1070 }
1071 assert(*Offset > 0);
1072 constexpr uint16_t MaxEpilogOffset = 0x0fff;
1073 if (*Offset > MaxEpilogOffset) {
1074 Asm->getContext().reportError(
1075 L: Loc, Msg: "Epilog offset is too large for Unwind v2 in " +
1076 FrameInfo.Function->getName());
1077 return false;
1078 }
1079
1080 // Validate that all epilogs are the same size.
1081 auto Size = GetOptionalAbsDifference(Assembler: *Asm, LHS: EpilogEnd, RHS: UnwindV2Start);
1082 if (Size != (EpilogSize - 1)) {
1083 Asm->getContext().reportError(
1084 L: Loc, Msg: "Size of this epilog does not match size of last epilog in " +
1085 FrameInfo.Function->getName());
1086 return false;
1087 }
1088
1089 auto HighBits = *Offset >> 8;
1090 Res = MCValue::get(Val: (HighBits << 12) | (Win64EH::UOP_Epilog << 8) |
1091 (*Offset & 0xFF));
1092 return true;
1093}
1094
1095bool MCUnwindV3EpilogOffsetTargetExpr::evaluateAsRelocatableImpl(
1096 MCValue &Res, const MCAssembler *Asm) const {
1097 // The first epilog descriptor is encoded relative to the fragment tail (the
1098 // first byte past the end of the fragment); subsequent descriptors are
1099 // encoded relative to the previous epilog's start. Both bases yield a
1100 // non-positive offset, as required by the V3 "same sign" rule.
1101 const MCSymbol *Base = PrevEpilogStart ? PrevEpilogStart : FrameInfo.End;
1102 if (!Base) {
1103 Asm->getContext().reportError(
1104 L: Loc, Msg: "Missing fragment end for V3 epilog offset in " +
1105 FrameInfo.Function->getName());
1106 return false;
1107 }
1108
1109 auto Offset = GetOptionalAbsDifference(Assembler: *Asm, LHS: EpilogStart, RHS: Base);
1110 if (!Offset) {
1111 Asm->getContext().reportError(
1112 L: Loc, Msg: "Failed to evaluate epilog offset for V3 unwind info in " +
1113 FrameInfo.Function->getName());
1114 return false;
1115 }
1116 if (*Offset < INT16_MIN || *Offset > INT16_MAX) {
1117 Asm->getContext().reportError(
1118 L: Loc, Msg: "Epilog offset " + Twine(*Offset) +
1119 " out of signed 16-bit range for V3 encoding in " +
1120 FrameInfo.Function->getName());
1121 return false;
1122 }
1123
1124 Res = MCValue::get(Val: *Offset);
1125 return true;
1126}
1127
1128void llvm::Win64EH::UnwindEmitter::Emit(MCStreamer &Streamer) const {
1129 // Emit the unwind info structs first.
1130 for (const auto &CFI : Streamer.getWinFrameInfos()) {
1131 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: CFI->TextSection);
1132 Streamer.switchSection(Section: XData);
1133 ::EmitUnwindInfo(streamer&: Streamer, info: CFI.get());
1134 }
1135
1136 // Now emit RUNTIME_FUNCTION entries.
1137 for (const auto &CFI : Streamer.getWinFrameInfos()) {
1138 MCSection *PData = Streamer.getAssociatedPDataSection(TextSec: CFI->TextSection);
1139 Streamer.switchSection(Section: PData);
1140 EmitRuntimeFunction(streamer&: Streamer, info: CFI.get());
1141 }
1142}
1143
1144void llvm::Win64EH::UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer,
1145 WinEH::FrameInfo *info,
1146 bool HandlerData) const {
1147 // Switch sections (the static function above is meant to be called from
1148 // here and from Emit().
1149 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: info->TextSection);
1150 Streamer.switchSection(Section: XData);
1151
1152 ::EmitUnwindInfo(streamer&: Streamer, info);
1153}
1154
1155static const MCExpr *GetSubDivExpr(MCStreamer &Streamer, const MCSymbol *LHS,
1156 const MCSymbol *RHS, int Div) {
1157 MCContext &Context = Streamer.getContext();
1158 const MCExpr *Expr =
1159 MCBinaryExpr::createSub(LHS: MCSymbolRefExpr::create(Symbol: LHS, Ctx&: Context),
1160 RHS: MCSymbolRefExpr::create(Symbol: RHS, Ctx&: Context), Ctx&: Context);
1161 if (Div != 1)
1162 Expr = MCBinaryExpr::createDiv(LHS: Expr, RHS: MCConstantExpr::create(Value: Div, Ctx&: Context),
1163 Ctx&: Context);
1164 return Expr;
1165}
1166
1167static std::optional<int64_t> GetOptionalAbsDifference(MCStreamer &Streamer,
1168 const MCSymbol *LHS,
1169 const MCSymbol *RHS) {
1170 MCObjectStreamer *OS = (MCObjectStreamer *)(&Streamer);
1171 return GetOptionalAbsDifference(Assembler: OS->getAssembler(), LHS, RHS);
1172}
1173
1174static int64_t GetAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS,
1175 const MCSymbol *RHS) {
1176 std::optional<int64_t> MaybeDiff =
1177 GetOptionalAbsDifference(Streamer, LHS, RHS);
1178 if (!MaybeDiff)
1179 report_fatal_error(reason: "Failed to evaluate function length in SEH unwind info");
1180 return *MaybeDiff;
1181}
1182
1183static void checkARM64Instructions(MCStreamer &Streamer,
1184 ArrayRef<WinEH::Instruction> Insns,
1185 const MCSymbol *Begin, const MCSymbol *End,
1186 StringRef Name, StringRef Type) {
1187 if (!End)
1188 return;
1189 std::optional<int64_t> MaybeDistance =
1190 GetOptionalAbsDifference(Streamer, LHS: End, RHS: Begin);
1191 if (!MaybeDistance)
1192 return;
1193 uint32_t Distance = (uint32_t)*MaybeDistance;
1194
1195 for (const auto &I : Insns) {
1196 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {
1197 default:
1198 break;
1199 case Win64EH::UOP_TrapFrame:
1200 case Win64EH::UOP_PushMachFrame:
1201 case Win64EH::UOP_Context:
1202 case Win64EH::UOP_ECContext:
1203 case Win64EH::UOP_ClearUnwoundToCall:
1204 // Can't reason about these opcodes and how they map to actual
1205 // instructions.
1206 return;
1207 }
1208 }
1209 // Exclude the end opcode which doesn't map to an instruction.
1210 uint32_t InstructionBytes = 4 * (Insns.size() - 1);
1211 if (Distance != InstructionBytes) {
1212 Streamer.getContext().reportError(
1213 L: SMLoc(), Msg: "Incorrect size for " + Name + " " + Type + ": " +
1214 Twine(Distance) +
1215 " bytes of instructions in range, but .seh directives "
1216 "corresponding to " +
1217 Twine(InstructionBytes) + " bytes\n");
1218 }
1219}
1220
1221static uint32_t ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) {
1222 uint32_t Count = 0;
1223 for (const auto &I : Insns) {
1224 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {
1225 default:
1226 llvm_unreachable("Unsupported ARM64 unwind code");
1227 case Win64EH::UOP_AllocSmall:
1228 Count += 1;
1229 break;
1230 case Win64EH::UOP_AllocMedium:
1231 Count += 2;
1232 break;
1233 case Win64EH::UOP_AllocLarge:
1234 Count += 4;
1235 break;
1236 case Win64EH::UOP_SaveR19R20X:
1237 Count += 1;
1238 break;
1239 case Win64EH::UOP_SaveFPLRX:
1240 Count += 1;
1241 break;
1242 case Win64EH::UOP_SaveFPLR:
1243 Count += 1;
1244 break;
1245 case Win64EH::UOP_SaveReg:
1246 Count += 2;
1247 break;
1248 case Win64EH::UOP_SaveRegP:
1249 Count += 2;
1250 break;
1251 case Win64EH::UOP_SaveRegPX:
1252 Count += 2;
1253 break;
1254 case Win64EH::UOP_SaveRegX:
1255 Count += 2;
1256 break;
1257 case Win64EH::UOP_SaveLRPair:
1258 Count += 2;
1259 break;
1260 case Win64EH::UOP_SaveFReg:
1261 Count += 2;
1262 break;
1263 case Win64EH::UOP_SaveFRegP:
1264 Count += 2;
1265 break;
1266 case Win64EH::UOP_SaveFRegX:
1267 Count += 2;
1268 break;
1269 case Win64EH::UOP_SaveFRegPX:
1270 Count += 2;
1271 break;
1272 case Win64EH::UOP_SetFP:
1273 Count += 1;
1274 break;
1275 case Win64EH::UOP_AddFP:
1276 Count += 2;
1277 break;
1278 case Win64EH::UOP_Nop:
1279 Count += 1;
1280 break;
1281 case Win64EH::UOP_End:
1282 Count += 1;
1283 break;
1284 case Win64EH::UOP_SaveNext:
1285 Count += 1;
1286 break;
1287 case Win64EH::UOP_TrapFrame:
1288 Count += 1;
1289 break;
1290 case Win64EH::UOP_PushMachFrame:
1291 Count += 1;
1292 break;
1293 case Win64EH::UOP_Context:
1294 Count += 1;
1295 break;
1296 case Win64EH::UOP_ECContext:
1297 Count += 1;
1298 break;
1299 case Win64EH::UOP_ClearUnwoundToCall:
1300 Count += 1;
1301 break;
1302 case Win64EH::UOP_PACSignLR:
1303 Count += 1;
1304 break;
1305 case Win64EH::UOP_AllocZ:
1306 Count += 2;
1307 break;
1308 case Win64EH::UOP_SaveAnyRegI:
1309 case Win64EH::UOP_SaveAnyRegIP:
1310 case Win64EH::UOP_SaveAnyRegD:
1311 case Win64EH::UOP_SaveAnyRegDP:
1312 case Win64EH::UOP_SaveAnyRegQ:
1313 case Win64EH::UOP_SaveAnyRegQP:
1314 case Win64EH::UOP_SaveAnyRegIX:
1315 case Win64EH::UOP_SaveAnyRegIPX:
1316 case Win64EH::UOP_SaveAnyRegDX:
1317 case Win64EH::UOP_SaveAnyRegDPX:
1318 case Win64EH::UOP_SaveAnyRegQX:
1319 case Win64EH::UOP_SaveAnyRegQPX:
1320 case Win64EH::UOP_SaveZReg:
1321 case Win64EH::UOP_SavePReg:
1322 Count += 3;
1323 break;
1324 }
1325 }
1326 return Count;
1327}
1328
1329// Unwind opcode encodings and restrictions are documented at
1330// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
1331static void ARM64EmitUnwindCode(MCStreamer &streamer,
1332 const WinEH::Instruction &inst) {
1333 uint8_t b, reg;
1334 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {
1335 default:
1336 llvm_unreachable("Unsupported ARM64 unwind code");
1337 case Win64EH::UOP_AllocSmall:
1338 b = (inst.Offset >> 4) & 0x1F;
1339 streamer.emitInt8(Value: b);
1340 break;
1341 case Win64EH::UOP_AllocMedium: {
1342 uint16_t hw = (inst.Offset >> 4) & 0x7FF;
1343 b = 0xC0;
1344 b |= (hw >> 8);
1345 streamer.emitInt8(Value: b);
1346 b = hw & 0xFF;
1347 streamer.emitInt8(Value: b);
1348 break;
1349 }
1350 case Win64EH::UOP_AllocLarge: {
1351 uint32_t w;
1352 b = 0xE0;
1353 streamer.emitInt8(Value: b);
1354 w = inst.Offset >> 4;
1355 b = (w & 0x00FF0000) >> 16;
1356 streamer.emitInt8(Value: b);
1357 b = (w & 0x0000FF00) >> 8;
1358 streamer.emitInt8(Value: b);
1359 b = w & 0x000000FF;
1360 streamer.emitInt8(Value: b);
1361 break;
1362 }
1363 case Win64EH::UOP_SetFP:
1364 b = 0xE1;
1365 streamer.emitInt8(Value: b);
1366 break;
1367 case Win64EH::UOP_AddFP:
1368 b = 0xE2;
1369 streamer.emitInt8(Value: b);
1370 b = (inst.Offset >> 3);
1371 streamer.emitInt8(Value: b);
1372 break;
1373 case Win64EH::UOP_Nop:
1374 b = 0xE3;
1375 streamer.emitInt8(Value: b);
1376 break;
1377 case Win64EH::UOP_SaveR19R20X:
1378 b = 0x20;
1379 b |= (inst.Offset >> 3) & 0x1F;
1380 streamer.emitInt8(Value: b);
1381 break;
1382 case Win64EH::UOP_SaveFPLRX:
1383 b = 0x80;
1384 b |= ((inst.Offset >> 3) - 1) & 0x3F;
1385 streamer.emitInt8(Value: b);
1386 break;
1387 case Win64EH::UOP_SaveFPLR:
1388 b = 0x40;
1389 b |= (inst.Offset >> 3) & 0x3F;
1390 streamer.emitInt8(Value: b);
1391 break;
1392 case Win64EH::UOP_SaveReg:
1393 assert(inst.Register >= 19 && "Saved reg must be >= 19");
1394 reg = inst.Register - 19;
1395 b = 0xD0 | ((reg & 0xC) >> 2);
1396 streamer.emitInt8(Value: b);
1397 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);
1398 streamer.emitInt8(Value: b);
1399 break;
1400 case Win64EH::UOP_SaveRegX:
1401 assert(inst.Register >= 19 && "Saved reg must be >= 19");
1402 reg = inst.Register - 19;
1403 b = 0xD4 | ((reg & 0x8) >> 3);
1404 streamer.emitInt8(Value: b);
1405 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1);
1406 streamer.emitInt8(Value: b);
1407 break;
1408 case Win64EH::UOP_SaveRegP:
1409 assert(inst.Register >= 19 && "Saved registers must be >= 19");
1410 reg = inst.Register - 19;
1411 b = 0xC8 | ((reg & 0xC) >> 2);
1412 streamer.emitInt8(Value: b);
1413 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);
1414 streamer.emitInt8(Value: b);
1415 break;
1416 case Win64EH::UOP_SaveRegPX:
1417 assert(inst.Register >= 19 && "Saved registers must be >= 19");
1418 reg = inst.Register - 19;
1419 b = 0xCC | ((reg & 0xC) >> 2);
1420 streamer.emitInt8(Value: b);
1421 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1);
1422 streamer.emitInt8(Value: b);
1423 break;
1424 case Win64EH::UOP_SaveLRPair:
1425 assert(inst.Register >= 19 && "Saved reg must be >= 19");
1426 reg = inst.Register - 19;
1427 assert((reg % 2) == 0 && "Saved reg must be 19+2*X");
1428 reg /= 2;
1429 b = 0xD6 | ((reg & 0x7) >> 2);
1430 streamer.emitInt8(Value: b);
1431 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);
1432 streamer.emitInt8(Value: b);
1433 break;
1434 case Win64EH::UOP_SaveFReg:
1435 assert(inst.Register >= 8 && "Saved dreg must be >= 8");
1436 reg = inst.Register - 8;
1437 b = 0xDC | ((reg & 0x4) >> 2);
1438 streamer.emitInt8(Value: b);
1439 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);
1440 streamer.emitInt8(Value: b);
1441 break;
1442 case Win64EH::UOP_SaveFRegX:
1443 assert(inst.Register >= 8 && "Saved dreg must be >= 8");
1444 reg = inst.Register - 8;
1445 b = 0xDE;
1446 streamer.emitInt8(Value: b);
1447 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1);
1448 streamer.emitInt8(Value: b);
1449 break;
1450 case Win64EH::UOP_SaveFRegP:
1451 assert(inst.Register >= 8 && "Saved dregs must be >= 8");
1452 reg = inst.Register - 8;
1453 b = 0xD8 | ((reg & 0x4) >> 2);
1454 streamer.emitInt8(Value: b);
1455 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);
1456 streamer.emitInt8(Value: b);
1457 break;
1458 case Win64EH::UOP_SaveFRegPX:
1459 assert(inst.Register >= 8 && "Saved dregs must be >= 8");
1460 reg = inst.Register - 8;
1461 b = 0xDA | ((reg & 0x4) >> 2);
1462 streamer.emitInt8(Value: b);
1463 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1);
1464 streamer.emitInt8(Value: b);
1465 break;
1466 case Win64EH::UOP_End:
1467 b = 0xE4;
1468 streamer.emitInt8(Value: b);
1469 break;
1470 case Win64EH::UOP_SaveNext:
1471 b = 0xE6;
1472 streamer.emitInt8(Value: b);
1473 break;
1474 case Win64EH::UOP_TrapFrame:
1475 b = 0xE8;
1476 streamer.emitInt8(Value: b);
1477 break;
1478 case Win64EH::UOP_PushMachFrame:
1479 b = 0xE9;
1480 streamer.emitInt8(Value: b);
1481 break;
1482 case Win64EH::UOP_Context:
1483 b = 0xEA;
1484 streamer.emitInt8(Value: b);
1485 break;
1486 case Win64EH::UOP_ECContext:
1487 b = 0xEB;
1488 streamer.emitInt8(Value: b);
1489 break;
1490 case Win64EH::UOP_ClearUnwoundToCall:
1491 b = 0xEC;
1492 streamer.emitInt8(Value: b);
1493 break;
1494 case Win64EH::UOP_PACSignLR:
1495 b = 0xFC;
1496 streamer.emitInt8(Value: b);
1497 break;
1498 case Win64EH::UOP_SaveAnyRegI:
1499 case Win64EH::UOP_SaveAnyRegIP:
1500 case Win64EH::UOP_SaveAnyRegD:
1501 case Win64EH::UOP_SaveAnyRegDP:
1502 case Win64EH::UOP_SaveAnyRegQ:
1503 case Win64EH::UOP_SaveAnyRegQP:
1504 case Win64EH::UOP_SaveAnyRegIX:
1505 case Win64EH::UOP_SaveAnyRegIPX:
1506 case Win64EH::UOP_SaveAnyRegDX:
1507 case Win64EH::UOP_SaveAnyRegDPX:
1508 case Win64EH::UOP_SaveAnyRegQX:
1509 case Win64EH::UOP_SaveAnyRegQPX: {
1510 // This assumes the opcodes are listed in the enum in a particular order.
1511 int Op = inst.Operation - Win64EH::UOP_SaveAnyRegI;
1512 int Writeback = Op / 6;
1513 int Paired = Op % 2;
1514 int Mode = (Op / 2) % 3;
1515 int Offset = inst.Offset >> 3;
1516 if (Writeback || Paired || Mode == 2)
1517 Offset >>= 1;
1518 if (Writeback)
1519 --Offset;
1520 b = 0xE7;
1521 streamer.emitInt8(Value: b);
1522 assert(inst.Register < 32);
1523 b = inst.Register | (Writeback << 5) | (Paired << 6);
1524 streamer.emitInt8(Value: b);
1525 b = Offset | (Mode << 6);
1526 streamer.emitInt8(Value: b);
1527 break;
1528 }
1529 case Win64EH::UOP_AllocZ: {
1530 b = 0xDF;
1531 streamer.emitInt8(Value: b);
1532 b = inst.Offset;
1533 streamer.emitInt8(Value: b);
1534 break;
1535 }
1536 case Win64EH::UOP_SaveZReg: {
1537 assert(inst.Register >= 8 && inst.Register <= 23);
1538 assert(inst.Offset < 256);
1539 b = 0xE7;
1540 streamer.emitInt8(Value: b);
1541 reg = inst.Register - 8;
1542 b = ((inst.Offset & 0xC0) >> 1) | reg;
1543 streamer.emitInt8(Value: b);
1544 b = 0xC0 | (inst.Offset & 0x3F);
1545 streamer.emitInt8(Value: b);
1546 break;
1547 }
1548 case Win64EH::UOP_SavePReg: {
1549 assert(inst.Register >= 4 && inst.Register <= 15);
1550 assert(inst.Offset < 256);
1551 b = 0xE7;
1552 streamer.emitInt8(Value: b);
1553 reg = inst.Register;
1554 b = ((inst.Offset & 0xC0) >> 1) | 0x10 | reg;
1555 streamer.emitInt8(Value: b);
1556 b = 0xC0 | (inst.Offset & 0x3F);
1557 streamer.emitInt8(Value: b);
1558 break;
1559 }
1560 }
1561}
1562
1563// Returns the epilog symbol of an epilog with the exact same unwind code
1564// sequence, if it exists. Otherwise, returns nullptr.
1565// EpilogInstrs - Unwind codes for the current epilog.
1566// Epilogs - Epilogs that potentialy match the current epilog.
1567static MCSymbol*
1568FindMatchingEpilog(const std::vector<WinEH::Instruction>& EpilogInstrs,
1569 const std::vector<MCSymbol *>& Epilogs,
1570 const WinEH::FrameInfo *info) {
1571 for (auto *EpilogStart : Epilogs) {
1572 auto InstrsIter = info->EpilogMap.find(Key: EpilogStart);
1573 assert(InstrsIter != info->EpilogMap.end() &&
1574 "Epilog not found in EpilogMap");
1575 const auto &Instrs = InstrsIter->second.Instructions;
1576
1577 if (Instrs.size() != EpilogInstrs.size())
1578 continue;
1579
1580 bool Match = true;
1581 for (unsigned i = 0; i < Instrs.size(); ++i)
1582 if (Instrs[i] != EpilogInstrs[i]) {
1583 Match = false;
1584 break;
1585 }
1586
1587 if (Match)
1588 return EpilogStart;
1589 }
1590 return nullptr;
1591}
1592
1593static void simplifyARM64Opcodes(std::vector<WinEH::Instruction> &Instructions,
1594 bool Reverse) {
1595 unsigned PrevOffset = -1;
1596 unsigned PrevRegister = -1;
1597
1598 // Iterate over instructions in a forward order (for prologues),
1599 // backwards for epilogues (i.e. always reverse compared to how the
1600 // opcodes are stored).
1601 for (WinEH::Instruction &Inst :
1602 llvm::reverse_conditionally(C&: Instructions, ShouldReverse: Reverse)) {
1603 // Convert 2-byte opcodes into equivalent 1-byte ones.
1604 if (Inst.Operation == Win64EH::UOP_SaveRegP && Inst.Register == 29) {
1605 Inst.Operation = Win64EH::UOP_SaveFPLR;
1606 Inst.Register = -1;
1607 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX &&
1608 Inst.Register == 29) {
1609 Inst.Operation = Win64EH::UOP_SaveFPLRX;
1610 Inst.Register = -1;
1611 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX &&
1612 Inst.Register == 19 && Inst.Offset <= 248) {
1613 Inst.Operation = Win64EH::UOP_SaveR19R20X;
1614 Inst.Register = -1;
1615 } else if (Inst.Operation == Win64EH::UOP_AddFP && Inst.Offset == 0) {
1616 Inst.Operation = Win64EH::UOP_SetFP;
1617 } else if (Inst.Operation == Win64EH::UOP_SaveRegP &&
1618 Inst.Register == PrevRegister + 2 &&
1619 Inst.Offset == PrevOffset + 16) {
1620 Inst.Operation = Win64EH::UOP_SaveNext;
1621 Inst.Register = -1;
1622 Inst.Offset = 0;
1623 // Intentionally not creating UOP_SaveNext for float register pairs,
1624 // as current versions of Windows (up to at least 20.04) is buggy
1625 // regarding SaveNext for float pairs.
1626 }
1627 // Update info about the previous instruction, for detecting if
1628 // the next one can be made a UOP_SaveNext
1629 if (Inst.Operation == Win64EH::UOP_SaveR19R20X) {
1630 PrevOffset = 0;
1631 PrevRegister = 19;
1632 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX) {
1633 PrevOffset = 0;
1634 PrevRegister = Inst.Register;
1635 } else if (Inst.Operation == Win64EH::UOP_SaveRegP) {
1636 PrevOffset = Inst.Offset;
1637 PrevRegister = Inst.Register;
1638 } else if (Inst.Operation == Win64EH::UOP_SaveNext) {
1639 PrevRegister += 2;
1640 PrevOffset += 16;
1641 } else {
1642 PrevRegister = -1;
1643 PrevOffset = -1;
1644 }
1645 }
1646}
1647
1648// Check if an epilog exists as a subset of the end of a prolog (backwards).
1649static int
1650getARM64OffsetInProlog(const std::vector<WinEH::Instruction> &Prolog,
1651 const std::vector<WinEH::Instruction> &Epilog) {
1652 // Can't find an epilog as a subset if it is longer than the prolog.
1653 if (Epilog.size() > Prolog.size())
1654 return -1;
1655
1656 // Check that the epilog actually is a perfect match for the end (backwrds)
1657 // of the prolog.
1658 for (int I = Epilog.size() - 1; I >= 0; I--) {
1659 if (Prolog[I] != Epilog[Epilog.size() - 1 - I])
1660 return -1;
1661 }
1662
1663 if (Epilog.size() == Prolog.size())
1664 return 0;
1665
1666 // If the epilog was a subset of the prolog, find its offset.
1667 return ARM64CountOfUnwindCodes(Insns: ArrayRef<WinEH::Instruction>(
1668 &Prolog[Epilog.size()], Prolog.size() - Epilog.size()));
1669}
1670
1671static int checkARM64PackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info,
1672 WinEH::FrameInfo::Segment *Seg,
1673 int PrologCodeBytes) {
1674 // Can only pack if there's one single epilog
1675 if (Seg->Epilogs.size() != 1)
1676 return -1;
1677
1678 MCSymbol *Sym = Seg->Epilogs.begin()->first;
1679 const std::vector<WinEH::Instruction> &Epilog =
1680 info->EpilogMap[Sym].Instructions;
1681
1682 // Check that the epilog actually is at the very end of the function,
1683 // otherwise it can't be packed.
1684 uint32_t DistanceFromEnd =
1685 (uint32_t)(Seg->Offset + Seg->Length - Seg->Epilogs.begin()->second);
1686 if (DistanceFromEnd / 4 != Epilog.size())
1687 return -1;
1688
1689 int RetVal = -1;
1690 // Even if we don't end up sharing opcodes with the prolog, we can still
1691 // write the offset as a packed offset, if the single epilog is located at
1692 // the end of the function and the offset (pointing after the prolog) fits
1693 // as a packed offset.
1694 if (PrologCodeBytes <= 31 &&
1695 PrologCodeBytes + ARM64CountOfUnwindCodes(Insns: Epilog) <= 124)
1696 RetVal = PrologCodeBytes;
1697
1698 int Offset = getARM64OffsetInProlog(Prolog: info->Instructions, Epilog);
1699 if (Offset < 0)
1700 return RetVal;
1701
1702 // Check that the offset and prolog size fits in the first word; it's
1703 // unclear whether the epilog count in the extension word can be taken
1704 // as packed epilog offset.
1705 if (Offset > 31 || PrologCodeBytes > 124)
1706 return RetVal;
1707
1708 // As we choose to express the epilog as part of the prolog, remove the
1709 // epilog from the map, so we don't try to emit its opcodes.
1710 info->EpilogMap.erase(Key: Sym);
1711 return Offset;
1712}
1713
1714static bool tryARM64PackedUnwind(WinEH::FrameInfo *info, uint32_t FuncLength,
1715 int PackedEpilogOffset) {
1716 if (PackedEpilogOffset == 0) {
1717 // Fully symmetric prolog and epilog, should be ok for packed format.
1718 // For CR=3, the corresponding synthesized epilog actually lacks the
1719 // SetFP opcode, but unwinding should work just fine despite that
1720 // (if at the SetFP opcode, the unwinder considers it as part of the
1721 // function body and just unwinds the full prolog instead).
1722 } else if (PackedEpilogOffset == 1) {
1723 // One single case of differences between prolog and epilog is allowed:
1724 // The epilog can lack a single SetFP that is the last opcode in the
1725 // prolog, for the CR=3 case.
1726 if (info->Instructions.back().Operation != Win64EH::UOP_SetFP)
1727 return false;
1728 } else {
1729 // Too much difference between prolog and epilog.
1730 return false;
1731 }
1732 unsigned RegI = 0, RegF = 0;
1733 int Predecrement = 0;
1734 enum {
1735 Start,
1736 Start2,
1737 Start3,
1738 IntRegs,
1739 FloatRegs,
1740 InputArgs,
1741 StackAdjust,
1742 FrameRecord,
1743 End
1744 } Location = Start;
1745 bool StandaloneLR = false, FPLRPair = false;
1746 bool PAC = false;
1747 int StackOffset = 0;
1748 int Nops = 0;
1749 // Iterate over the prolog and check that all opcodes exactly match
1750 // the canonical order and form. A more lax check could verify that
1751 // all saved registers are in the expected locations, but not enforce
1752 // the order - that would work fine when unwinding from within
1753 // functions, but not be exactly right if unwinding happens within
1754 // prologs/epilogs.
1755 for (auto It = info->Instructions.begin(), EndIt = info->Instructions.end();
1756 It != EndIt; It++) {
1757 const WinEH::Instruction &Inst = *It;
1758 switch (Inst.Operation) {
1759 case Win64EH::UOP_End:
1760 if (Location != Start)
1761 return false;
1762 Location = Start2;
1763 break;
1764 case Win64EH::UOP_PACSignLR:
1765 if (Location != Start2)
1766 return false;
1767 PAC = true;
1768 Location = Start3;
1769 break;
1770 case Win64EH::UOP_SaveR19R20X:
1771 if (Location != Start2 && Location != Start3)
1772 return false;
1773 Predecrement = Inst.Offset;
1774 RegI = 2;
1775 Location = IntRegs;
1776 break;
1777 case Win64EH::UOP_SaveRegX:
1778 if (Location != Start2 && Location != Start3)
1779 return false;
1780 Predecrement = Inst.Offset;
1781 if (Inst.Register == 19)
1782 RegI += 1;
1783 else if (Inst.Register == 30)
1784 StandaloneLR = true;
1785 else
1786 return false;
1787 // Odd register; can't be any further int registers.
1788 Location = FloatRegs;
1789 break;
1790 case Win64EH::UOP_SaveRegPX:
1791 // Can't have this in a canonical prologue. Either this has been
1792 // canonicalized into SaveR19R20X or SaveFPLRX, or it's an unsupported
1793 // register pair.
1794 // It can't be canonicalized into SaveR19R20X if the offset is
1795 // larger than 248 bytes, but even with the maximum case with
1796 // RegI=10/RegF=8/CR=1/H=1, we end up with SavSZ = 216, which should
1797 // fit into SaveR19R20X.
1798 // The unwinding opcodes can't describe the otherwise seemingly valid
1799 // case for RegI=1 CR=1, that would start with a
1800 // "stp x19, lr, [sp, #-...]!" as that fits neither SaveRegPX nor
1801 // SaveLRPair.
1802 return false;
1803 case Win64EH::UOP_SaveRegP:
1804 if (Location != IntRegs || Inst.Offset != 8 * RegI ||
1805 Inst.Register != 19 + RegI)
1806 return false;
1807 RegI += 2;
1808 break;
1809 case Win64EH::UOP_SaveReg:
1810 if (Location != IntRegs || Inst.Offset != 8 * RegI)
1811 return false;
1812 if (Inst.Register == 19 + RegI)
1813 RegI += 1;
1814 else if (Inst.Register == 30)
1815 StandaloneLR = true;
1816 else
1817 return false;
1818 // Odd register; can't be any further int registers.
1819 Location = FloatRegs;
1820 break;
1821 case Win64EH::UOP_SaveLRPair:
1822 if (Location != IntRegs || Inst.Offset != 8 * RegI ||
1823 Inst.Register != 19 + RegI)
1824 return false;
1825 RegI += 1;
1826 StandaloneLR = true;
1827 Location = FloatRegs;
1828 break;
1829 case Win64EH::UOP_SaveFRegX:
1830 // Packed unwind can't handle prologs that only save one single
1831 // float register.
1832 return false;
1833 case Win64EH::UOP_SaveFReg:
1834 if (Location != FloatRegs || RegF == 0 || Inst.Register != 8 + RegF ||
1835 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF))
1836 return false;
1837 RegF += 1;
1838 Location = InputArgs;
1839 break;
1840 case Win64EH::UOP_SaveFRegPX:
1841 if ((Location != Start2 && Location != Start3) || Inst.Register != 8)
1842 return false;
1843 Predecrement = Inst.Offset;
1844 RegF = 2;
1845 Location = FloatRegs;
1846 break;
1847 case Win64EH::UOP_SaveFRegP:
1848 if ((Location != IntRegs && Location != FloatRegs) ||
1849 Inst.Register != 8 + RegF ||
1850 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF))
1851 return false;
1852 RegF += 2;
1853 Location = FloatRegs;
1854 break;
1855 case Win64EH::UOP_SaveNext:
1856 if (Location == IntRegs)
1857 RegI += 2;
1858 else if (Location == FloatRegs)
1859 RegF += 2;
1860 else
1861 return false;
1862 break;
1863 case Win64EH::UOP_Nop:
1864 if (Location != IntRegs && Location != FloatRegs && Location != InputArgs)
1865 return false;
1866 Location = InputArgs;
1867 Nops++;
1868 break;
1869 case Win64EH::UOP_AllocSmall:
1870 case Win64EH::UOP_AllocMedium:
1871 if (Location != Start2 && Location != Start3 && Location != IntRegs &&
1872 Location != FloatRegs && Location != InputArgs &&
1873 Location != StackAdjust)
1874 return false;
1875 // Becuase there's no save_lrpair_x opcode, the case of CR=01,
1876 // RegI=1 is handled as a special case with a pair of instructions; an
1877 // alloc followed by a regular save_lrpair. So when encountering an
1878 // alloc here, check if this is the start of such an instruction pair.
1879 if (Location == Start2) { // Can't have this at Start3, after PACSignLR
1880 auto NextIt = It + 1;
1881 if (NextIt != EndIt) {
1882 const WinEH::Instruction &NextInst = *NextIt;
1883 if (NextInst.Operation == Win64EH::UOP_SaveLRPair &&
1884 NextInst.Offset == 0 && NextInst.Register == 19) {
1885 assert(Predecrement == 0);
1886 assert(RegI == 0);
1887 assert(!StandaloneLR);
1888 Predecrement = Inst.Offset;
1889 RegI = 1;
1890 StandaloneLR = true;
1891 Location = FloatRegs;
1892 It++; // Consume both the Alloc and the SaveLRPair
1893 continue;
1894 }
1895 }
1896 }
1897 // Can have either a single decrement, or a pair of decrements with
1898 // 4080 and another decrement.
1899 if (StackOffset == 0)
1900 StackOffset = Inst.Offset;
1901 else if (StackOffset != 4080)
1902 return false;
1903 else
1904 StackOffset += Inst.Offset;
1905 Location = StackAdjust;
1906 break;
1907 case Win64EH::UOP_SaveFPLRX:
1908 // Not allowing FPLRX after StackAdjust; if a StackAdjust is used, it
1909 // should be followed by a FPLR instead.
1910 if (Location != Start2 && Location != Start3 && Location != IntRegs &&
1911 Location != FloatRegs && Location != InputArgs)
1912 return false;
1913 StackOffset = Inst.Offset;
1914 Location = FrameRecord;
1915 FPLRPair = true;
1916 break;
1917 case Win64EH::UOP_SaveFPLR:
1918 // This can only follow after a StackAdjust
1919 if (Location != StackAdjust || Inst.Offset != 0)
1920 return false;
1921 Location = FrameRecord;
1922 FPLRPair = true;
1923 break;
1924 case Win64EH::UOP_SetFP:
1925 if (Location != FrameRecord)
1926 return false;
1927 Location = End;
1928 break;
1929 case Win64EH::UOP_SaveAnyRegI:
1930 case Win64EH::UOP_SaveAnyRegIP:
1931 case Win64EH::UOP_SaveAnyRegD:
1932 case Win64EH::UOP_SaveAnyRegDP:
1933 case Win64EH::UOP_SaveAnyRegQ:
1934 case Win64EH::UOP_SaveAnyRegQP:
1935 case Win64EH::UOP_SaveAnyRegIX:
1936 case Win64EH::UOP_SaveAnyRegIPX:
1937 case Win64EH::UOP_SaveAnyRegDX:
1938 case Win64EH::UOP_SaveAnyRegDPX:
1939 case Win64EH::UOP_SaveAnyRegQX:
1940 case Win64EH::UOP_SaveAnyRegQPX:
1941 // These are never canonical; they don't show up with the usual Arm64
1942 // calling convention.
1943 return false;
1944 case Win64EH::UOP_AllocLarge:
1945 // Allocations this large can't be represented in packed unwind (and
1946 // usually don't fit the canonical form anyway because we need to use
1947 // __chkstk to allocate the stack space).
1948 return false;
1949 case Win64EH::UOP_AddFP:
1950 // "add x29, sp, #N" doesn't show up in the canonical pattern (except for
1951 // N=0, which is UOP_SetFP).
1952 return false;
1953 case Win64EH::UOP_AllocZ:
1954 case Win64EH::UOP_SaveZReg:
1955 case Win64EH::UOP_SavePReg:
1956 // Canonical prologues don't support spilling SVE registers.
1957 return false;
1958 case Win64EH::UOP_TrapFrame:
1959 case Win64EH::UOP_Context:
1960 case Win64EH::UOP_ECContext:
1961 case Win64EH::UOP_ClearUnwoundToCall:
1962 case Win64EH::UOP_PushMachFrame:
1963 // These are special opcodes that aren't normally generated.
1964 return false;
1965 default:
1966 report_fatal_error(reason: "Unknown Arm64 unwind opcode");
1967 }
1968 }
1969 if (RegI > 10 || RegF > 8)
1970 return false;
1971 if (StandaloneLR && FPLRPair)
1972 return false;
1973 if (FPLRPair && Location != End)
1974 return false;
1975 if (Nops != 0 && Nops != 4)
1976 return false;
1977 if (PAC && !FPLRPair)
1978 return false;
1979 int H = Nops == 4;
1980 // For packed unwind info with the H bit set, the prolog and epilog
1981 // actually shouldn't be symmetrical; the epilog shouldn't have any
1982 // nop instructions/opcodes while the prolog has them. We currently
1983 // require exactly symmetrical prologs/epilogs, which is wrong for this
1984 // case - therefore, don't emit packed unwind info for this case.
1985 // See https://github.com/llvm/llvm-project/issues/54879 for details.
1986 //
1987 // Additionally - older versions of Windows also deviated from the
1988 // documentation here; older versions of Windows (at least up until
1989 // 10.0.22000.2176) incorrectly did assume that the epilog has matching
1990 // nop instructions. This is fixed at least in version 10.0.26100.6899.
1991 // As long as we can't assume that the generated code always will run on
1992 // a new enough version, don't emit the packed format here, even if the
1993 // implementation would be fixed to match for the asymmetrical form
1994 // according to the documentation.
1995 if (H)
1996 return false;
1997 // Older versions of Windows (at least in 10.0.22000.2176) incorrectly
1998 // unwind packed unwind info with CR=01, RegI=1, RegF>0, see
1999 // https://github.com/llvm/llvm-project/issues/169588#issuecomment-3584907886.
2000 // This issue only exists in older versions; current versions
2001 // (10.0.26100.6899) do handle it correctly. As long as we can't be sure
2002 // that we won't run on older versions, avoid producing the packed form
2003 // here.
2004 if (StandaloneLR && RegI == 1 && RegF > 0)
2005 return false;
2006 int IntSZ = 8 * RegI;
2007 if (StandaloneLR)
2008 IntSZ += 8;
2009 int FpSZ = 8 * RegF; // RegF not yet decremented
2010 int SavSZ = (IntSZ + FpSZ + 8 * 8 * H + 0xF) & ~0xF;
2011 if (Predecrement != SavSZ)
2012 return false;
2013 if (FPLRPair && StackOffset < 16)
2014 return false;
2015 if (StackOffset % 16)
2016 return false;
2017 uint32_t FrameSize = (StackOffset + SavSZ) / 16;
2018 if (FrameSize > 0x1FF)
2019 return false;
2020 assert(RegF != 1 && "One single float reg not allowed");
2021 if (RegF > 0)
2022 RegF--; // Convert from actual number of registers, to value stored
2023 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier");
2024 int Flag = 0x01; // Function segments not supported yet
2025 int CR = PAC ? 2 : FPLRPair ? 3 : StandaloneLR ? 1 : 0;
2026 info->PackedInfo |= Flag << 0;
2027 info->PackedInfo |= (FuncLength & 0x7FF) << 2;
2028 info->PackedInfo |= (RegF & 0x7) << 13;
2029 info->PackedInfo |= (RegI & 0xF) << 16;
2030 info->PackedInfo |= (H & 0x1) << 20;
2031 info->PackedInfo |= (CR & 0x3) << 21;
2032 info->PackedInfo |= (FrameSize & 0x1FF) << 23;
2033 return true;
2034}
2035
2036static void ARM64ProcessEpilogs(WinEH::FrameInfo *info,
2037 WinEH::FrameInfo::Segment *Seg,
2038 uint32_t &TotalCodeBytes,
2039 MapVector<MCSymbol *, uint32_t> &EpilogInfo) {
2040
2041 std::vector<MCSymbol *> EpilogStarts;
2042 for (auto &I : Seg->Epilogs)
2043 EpilogStarts.push_back(x: I.first);
2044
2045 // Epilogs processed so far.
2046 std::vector<MCSymbol *> AddedEpilogs;
2047 for (auto *S : EpilogStarts) {
2048 MCSymbol *EpilogStart = S;
2049 auto &EpilogInstrs = info->EpilogMap[S].Instructions;
2050 uint32_t CodeBytes = ARM64CountOfUnwindCodes(Insns: EpilogInstrs);
2051
2052 MCSymbol* MatchingEpilog =
2053 FindMatchingEpilog(EpilogInstrs, Epilogs: AddedEpilogs, info);
2054 int PrologOffset;
2055 if (MatchingEpilog) {
2056 assert(EpilogInfo.contains(MatchingEpilog) &&
2057 "Duplicate epilog not found");
2058 EpilogInfo[EpilogStart] = EpilogInfo.lookup(Key: MatchingEpilog);
2059 // Clear the unwind codes in the EpilogMap, so that they don't get output
2060 // in ARM64EmitUnwindInfoForSegment().
2061 EpilogInstrs.clear();
2062 } else if ((PrologOffset = getARM64OffsetInProlog(Prolog: info->Instructions,
2063 Epilog: EpilogInstrs)) >= 0) {
2064 EpilogInfo[EpilogStart] = PrologOffset;
2065 // If the segment doesn't have a prolog, an end_c will be emitted before
2066 // prolog opcodes. So epilog start index in opcodes array is moved by 1.
2067 if (!Seg->HasProlog)
2068 EpilogInfo[EpilogStart] += 1;
2069 // Clear the unwind codes in the EpilogMap, so that they don't get output
2070 // in ARM64EmitUnwindInfoForSegment().
2071 EpilogInstrs.clear();
2072 } else {
2073 EpilogInfo[EpilogStart] = TotalCodeBytes;
2074 TotalCodeBytes += CodeBytes;
2075 AddedEpilogs.push_back(x: EpilogStart);
2076 }
2077 }
2078}
2079
2080static void ARM64FindSegmentsInFunction(MCStreamer &streamer,
2081 WinEH::FrameInfo *info,
2082 int64_t RawFuncLength) {
2083 if (info->PrologEnd)
2084 checkARM64Instructions(Streamer&: streamer, Insns: info->Instructions, Begin: info->Begin,
2085 End: info->PrologEnd, Name: info->Function->getName(),
2086 Type: "prologue");
2087 struct EpilogStartEnd {
2088 MCSymbol *Start;
2089 int64_t Offset;
2090 int64_t End;
2091 };
2092 // Record Start and End of each epilog.
2093 SmallVector<struct EpilogStartEnd, 4> Epilogs;
2094 for (auto &I : info->EpilogMap) {
2095 MCSymbol *Start = I.first;
2096 auto &Instrs = I.second.Instructions;
2097 int64_t Offset = GetAbsDifference(Streamer&: streamer, LHS: Start, RHS: info->Begin);
2098 checkARM64Instructions(Streamer&: streamer, Insns: Instrs, Begin: Start, End: I.second.End,
2099 Name: info->Function->getName(), Type: "epilogue");
2100 assert((Epilogs.size() == 0 || Offset >= Epilogs.back().End) &&
2101 "Epilogs should be monotonically ordered");
2102 // Exclue the end opcode from Instrs.size() when calculating the end of the
2103 // epilog.
2104 Epilogs.push_back(Elt: {.Start: Start, .Offset: Offset, .End: Offset + (int64_t)(Instrs.size() - 1) * 4});
2105 }
2106
2107 unsigned E = 0;
2108 int64_t SegLimit = 0xFFFFC;
2109 int64_t SegOffset = 0;
2110
2111 if (RawFuncLength > SegLimit) {
2112
2113 int64_t RemainingLength = RawFuncLength;
2114
2115 while (RemainingLength > SegLimit) {
2116 // Try divide the function into segments, requirements:
2117 // 1. Segment length <= 0xFFFFC;
2118 // 2. Each Prologue or Epilogue must be fully within a segment.
2119 int64_t SegLength = SegLimit;
2120 int64_t SegEnd = SegOffset + SegLength;
2121 // Keep record on symbols and offsets of epilogs in this segment.
2122 MapVector<MCSymbol *, int64_t> EpilogsInSegment;
2123
2124 while (E < Epilogs.size() && Epilogs[E].End < SegEnd) {
2125 // Epilogs within current segment.
2126 EpilogsInSegment[Epilogs[E].Start] = Epilogs[E].Offset;
2127 ++E;
2128 }
2129
2130 // At this point, we have:
2131 // 1. Put all epilogs in segments already. No action needed here; or
2132 // 2. Found an epilog that will cross segments boundry. We need to
2133 // move back current segment's end boundry, so the epilog is entirely
2134 // in the next segment; or
2135 // 3. Left at least one epilog that is entirely after this segment.
2136 // It'll be handled by the next iteration, or the last segment.
2137 if (E < Epilogs.size() && Epilogs[E].Offset <= SegEnd)
2138 // Move back current Segment's end boundry.
2139 SegLength = Epilogs[E].Offset - SegOffset;
2140
2141 auto Seg = WinEH::FrameInfo::Segment(
2142 SegOffset, SegLength, /* HasProlog */!SegOffset);
2143 Seg.Epilogs = std::move(EpilogsInSegment);
2144 info->Segments.push_back(x: Seg);
2145
2146 SegOffset += SegLength;
2147 RemainingLength -= SegLength;
2148 }
2149 }
2150
2151 // Add the last segment when RawFuncLength > 0xFFFFC,
2152 // or the only segment otherwise.
2153 auto LastSeg =
2154 WinEH::FrameInfo::Segment(SegOffset, RawFuncLength - SegOffset,
2155 /* HasProlog */!SegOffset);
2156 for (; E < Epilogs.size(); ++E)
2157 LastSeg.Epilogs[Epilogs[E].Start] = Epilogs[E].Offset;
2158 info->Segments.push_back(x: LastSeg);
2159}
2160
2161static void ARM64EmitUnwindInfoForSegment(MCStreamer &streamer,
2162 WinEH::FrameInfo *info,
2163 WinEH::FrameInfo::Segment &Seg,
2164 bool TryPacked = true) {
2165 MCContext &context = streamer.getContext();
2166 MCSymbol *Label = context.createTempSymbol();
2167
2168 streamer.emitValueToAlignment(Alignment: Align(4));
2169 streamer.emitLabel(Symbol: Label);
2170 Seg.Symbol = Label;
2171 // Use the 1st segemnt's label as function's.
2172 if (Seg.Offset == 0)
2173 info->Symbol = Label;
2174
2175 bool HasProlog = Seg.HasProlog;
2176 bool HasEpilogs = (Seg.Epilogs.size() != 0);
2177
2178 uint32_t SegLength = (uint32_t)Seg.Length / 4;
2179 uint32_t PrologCodeBytes = info->PrologCodeBytes;
2180
2181 int PackedEpilogOffset = HasEpilogs ?
2182 checkARM64PackedEpilog(streamer, info, Seg: &Seg, PrologCodeBytes) : -1;
2183
2184 // TODO:
2185 // 1. Enable packed unwind info (.pdata only) for multi-segment functions.
2186 // 2. Emit packed unwind info (.pdata only) for segments that have neithor
2187 // prolog nor epilog.
2188 if (info->Segments.size() == 1 && PackedEpilogOffset >= 0 &&
2189 uint32_t(PackedEpilogOffset) < PrologCodeBytes &&
2190 !info->HandlesExceptions && SegLength <= 0x7ff && TryPacked) {
2191 // Matching prolog/epilog and no exception handlers; check if the
2192 // prolog matches the patterns that can be described by the packed
2193 // format.
2194
2195 // info->Symbol was already set even if we didn't actually write any
2196 // unwind info there. Keep using that as indicator that this unwind
2197 // info has been generated already.
2198 if (tryARM64PackedUnwind(info, FuncLength: SegLength, PackedEpilogOffset))
2199 return;
2200 }
2201
2202 // If the prolog is not in this segment, we need to emit an end_c, which takes
2203 // 1 byte, before prolog unwind ops.
2204 if (!HasProlog) {
2205 PrologCodeBytes += 1;
2206 if (PackedEpilogOffset >= 0)
2207 PackedEpilogOffset += 1;
2208 // If a segment has neither prolog nor epilog, "With full .xdata record,
2209 // Epilog Count = 1. Epilog Start Index points to end_c."
2210 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#function-fragments
2211 // TODO: We can remove this if testing shows zero epilog scope is ok with
2212 // MS unwinder.
2213 if (!HasEpilogs)
2214 // Pack the fake epilog into phantom prolog.
2215 PackedEpilogOffset = 0;
2216 }
2217
2218 uint32_t TotalCodeBytes = PrologCodeBytes;
2219
2220 // Process epilogs.
2221 MapVector<MCSymbol *, uint32_t> EpilogInfo;
2222 ARM64ProcessEpilogs(info, Seg: &Seg, TotalCodeBytes, EpilogInfo);
2223
2224 // Code Words, Epilog count, E, X, Vers, Function Length
2225 uint32_t row1 = 0x0;
2226 uint32_t CodeWords = TotalCodeBytes / 4;
2227 uint32_t CodeWordsMod = TotalCodeBytes % 4;
2228 if (CodeWordsMod)
2229 CodeWords++;
2230 uint32_t EpilogCount =
2231 PackedEpilogOffset >= 0 ? PackedEpilogOffset : Seg.Epilogs.size();
2232 bool ExtensionWord = EpilogCount > 31 || TotalCodeBytes > 124;
2233 if (!ExtensionWord) {
2234 row1 |= (EpilogCount & 0x1F) << 22;
2235 row1 |= (CodeWords & 0x1F) << 27;
2236 }
2237 if (info->HandlesExceptions) // X
2238 row1 |= 1 << 20;
2239 if (PackedEpilogOffset >= 0) // E
2240 row1 |= 1 << 21;
2241 row1 |= SegLength & 0x3FFFF;
2242 streamer.emitInt32(Value: row1);
2243
2244 // Extended Code Words, Extended Epilog Count
2245 if (ExtensionWord) {
2246 // FIXME: We should be able to split unwind info into multiple sections.
2247 if (CodeWords > 0xFF || EpilogCount > 0xFFFF)
2248 report_fatal_error(
2249 reason: "SEH unwind data splitting is only implemented for large functions, "
2250 "cases of too many code words or too many epilogs will be done "
2251 "later");
2252 uint32_t row2 = 0x0;
2253 row2 |= (CodeWords & 0xFF) << 16;
2254 row2 |= (EpilogCount & 0xFFFF);
2255 streamer.emitInt32(Value: row2);
2256 }
2257
2258 if (PackedEpilogOffset < 0) {
2259 // Epilog Start Index, Epilog Start Offset
2260 for (auto &I : EpilogInfo) {
2261 MCSymbol *EpilogStart = I.first;
2262 uint32_t EpilogIndex = I.second;
2263 // Epilog offset within the Segment.
2264 uint32_t EpilogOffset = (uint32_t)(Seg.Epilogs[EpilogStart] - Seg.Offset);
2265 if (EpilogOffset)
2266 EpilogOffset /= 4;
2267 uint32_t row3 = EpilogOffset;
2268 row3 |= (EpilogIndex & 0x3FF) << 22;
2269 streamer.emitInt32(Value: row3);
2270 }
2271 }
2272
2273 // Note that even for segments that have no prolog, we still need to emit
2274 // prolog unwinding opcodes so that the unwinder knows how to unwind from
2275 // such a segment.
2276 // The end_c opcode at the start indicates to the unwinder that the actual
2277 // prolog is outside of the current segment, and the unwinder shouldn't try
2278 // to check for unwinding from a partial prolog.
2279 if (!HasProlog)
2280 // Emit an end_c.
2281 streamer.emitInt8(Value: (uint8_t)0xE5);
2282
2283 // Emit prolog unwind instructions (in reverse order).
2284 for (auto Inst : llvm::reverse(C&: info->Instructions))
2285 ARM64EmitUnwindCode(streamer, inst: Inst);
2286
2287 // Emit epilog unwind instructions
2288 for (auto &I : Seg.Epilogs) {
2289 auto &EpilogInstrs = info->EpilogMap[I.first].Instructions;
2290 for (const WinEH::Instruction &inst : EpilogInstrs)
2291 ARM64EmitUnwindCode(streamer, inst);
2292 }
2293
2294 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes;
2295 assert(BytesMod >= 0);
2296 for (int i = 0; i < BytesMod; i++)
2297 streamer.emitInt8(Value: 0xE3);
2298
2299 if (info->HandlesExceptions)
2300 streamer.emitValue(
2301 Value: MCSymbolRefExpr::create(Symbol: info->ExceptionHandler,
2302 specifier: MCSymbolRefExpr::VK_COFF_IMGREL32, Ctx&: context),
2303 Size: 4);
2304}
2305
2306// Populate the .xdata section. The format of .xdata on ARM64 is documented at
2307// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
2308static void ARM64EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info,
2309 bool TryPacked = true) {
2310 // If this UNWIND_INFO already has a symbol, it's already been emitted.
2311 if (info->Symbol)
2312 return;
2313 // If there's no unwind info here (not even a terminating UOP_End), the
2314 // unwind info is considered bogus and skipped. If this was done in
2315 // response to an explicit .seh_handlerdata, the associated trailing
2316 // handler data is left orphaned in the xdata section.
2317 if (info->empty()) {
2318 info->EmitAttempted = true;
2319 return;
2320 }
2321 if (info->EmitAttempted) {
2322 // If we tried to emit unwind info before (due to an explicit
2323 // .seh_handlerdata directive), but skipped it (because there was no
2324 // valid information to emit at the time), and it later got valid unwind
2325 // opcodes, we can't emit it here, because the trailing handler data
2326 // was already emitted elsewhere in the xdata section.
2327 streamer.getContext().reportError(
2328 L: SMLoc(), Msg: "Earlier .seh_handlerdata for " + info->Function->getName() +
2329 " skipped due to no unwind info at the time "
2330 "(.seh_handlerdata too early?), but the function later "
2331 "did get unwind info that can't be emitted");
2332 return;
2333 }
2334
2335 simplifyARM64Opcodes(Instructions&: info->Instructions, Reverse: false);
2336 for (auto &I : info->EpilogMap)
2337 simplifyARM64Opcodes(Instructions&: I.second.Instructions, Reverse: true);
2338
2339 int64_t RawFuncLength;
2340 if (!info->FuncletOrFuncEnd) {
2341 report_fatal_error(reason: "FuncletOrFuncEnd not set");
2342 } else {
2343 // FIXME: GetAbsDifference tries to compute the length of the function
2344 // immediately, before the whole file is emitted, but in general
2345 // that's impossible: the size in bytes of certain assembler directives
2346 // like .align and .fill is not known until the whole file is parsed and
2347 // relaxations are applied. Currently, GetAbsDifference fails with a fatal
2348 // error in that case. (We mostly don't hit this because inline assembly
2349 // specifying those directives is rare, and we don't normally try to
2350 // align loops on AArch64.)
2351 //
2352 // There are two potential approaches to delaying the computation. One,
2353 // we could emit something like ".word (endfunc-beginfunc)/4+0x10800000",
2354 // as long as we have some conservative estimate we could use to prove
2355 // that we don't need to split the unwind data. Emitting the constant
2356 // is straightforward, but there's no existing code for estimating the
2357 // size of the function.
2358 //
2359 // The other approach would be to use a dedicated, relaxable fragment,
2360 // which could grow to accommodate splitting the unwind data if
2361 // necessary. This is more straightforward, since it automatically works
2362 // without any new infrastructure, and it's consistent with how we handle
2363 // relaxation in other contexts. But it would require some refactoring
2364 // to move parts of the pdata/xdata emission into the implementation of
2365 // a fragment. We could probably continue to encode the unwind codes
2366 // here, but we'd have to emit the pdata, the xdata header, and the
2367 // epilogue scopes later, since they depend on whether the we need to
2368 // split the unwind data.
2369 //
2370 // If this is fixed, remove code in AArch64ISelLowering.cpp that
2371 // disables loop alignment on Windows.
2372 RawFuncLength = GetAbsDifference(Streamer&: streamer, LHS: info->FuncletOrFuncEnd,
2373 RHS: info->Begin);
2374 }
2375
2376 ARM64FindSegmentsInFunction(streamer, info, RawFuncLength);
2377
2378 info->PrologCodeBytes = ARM64CountOfUnwindCodes(Insns: info->Instructions);
2379 for (auto &S : info->Segments)
2380 ARM64EmitUnwindInfoForSegment(streamer, info, Seg&: S, TryPacked);
2381
2382 // Clear prolog instructions after unwind info is emitted for all segments.
2383 info->Instructions.clear();
2384}
2385
2386static uint32_t ARMCountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) {
2387 uint32_t Count = 0;
2388 for (const auto &I : Insns) {
2389 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {
2390 default:
2391 llvm_unreachable("Unsupported ARM unwind code");
2392 case Win64EH::UOP_AllocSmall:
2393 Count += 1;
2394 break;
2395 case Win64EH::UOP_AllocLarge:
2396 Count += 3;
2397 break;
2398 case Win64EH::UOP_AllocHuge:
2399 Count += 4;
2400 break;
2401 case Win64EH::UOP_WideAllocMedium:
2402 Count += 2;
2403 break;
2404 case Win64EH::UOP_WideAllocLarge:
2405 Count += 3;
2406 break;
2407 case Win64EH::UOP_WideAllocHuge:
2408 Count += 4;
2409 break;
2410 case Win64EH::UOP_WideSaveRegMask:
2411 Count += 2;
2412 break;
2413 case Win64EH::UOP_SaveSP:
2414 Count += 1;
2415 break;
2416 case Win64EH::UOP_SaveRegsR4R7LR:
2417 Count += 1;
2418 break;
2419 case Win64EH::UOP_WideSaveRegsR4R11LR:
2420 Count += 1;
2421 break;
2422 case Win64EH::UOP_SaveFRegD8D15:
2423 Count += 1;
2424 break;
2425 case Win64EH::UOP_SaveRegMask:
2426 Count += 2;
2427 break;
2428 case Win64EH::UOP_SaveLR:
2429 Count += 2;
2430 break;
2431 case Win64EH::UOP_SaveFRegD0D15:
2432 Count += 2;
2433 break;
2434 case Win64EH::UOP_SaveFRegD16D31:
2435 Count += 2;
2436 break;
2437 case Win64EH::UOP_Nop:
2438 case Win64EH::UOP_WideNop:
2439 case Win64EH::UOP_End:
2440 case Win64EH::UOP_EndNop:
2441 case Win64EH::UOP_WideEndNop:
2442 Count += 1;
2443 break;
2444 case Win64EH::UOP_Custom: {
2445 int J;
2446 for (J = 3; J > 0; J--)
2447 if (I.Offset & (0xffu << (8 * J)))
2448 break;
2449 Count += J + 1;
2450 break;
2451 }
2452 }
2453 }
2454 return Count;
2455}
2456
2457static uint32_t ARMCountOfInstructionBytes(ArrayRef<WinEH::Instruction> Insns,
2458 bool *HasCustom = nullptr) {
2459 uint32_t Count = 0;
2460 for (const auto &I : Insns) {
2461 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {
2462 default:
2463 llvm_unreachable("Unsupported ARM unwind code");
2464 case Win64EH::UOP_AllocSmall:
2465 case Win64EH::UOP_AllocLarge:
2466 case Win64EH::UOP_AllocHuge:
2467 Count += 2;
2468 break;
2469 case Win64EH::UOP_WideAllocMedium:
2470 case Win64EH::UOP_WideAllocLarge:
2471 case Win64EH::UOP_WideAllocHuge:
2472 Count += 4;
2473 break;
2474 case Win64EH::UOP_WideSaveRegMask:
2475 case Win64EH::UOP_WideSaveRegsR4R11LR:
2476 Count += 4;
2477 break;
2478 case Win64EH::UOP_SaveSP:
2479 Count += 2;
2480 break;
2481 case Win64EH::UOP_SaveRegMask:
2482 case Win64EH::UOP_SaveRegsR4R7LR:
2483 Count += 2;
2484 break;
2485 case Win64EH::UOP_SaveFRegD8D15:
2486 case Win64EH::UOP_SaveFRegD0D15:
2487 case Win64EH::UOP_SaveFRegD16D31:
2488 Count += 4;
2489 break;
2490 case Win64EH::UOP_SaveLR:
2491 Count += 4;
2492 break;
2493 case Win64EH::UOP_Nop:
2494 case Win64EH::UOP_EndNop:
2495 Count += 2;
2496 break;
2497 case Win64EH::UOP_WideNop:
2498 case Win64EH::UOP_WideEndNop:
2499 Count += 4;
2500 break;
2501 case Win64EH::UOP_End:
2502 // This doesn't map to any instruction
2503 break;
2504 case Win64EH::UOP_Custom:
2505 // We can't reason about what instructions this maps to; return a
2506 // phony number to make sure we don't accidentally do epilog packing.
2507 Count += 1000;
2508 if (HasCustom)
2509 *HasCustom = true;
2510 break;
2511 }
2512 }
2513 return Count;
2514}
2515
2516static void checkARMInstructions(MCStreamer &Streamer,
2517 ArrayRef<WinEH::Instruction> Insns,
2518 const MCSymbol *Begin, const MCSymbol *End,
2519 StringRef Name, StringRef Type) {
2520 if (!End)
2521 return;
2522 std::optional<int64_t> MaybeDistance =
2523 GetOptionalAbsDifference(Streamer, LHS: End, RHS: Begin);
2524 if (!MaybeDistance)
2525 return;
2526 uint32_t Distance = (uint32_t)*MaybeDistance;
2527 bool HasCustom = false;
2528 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Insns, HasCustom: &HasCustom);
2529 if (HasCustom)
2530 return;
2531 if (Distance != InstructionBytes) {
2532 Streamer.getContext().reportError(
2533 L: SMLoc(), Msg: "Incorrect size for " + Name + " " + Type + ": " +
2534 Twine(Distance) +
2535 " bytes of instructions in range, but .seh directives "
2536 "corresponding to " +
2537 Twine(InstructionBytes) + " bytes\n");
2538 }
2539}
2540
2541static bool isARMTerminator(const WinEH::Instruction &inst) {
2542 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {
2543 case Win64EH::UOP_End:
2544 case Win64EH::UOP_EndNop:
2545 case Win64EH::UOP_WideEndNop:
2546 return true;
2547 default:
2548 return false;
2549 }
2550}
2551
2552// Unwind opcode encodings and restrictions are documented at
2553// https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling
2554static void ARMEmitUnwindCode(MCStreamer &streamer,
2555 const WinEH::Instruction &inst) {
2556 uint32_t w, lr;
2557 int i;
2558 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {
2559 default:
2560 llvm_unreachable("Unsupported ARM unwind code");
2561 case Win64EH::UOP_AllocSmall:
2562 assert((inst.Offset & 3) == 0);
2563 assert(inst.Offset / 4 <= 0x7f);
2564 streamer.emitInt8(Value: inst.Offset / 4);
2565 break;
2566 case Win64EH::UOP_WideSaveRegMask:
2567 assert((inst.Register & ~0x5fff) == 0);
2568 lr = (inst.Register >> 14) & 1;
2569 w = 0x8000 | (inst.Register & 0x1fff) | (lr << 13);
2570 streamer.emitInt8(Value: (w >> 8) & 0xff);
2571 streamer.emitInt8(Value: (w >> 0) & 0xff);
2572 break;
2573 case Win64EH::UOP_SaveSP:
2574 assert(inst.Register <= 0x0f);
2575 streamer.emitInt8(Value: 0xc0 | inst.Register);
2576 break;
2577 case Win64EH::UOP_SaveRegsR4R7LR:
2578 assert(inst.Register >= 4 && inst.Register <= 7);
2579 assert(inst.Offset <= 1);
2580 streamer.emitInt8(Value: 0xd0 | (inst.Register - 4) | (inst.Offset << 2));
2581 break;
2582 case Win64EH::UOP_WideSaveRegsR4R11LR:
2583 assert(inst.Register >= 8 && inst.Register <= 11);
2584 assert(inst.Offset <= 1);
2585 streamer.emitInt8(Value: 0xd8 | (inst.Register - 8) | (inst.Offset << 2));
2586 break;
2587 case Win64EH::UOP_SaveFRegD8D15:
2588 assert(inst.Register >= 8 && inst.Register <= 15);
2589 streamer.emitInt8(Value: 0xe0 | (inst.Register - 8));
2590 break;
2591 case Win64EH::UOP_WideAllocMedium:
2592 assert((inst.Offset & 3) == 0);
2593 assert(inst.Offset / 4 <= 0x3ff);
2594 w = 0xe800 | (inst.Offset / 4);
2595 streamer.emitInt8(Value: (w >> 8) & 0xff);
2596 streamer.emitInt8(Value: (w >> 0) & 0xff);
2597 break;
2598 case Win64EH::UOP_SaveRegMask:
2599 assert((inst.Register & ~0x40ff) == 0);
2600 lr = (inst.Register >> 14) & 1;
2601 w = 0xec00 | (inst.Register & 0x0ff) | (lr << 8);
2602 streamer.emitInt8(Value: (w >> 8) & 0xff);
2603 streamer.emitInt8(Value: (w >> 0) & 0xff);
2604 break;
2605 case Win64EH::UOP_SaveLR:
2606 assert((inst.Offset & 3) == 0);
2607 assert(inst.Offset / 4 <= 0x0f);
2608 streamer.emitInt8(Value: 0xef);
2609 streamer.emitInt8(Value: inst.Offset / 4);
2610 break;
2611 case Win64EH::UOP_SaveFRegD0D15:
2612 assert(inst.Register <= 15);
2613 assert(inst.Offset <= 15);
2614 assert(inst.Register <= inst.Offset);
2615 streamer.emitInt8(Value: 0xf5);
2616 streamer.emitInt8(Value: (inst.Register << 4) | inst.Offset);
2617 break;
2618 case Win64EH::UOP_SaveFRegD16D31:
2619 assert(inst.Register >= 16 && inst.Register <= 31);
2620 assert(inst.Offset >= 16 && inst.Offset <= 31);
2621 assert(inst.Register <= inst.Offset);
2622 streamer.emitInt8(Value: 0xf6);
2623 streamer.emitInt8(Value: ((inst.Register - 16) << 4) | (inst.Offset - 16));
2624 break;
2625 case Win64EH::UOP_AllocLarge:
2626 assert((inst.Offset & 3) == 0);
2627 assert(inst.Offset / 4 <= 0xffff);
2628 w = inst.Offset / 4;
2629 streamer.emitInt8(Value: 0xf7);
2630 streamer.emitInt8(Value: (w >> 8) & 0xff);
2631 streamer.emitInt8(Value: (w >> 0) & 0xff);
2632 break;
2633 case Win64EH::UOP_AllocHuge:
2634 assert((inst.Offset & 3) == 0);
2635 assert(inst.Offset / 4 <= 0xffffff);
2636 w = inst.Offset / 4;
2637 streamer.emitInt8(Value: 0xf8);
2638 streamer.emitInt8(Value: (w >> 16) & 0xff);
2639 streamer.emitInt8(Value: (w >> 8) & 0xff);
2640 streamer.emitInt8(Value: (w >> 0) & 0xff);
2641 break;
2642 case Win64EH::UOP_WideAllocLarge:
2643 assert((inst.Offset & 3) == 0);
2644 assert(inst.Offset / 4 <= 0xffff);
2645 w = inst.Offset / 4;
2646 streamer.emitInt8(Value: 0xf9);
2647 streamer.emitInt8(Value: (w >> 8) & 0xff);
2648 streamer.emitInt8(Value: (w >> 0) & 0xff);
2649 break;
2650 case Win64EH::UOP_WideAllocHuge:
2651 assert((inst.Offset & 3) == 0);
2652 assert(inst.Offset / 4 <= 0xffffff);
2653 w = inst.Offset / 4;
2654 streamer.emitInt8(Value: 0xfa);
2655 streamer.emitInt8(Value: (w >> 16) & 0xff);
2656 streamer.emitInt8(Value: (w >> 8) & 0xff);
2657 streamer.emitInt8(Value: (w >> 0) & 0xff);
2658 break;
2659 case Win64EH::UOP_Nop:
2660 streamer.emitInt8(Value: 0xfb);
2661 break;
2662 case Win64EH::UOP_WideNop:
2663 streamer.emitInt8(Value: 0xfc);
2664 break;
2665 case Win64EH::UOP_EndNop:
2666 streamer.emitInt8(Value: 0xfd);
2667 break;
2668 case Win64EH::UOP_WideEndNop:
2669 streamer.emitInt8(Value: 0xfe);
2670 break;
2671 case Win64EH::UOP_End:
2672 streamer.emitInt8(Value: 0xff);
2673 break;
2674 case Win64EH::UOP_Custom:
2675 for (i = 3; i > 0; i--)
2676 if (inst.Offset & (0xffu << (8 * i)))
2677 break;
2678 for (; i >= 0; i--)
2679 streamer.emitInt8(Value: (inst.Offset >> (8 * i)) & 0xff);
2680 break;
2681 }
2682}
2683
2684// Check if an epilog exists as a subset of the end of a prolog (backwards).
2685// An epilog may end with one out of three different end opcodes; if this
2686// is the first epilog that shares opcodes with the prolog, we can tolerate
2687// that this opcode differs (and the caller will update the prolog to use
2688// the same end opcode as the epilog). If another epilog already shares
2689// opcodes with the prolog, the ending opcode must be a strict match.
2690static int getARMOffsetInProlog(const std::vector<WinEH::Instruction> &Prolog,
2691 const std::vector<WinEH::Instruction> &Epilog,
2692 bool CanTweakProlog) {
2693 // Can't find an epilog as a subset if it is longer than the prolog.
2694 if (Epilog.size() > Prolog.size())
2695 return -1;
2696
2697 // Check that the epilog actually is a perfect match for the end (backwrds)
2698 // of the prolog.
2699 // If we can adjust the prolog afterwards, don't check that the end opcodes
2700 // match.
2701 int EndIdx = CanTweakProlog ? 1 : 0;
2702 for (int I = Epilog.size() - 1; I >= EndIdx; I--) {
2703 // TODO: Could also allow minor mismatches, e.g. "add sp, #16" vs
2704 // "push {r0-r3}".
2705 if (Prolog[I] != Epilog[Epilog.size() - 1 - I])
2706 return -1;
2707 }
2708
2709 if (CanTweakProlog) {
2710 // Check that both prolog and epilog end with an expected end opcode.
2711 if (Prolog.front().Operation != Win64EH::UOP_End)
2712 return -1;
2713 if (Epilog.back().Operation != Win64EH::UOP_End &&
2714 Epilog.back().Operation != Win64EH::UOP_EndNop &&
2715 Epilog.back().Operation != Win64EH::UOP_WideEndNop)
2716 return -1;
2717 }
2718
2719 // If the epilog was a subset of the prolog, find its offset.
2720 if (Epilog.size() == Prolog.size())
2721 return 0;
2722 return ARMCountOfUnwindCodes(Insns: ArrayRef<WinEH::Instruction>(
2723 &Prolog[Epilog.size()], Prolog.size() - Epilog.size()));
2724}
2725
2726static int checkARMPackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info,
2727 int PrologCodeBytes) {
2728 // Can only pack if there's one single epilog
2729 if (info->EpilogMap.size() != 1)
2730 return -1;
2731
2732 const WinEH::FrameInfo::Epilog &EpilogInfo = info->EpilogMap.begin()->second;
2733 // Can only pack if the epilog is unconditional
2734 if (EpilogInfo.Condition != 0xe) // ARMCC::AL
2735 return -1;
2736
2737 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions;
2738 // Make sure we have at least the trailing end opcode
2739 if (info->Instructions.empty() || Epilog.empty())
2740 return -1;
2741
2742 // Check that the epilog actually is at the very end of the function,
2743 // otherwise it can't be packed.
2744 std::optional<int64_t> MaybeDistance = GetOptionalAbsDifference(
2745 Streamer&: streamer, LHS: info->FuncletOrFuncEnd, RHS: info->EpilogMap.begin()->first);
2746 if (!MaybeDistance)
2747 return -1;
2748 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance;
2749 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Insns: Epilog);
2750 if (DistanceFromEnd != InstructionBytes)
2751 return -1;
2752
2753 int RetVal = -1;
2754 // Even if we don't end up sharing opcodes with the prolog, we can still
2755 // write the offset as a packed offset, if the single epilog is located at
2756 // the end of the function and the offset (pointing after the prolog) fits
2757 // as a packed offset.
2758 if (PrologCodeBytes <= 31 &&
2759 PrologCodeBytes + ARMCountOfUnwindCodes(Insns: Epilog) <= 63)
2760 RetVal = PrologCodeBytes;
2761
2762 int Offset =
2763 getARMOffsetInProlog(Prolog: info->Instructions, Epilog, /*CanTweakProlog=*/true);
2764 if (Offset < 0)
2765 return RetVal;
2766
2767 // Check that the offset and prolog size fits in the first word; it's
2768 // unclear whether the epilog count in the extension word can be taken
2769 // as packed epilog offset.
2770 if (Offset > 31 || PrologCodeBytes > 63)
2771 return RetVal;
2772
2773 // Replace the regular end opcode of the prolog with the one from the
2774 // epilog.
2775 info->Instructions.front() = Epilog.back();
2776
2777 // As we choose to express the epilog as part of the prolog, remove the
2778 // epilog from the map, so we don't try to emit its opcodes.
2779 info->EpilogMap.clear();
2780 return Offset;
2781}
2782
2783static bool parseRegMask(unsigned Mask, bool &HasLR, bool &HasR11,
2784 unsigned &Folded, int &IntRegs) {
2785 if (Mask & (1 << 14)) {
2786 HasLR = true;
2787 Mask &= ~(1 << 14);
2788 }
2789 if (Mask & (1 << 11)) {
2790 HasR11 = true;
2791 Mask &= ~(1 << 11);
2792 }
2793 Folded = 0;
2794 IntRegs = -1;
2795 if (!Mask)
2796 return true;
2797 int First = 0;
2798 // Shift right until we have the bits at the bottom
2799 while ((Mask & 1) == 0) {
2800 First++;
2801 Mask >>= 1;
2802 }
2803 if ((Mask & (Mask + 1)) != 0)
2804 return false; // Not a consecutive series of bits? Can't be packed.
2805 // Count the bits
2806 int N = 0;
2807 while (Mask & (1 << N))
2808 N++;
2809 if (First < 4) {
2810 if (First + N < 4)
2811 return false;
2812 Folded = 4 - First;
2813 N -= Folded;
2814 First = 4;
2815 }
2816 if (First > 4)
2817 return false; // Can't be packed
2818 if (N >= 1)
2819 IntRegs = N - 1;
2820 return true;
2821}
2822
2823static bool tryARMPackedUnwind(MCStreamer &streamer, WinEH::FrameInfo *info,
2824 uint32_t FuncLength) {
2825 int Step = 0;
2826 bool Homing = false;
2827 bool HasR11 = false;
2828 bool HasChain = false;
2829 bool HasLR = false;
2830 int IntRegs = -1; // r4 - r(4+N)
2831 int FloatRegs = -1; // d8 - d(8+N)
2832 unsigned PF = 0; // Number of extra pushed registers
2833 unsigned StackAdjust = 0;
2834 // Iterate over the prolog and check that all opcodes exactly match
2835 // the canonical order and form.
2836 for (const WinEH::Instruction &Inst : info->Instructions) {
2837 switch (Inst.Operation) {
2838 default:
2839 llvm_unreachable("Unsupported ARM unwind code");
2840 case Win64EH::UOP_Custom:
2841 case Win64EH::UOP_AllocLarge:
2842 case Win64EH::UOP_AllocHuge:
2843 case Win64EH::UOP_WideAllocLarge:
2844 case Win64EH::UOP_WideAllocHuge:
2845 case Win64EH::UOP_SaveFRegD0D15:
2846 case Win64EH::UOP_SaveFRegD16D31:
2847 // Can't be packed
2848 return false;
2849 case Win64EH::UOP_SaveSP:
2850 // Can't be packed; we can't rely on restoring sp from r11 when
2851 // unwinding a packed prologue.
2852 return false;
2853 case Win64EH::UOP_SaveLR:
2854 // Can't be present in a packed prologue
2855 return false;
2856
2857 case Win64EH::UOP_End:
2858 case Win64EH::UOP_EndNop:
2859 case Win64EH::UOP_WideEndNop:
2860 if (Step != 0)
2861 return false;
2862 Step = 1;
2863 break;
2864
2865 case Win64EH::UOP_SaveRegsR4R7LR:
2866 case Win64EH::UOP_WideSaveRegsR4R11LR:
2867 // push {r4-r11,lr}
2868 if (Step != 1 && Step != 2)
2869 return false;
2870 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX
2871 assert(Inst.Offset <= 1); // Lr
2872 IntRegs = Inst.Register - 4;
2873 if (Inst.Register == 11) {
2874 HasR11 = true;
2875 IntRegs--;
2876 }
2877 if (Inst.Offset)
2878 HasLR = true;
2879 Step = 3;
2880 break;
2881
2882 case Win64EH::UOP_SaveRegMask:
2883 if (Step == 1 && Inst.Register == 0x0f) {
2884 // push {r0-r3}
2885 Homing = true;
2886 Step = 2;
2887 break;
2888 }
2889 [[fallthrough]];
2890 case Win64EH::UOP_WideSaveRegMask:
2891 if (Step != 1 && Step != 2)
2892 return false;
2893 // push {r4-r9,r11,lr}
2894 // push {r11,lr}
2895 // push {r1-r5}
2896 if (!parseRegMask(Mask: Inst.Register, HasLR, HasR11, Folded&: PF, IntRegs))
2897 return false;
2898 Step = 3;
2899 break;
2900
2901 case Win64EH::UOP_Nop:
2902 // mov r11, sp
2903 if (Step != 3 || !HasR11 || IntRegs >= 0 || PF > 0)
2904 return false;
2905 HasChain = true;
2906 Step = 4;
2907 break;
2908 case Win64EH::UOP_WideNop:
2909 // add.w r11, sp, #xx
2910 if (Step != 3 || !HasR11 || (IntRegs < 0 && PF == 0))
2911 return false;
2912 HasChain = true;
2913 Step = 4;
2914 break;
2915
2916 case Win64EH::UOP_SaveFRegD8D15:
2917 if (Step != 1 && Step != 2 && Step != 3 && Step != 4)
2918 return false;
2919 assert(Inst.Register >= 8 && Inst.Register <= 15);
2920 if (Inst.Register == 15)
2921 return false; // Can't pack this case, R==7 means no IntRegs
2922 if (IntRegs >= 0)
2923 return false;
2924 FloatRegs = Inst.Register - 8;
2925 Step = 5;
2926 break;
2927
2928 case Win64EH::UOP_AllocSmall:
2929 case Win64EH::UOP_WideAllocMedium:
2930 if (Step != 1 && Step != 2 && Step != 3 && Step != 4 && Step != 5)
2931 return false;
2932 if (PF > 0) // Can't have both folded and explicit stack allocation
2933 return false;
2934 if (Inst.Offset / 4 >= 0x3f4)
2935 return false;
2936 StackAdjust = Inst.Offset / 4;
2937 Step = 6;
2938 break;
2939 }
2940 }
2941 if (HasR11 && !HasChain) {
2942 if (IntRegs + 4 == 10) {
2943 // r11 stored, but not chaining; can be packed if already saving r4-r10
2944 // and we can fit r11 into this range.
2945 IntRegs++;
2946 HasR11 = false;
2947 } else
2948 return false;
2949 }
2950 if (HasChain && !HasLR)
2951 return false;
2952
2953 // Packed uneind info can't express multiple epilogues.
2954 if (info->EpilogMap.size() > 1)
2955 return false;
2956
2957 unsigned EF = 0;
2958 int Ret = 0;
2959 if (info->EpilogMap.size() == 0) {
2960 Ret = 3; // No epilogue
2961 } else {
2962 // As the prologue and epilogue aren't exact mirrors of each other,
2963 // we have to check the epilogue too and see if it matches what we've
2964 // concluded from the prologue.
2965 const WinEH::FrameInfo::Epilog &EpilogInfo =
2966 info->EpilogMap.begin()->second;
2967 if (EpilogInfo.Condition != 0xe) // ARMCC::AL
2968 return false;
2969 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions;
2970 std::optional<int64_t> MaybeDistance = GetOptionalAbsDifference(
2971 Streamer&: streamer, LHS: info->FuncletOrFuncEnd, RHS: info->EpilogMap.begin()->first);
2972 if (!MaybeDistance)
2973 return false;
2974 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance;
2975 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Insns: Epilog);
2976 if (DistanceFromEnd != InstructionBytes)
2977 return false;
2978
2979 bool GotStackAdjust = false;
2980 bool GotFloatRegs = false;
2981 bool GotIntRegs = false;
2982 bool GotHomingRestore = false;
2983 bool GotLRRestore = false;
2984 bool NeedsReturn = false;
2985 bool GotReturn = false;
2986
2987 Step = 6;
2988 for (const WinEH::Instruction &Inst : Epilog) {
2989 switch (Inst.Operation) {
2990 default:
2991 llvm_unreachable("Unsupported ARM unwind code");
2992 case Win64EH::UOP_Custom:
2993 case Win64EH::UOP_AllocLarge:
2994 case Win64EH::UOP_AllocHuge:
2995 case Win64EH::UOP_WideAllocLarge:
2996 case Win64EH::UOP_WideAllocHuge:
2997 case Win64EH::UOP_SaveFRegD0D15:
2998 case Win64EH::UOP_SaveFRegD16D31:
2999 case Win64EH::UOP_SaveSP:
3000 case Win64EH::UOP_Nop:
3001 case Win64EH::UOP_WideNop:
3002 // Can't be packed in an epilogue
3003 return false;
3004
3005 case Win64EH::UOP_AllocSmall:
3006 case Win64EH::UOP_WideAllocMedium:
3007 if (Inst.Offset / 4 >= 0x3f4)
3008 return false;
3009 if (Step == 6) {
3010 if (Homing && FloatRegs < 0 && IntRegs < 0 && StackAdjust == 0 &&
3011 PF == 0 && Inst.Offset == 16) {
3012 GotHomingRestore = true;
3013 Step = 10;
3014 } else {
3015 if (StackAdjust > 0) {
3016 // Got stack adjust in prologue too; must match.
3017 if (StackAdjust != Inst.Offset / 4)
3018 return false;
3019 GotStackAdjust = true;
3020 } else if (PF == Inst.Offset / 4) {
3021 // Folded prologue, non-folded epilogue
3022 StackAdjust = Inst.Offset / 4;
3023 GotStackAdjust = true;
3024 } else {
3025 // StackAdjust == 0 in prologue, mismatch
3026 return false;
3027 }
3028 Step = 7;
3029 }
3030 } else if (Step == 7 || Step == 8 || Step == 9) {
3031 if (!Homing || Inst.Offset != 16)
3032 return false;
3033 GotHomingRestore = true;
3034 Step = 10;
3035 } else
3036 return false;
3037 break;
3038
3039 case Win64EH::UOP_SaveFRegD8D15:
3040 if (Step != 6 && Step != 7)
3041 return false;
3042 assert(Inst.Register >= 8 && Inst.Register <= 15);
3043 if (FloatRegs != (int)(Inst.Register - 8))
3044 return false;
3045 GotFloatRegs = true;
3046 Step = 8;
3047 break;
3048
3049 case Win64EH::UOP_SaveRegsR4R7LR:
3050 case Win64EH::UOP_WideSaveRegsR4R11LR: {
3051 // push {r4-r11,lr}
3052 if (Step != 6 && Step != 7 && Step != 8)
3053 return false;
3054 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX
3055 assert(Inst.Offset <= 1); // Lr
3056 if (Homing && HasLR) {
3057 // If homing and LR is backed up, we can either restore LR here
3058 // and return with Ret == 1 or 2, or return with SaveLR below
3059 if (Inst.Offset) {
3060 GotLRRestore = true;
3061 NeedsReturn = true;
3062 } else {
3063 // Expecting a separate SaveLR below
3064 }
3065 } else {
3066 if (HasLR != (Inst.Offset == 1))
3067 return false;
3068 }
3069 GotLRRestore = Inst.Offset == 1;
3070 if (IntRegs < 0) // This opcode must include r4
3071 return false;
3072 int Expected = IntRegs;
3073 if (HasChain) {
3074 // Can't express r11 here unless IntRegs describe r4-r10
3075 if (IntRegs != 6)
3076 return false;
3077 Expected++;
3078 }
3079 if (Expected != (int)(Inst.Register - 4))
3080 return false;
3081 GotIntRegs = true;
3082 Step = 9;
3083 break;
3084 }
3085
3086 case Win64EH::UOP_SaveRegMask:
3087 case Win64EH::UOP_WideSaveRegMask: {
3088 if (Step != 6 && Step != 7 && Step != 8)
3089 return false;
3090 // push {r4-r9,r11,lr}
3091 // push {r11,lr}
3092 // push {r1-r5}
3093 bool CurHasLR = false, CurHasR11 = false;
3094 int Regs;
3095 if (!parseRegMask(Mask: Inst.Register, HasLR&: CurHasLR, HasR11&: CurHasR11, Folded&: EF, IntRegs&: Regs))
3096 return false;
3097 if (EF > 0) {
3098 if (EF != PF && EF != StackAdjust)
3099 return false;
3100 }
3101 if (Homing && HasLR) {
3102 // If homing and LR is backed up, we can either restore LR here
3103 // and return with Ret == 1 or 2, or return with SaveLR below
3104 if (CurHasLR) {
3105 GotLRRestore = true;
3106 NeedsReturn = true;
3107 } else {
3108 // Expecting a separate SaveLR below
3109 }
3110 } else {
3111 if (CurHasLR != HasLR)
3112 return false;
3113 GotLRRestore = CurHasLR;
3114 }
3115 int Expected = IntRegs;
3116 if (HasChain) {
3117 // If we have chaining, the mask must have included r11.
3118 if (!CurHasR11)
3119 return false;
3120 } else if (Expected == 7) {
3121 // If we don't have chaining, the mask could still include r11,
3122 // expressed as part of IntRegs Instead.
3123 Expected--;
3124 if (!CurHasR11)
3125 return false;
3126 } else {
3127 // Neither HasChain nor r11 included in IntRegs, must not have r11
3128 // here either.
3129 if (CurHasR11)
3130 return false;
3131 }
3132 if (Expected != Regs)
3133 return false;
3134 GotIntRegs = true;
3135 Step = 9;
3136 break;
3137 }
3138
3139 case Win64EH::UOP_SaveLR:
3140 if (Step != 6 && Step != 7 && Step != 8 && Step != 9)
3141 return false;
3142 if (!Homing || Inst.Offset != 20 || GotLRRestore)
3143 return false;
3144 GotLRRestore = true;
3145 GotHomingRestore = true;
3146 Step = 10;
3147 break;
3148
3149 case Win64EH::UOP_EndNop:
3150 case Win64EH::UOP_WideEndNop:
3151 GotReturn = true;
3152 Ret = (Inst.Operation == Win64EH::UOP_EndNop) ? 1 : 2;
3153 [[fallthrough]];
3154 case Win64EH::UOP_End:
3155 if (Step != 6 && Step != 7 && Step != 8 && Step != 9 && Step != 10)
3156 return false;
3157 Step = 11;
3158 break;
3159 }
3160 }
3161
3162 if (Step != 11)
3163 return false;
3164 if (StackAdjust > 0 && !GotStackAdjust && EF == 0)
3165 return false;
3166 if (FloatRegs >= 0 && !GotFloatRegs)
3167 return false;
3168 if (IntRegs >= 0 && !GotIntRegs)
3169 return false;
3170 if (Homing && !GotHomingRestore)
3171 return false;
3172 if (HasLR && !GotLRRestore)
3173 return false;
3174 if (NeedsReturn && !GotReturn)
3175 return false;
3176 }
3177
3178 assert(PF == 0 || EF == 0 ||
3179 StackAdjust == 0); // Can't have adjust in all three
3180 if (PF > 0 || EF > 0) {
3181 StackAdjust = PF > 0 ? (PF - 1) : (EF - 1);
3182 assert(StackAdjust <= 3);
3183 StackAdjust |= 0x3f0;
3184 if (PF > 0)
3185 StackAdjust |= 1 << 2;
3186 if (EF > 0)
3187 StackAdjust |= 1 << 3;
3188 }
3189
3190 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier");
3191 int Flag = info->Fragment ? 0x02 : 0x01;
3192 int H = Homing ? 1 : 0;
3193 int L = HasLR ? 1 : 0;
3194 int C = HasChain ? 1 : 0;
3195 assert(IntRegs < 0 || FloatRegs < 0);
3196 unsigned Reg, R;
3197 if (IntRegs >= 0) {
3198 Reg = IntRegs;
3199 assert(Reg <= 7);
3200 R = 0;
3201 } else if (FloatRegs >= 0) {
3202 Reg = FloatRegs;
3203 assert(Reg < 7);
3204 R = 1;
3205 } else {
3206 // No int or float regs stored (except possibly R11,LR)
3207 Reg = 7;
3208 R = 1;
3209 }
3210 info->PackedInfo |= Flag << 0;
3211 info->PackedInfo |= (FuncLength & 0x7FF) << 2;
3212 info->PackedInfo |= (Ret & 0x3) << 13;
3213 info->PackedInfo |= H << 15;
3214 info->PackedInfo |= Reg << 16;
3215 info->PackedInfo |= R << 19;
3216 info->PackedInfo |= L << 20;
3217 info->PackedInfo |= C << 21;
3218 assert(StackAdjust <= 0x3ff);
3219 info->PackedInfo |= StackAdjust << 22;
3220 return true;
3221}
3222
3223// Populate the .xdata section. The format of .xdata on ARM is documented at
3224// https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling
3225static void ARMEmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info,
3226 bool TryPacked = true) {
3227 // If this UNWIND_INFO already has a symbol, it's already been emitted.
3228 if (info->Symbol)
3229 return;
3230 // If there's no unwind info here (not even a terminating UOP_End), the
3231 // unwind info is considered bogus and skipped. If this was done in
3232 // response to an explicit .seh_handlerdata, the associated trailing
3233 // handler data is left orphaned in the xdata section.
3234 if (info->empty()) {
3235 info->EmitAttempted = true;
3236 return;
3237 }
3238 if (info->EmitAttempted) {
3239 // If we tried to emit unwind info before (due to an explicit
3240 // .seh_handlerdata directive), but skipped it (because there was no
3241 // valid information to emit at the time), and it later got valid unwind
3242 // opcodes, we can't emit it here, because the trailing handler data
3243 // was already emitted elsewhere in the xdata section.
3244 streamer.getContext().reportError(
3245 L: SMLoc(), Msg: "Earlier .seh_handlerdata for " + info->Function->getName() +
3246 " skipped due to no unwind info at the time "
3247 "(.seh_handlerdata too early?), but the function later "
3248 "did get unwind info that can't be emitted");
3249 return;
3250 }
3251
3252 MCContext &context = streamer.getContext();
3253 MCSymbol *Label = context.createTempSymbol();
3254
3255 streamer.emitValueToAlignment(Alignment: Align(4));
3256 streamer.emitLabel(Symbol: Label);
3257 info->Symbol = Label;
3258
3259 if (!info->PrologEnd)
3260 streamer.getContext().reportError(L: SMLoc(), Msg: "Prologue in " +
3261 info->Function->getName() +
3262 " not correctly terminated");
3263
3264 if (info->PrologEnd && !info->Fragment)
3265 checkARMInstructions(Streamer&: streamer, Insns: info->Instructions, Begin: info->Begin,
3266 End: info->PrologEnd, Name: info->Function->getName(),
3267 Type: "prologue");
3268 for (auto &I : info->EpilogMap) {
3269 MCSymbol *EpilogStart = I.first;
3270 auto &Epilog = I.second;
3271 checkARMInstructions(Streamer&: streamer, Insns: Epilog.Instructions, Begin: EpilogStart, End: Epilog.End,
3272 Name: info->Function->getName(), Type: "epilogue");
3273 if (Epilog.Instructions.empty() ||
3274 !isARMTerminator(inst: Epilog.Instructions.back()))
3275 streamer.getContext().reportError(
3276 L: SMLoc(), Msg: "Epilogue in " + info->Function->getName() +
3277 " not correctly terminated");
3278 }
3279
3280 std::optional<int64_t> RawFuncLength;
3281 const MCExpr *FuncLengthExpr = nullptr;
3282 if (!info->FuncletOrFuncEnd) {
3283 report_fatal_error(reason: "FuncletOrFuncEnd not set");
3284 } else {
3285 // As the size of many thumb2 instructions isn't known until later,
3286 // we can't always rely on being able to calculate the absolute
3287 // length of the function here. If we can't calculate it, defer it
3288 // to a relocation.
3289 //
3290 // In such a case, we won't know if the function is too long so that
3291 // the unwind info would need to be split (but this isn't implemented
3292 // anyway).
3293 RawFuncLength =
3294 GetOptionalAbsDifference(Streamer&: streamer, LHS: info->FuncletOrFuncEnd, RHS: info->Begin);
3295 if (!RawFuncLength)
3296 FuncLengthExpr =
3297 GetSubDivExpr(Streamer&: streamer, LHS: info->FuncletOrFuncEnd, RHS: info->Begin, Div: 2);
3298 }
3299 uint32_t FuncLength = 0;
3300 if (RawFuncLength)
3301 FuncLength = (uint32_t)*RawFuncLength / 2;
3302 if (FuncLength > 0x3FFFF)
3303 report_fatal_error(reason: "SEH unwind data splitting not yet implemented");
3304 uint32_t PrologCodeBytes = ARMCountOfUnwindCodes(Insns: info->Instructions);
3305 uint32_t TotalCodeBytes = PrologCodeBytes;
3306
3307 if (!info->HandlesExceptions && RawFuncLength && FuncLength <= 0x7ff &&
3308 TryPacked) {
3309 // No exception handlers; check if the prolog and epilog matches the
3310 // patterns that can be described by the packed format. If we don't
3311 // know the exact function length yet, we can't do this.
3312
3313 // info->Symbol was already set even if we didn't actually write any
3314 // unwind info there. Keep using that as indicator that this unwind
3315 // info has been generated already.
3316
3317 if (tryARMPackedUnwind(streamer, info, FuncLength))
3318 return;
3319 }
3320
3321 int PackedEpilogOffset =
3322 checkARMPackedEpilog(streamer, info, PrologCodeBytes);
3323
3324 // Process epilogs.
3325 MapVector<MCSymbol *, uint32_t> EpilogInfo;
3326 // Epilogs processed so far.
3327 std::vector<MCSymbol *> AddedEpilogs;
3328
3329 bool CanTweakProlog = true;
3330 for (auto &I : info->EpilogMap) {
3331 MCSymbol *EpilogStart = I.first;
3332 auto &EpilogInstrs = I.second.Instructions;
3333 uint32_t CodeBytes = ARMCountOfUnwindCodes(Insns: EpilogInstrs);
3334
3335 MCSymbol *MatchingEpilog =
3336 FindMatchingEpilog(EpilogInstrs, Epilogs: AddedEpilogs, info);
3337 int PrologOffset;
3338 if (MatchingEpilog) {
3339 assert(EpilogInfo.contains(MatchingEpilog) &&
3340 "Duplicate epilog not found");
3341 EpilogInfo[EpilogStart] = EpilogInfo.lookup(Key: MatchingEpilog);
3342 // Clear the unwind codes in the EpilogMap, so that they don't get output
3343 // in the logic below.
3344 EpilogInstrs.clear();
3345 } else if ((PrologOffset = getARMOffsetInProlog(
3346 Prolog: info->Instructions, Epilog: EpilogInstrs, CanTweakProlog)) >= 0) {
3347 if (CanTweakProlog) {
3348 // Replace the regular end opcode of the prolog with the one from the
3349 // epilog.
3350 info->Instructions.front() = EpilogInstrs.back();
3351 // Later epilogs need a strict match for the end opcode.
3352 CanTweakProlog = false;
3353 }
3354 EpilogInfo[EpilogStart] = PrologOffset;
3355 // Clear the unwind codes in the EpilogMap, so that they don't get output
3356 // in the logic below.
3357 EpilogInstrs.clear();
3358 } else {
3359 EpilogInfo[EpilogStart] = TotalCodeBytes;
3360 TotalCodeBytes += CodeBytes;
3361 AddedEpilogs.push_back(x: EpilogStart);
3362 }
3363 }
3364
3365 // Code Words, Epilog count, F, E, X, Vers, Function Length
3366 uint32_t row1 = 0x0;
3367 uint32_t CodeWords = TotalCodeBytes / 4;
3368 uint32_t CodeWordsMod = TotalCodeBytes % 4;
3369 if (CodeWordsMod)
3370 CodeWords++;
3371 uint32_t EpilogCount =
3372 PackedEpilogOffset >= 0 ? PackedEpilogOffset : info->EpilogMap.size();
3373 bool ExtensionWord = EpilogCount > 31 || CodeWords > 15;
3374 if (!ExtensionWord) {
3375 row1 |= (EpilogCount & 0x1F) << 23;
3376 row1 |= (CodeWords & 0x0F) << 28;
3377 }
3378 if (info->HandlesExceptions) // X
3379 row1 |= 1 << 20;
3380 if (PackedEpilogOffset >= 0) // E
3381 row1 |= 1 << 21;
3382 if (info->Fragment) // F
3383 row1 |= 1 << 22;
3384 row1 |= FuncLength & 0x3FFFF;
3385 if (RawFuncLength)
3386 streamer.emitInt32(Value: row1);
3387 else
3388 streamer.emitValue(
3389 Value: MCBinaryExpr::createOr(LHS: FuncLengthExpr,
3390 RHS: MCConstantExpr::create(Value: row1, Ctx&: context), Ctx&: context),
3391 Size: 4);
3392
3393 // Extended Code Words, Extended Epilog Count
3394 if (ExtensionWord) {
3395 // FIXME: We should be able to split unwind info into multiple sections.
3396 if (CodeWords > 0xFF || EpilogCount > 0xFFFF)
3397 report_fatal_error(reason: "SEH unwind data splitting not yet implemented");
3398 uint32_t row2 = 0x0;
3399 row2 |= (CodeWords & 0xFF) << 16;
3400 row2 |= (EpilogCount & 0xFFFF);
3401 streamer.emitInt32(Value: row2);
3402 }
3403
3404 if (PackedEpilogOffset < 0) {
3405 // Epilog Start Index, Epilog Start Offset
3406 for (auto &I : EpilogInfo) {
3407 MCSymbol *EpilogStart = I.first;
3408 uint32_t EpilogIndex = I.second;
3409
3410 std::optional<int64_t> MaybeEpilogOffset =
3411 GetOptionalAbsDifference(Streamer&: streamer, LHS: EpilogStart, RHS: info->Begin);
3412 const MCExpr *OffsetExpr = nullptr;
3413 uint32_t EpilogOffset = 0;
3414 if (MaybeEpilogOffset)
3415 EpilogOffset = *MaybeEpilogOffset / 2;
3416 else
3417 OffsetExpr = GetSubDivExpr(Streamer&: streamer, LHS: EpilogStart, RHS: info->Begin, Div: 2);
3418
3419 assert(info->EpilogMap.contains(EpilogStart));
3420 unsigned Condition = info->EpilogMap[EpilogStart].Condition;
3421 assert(Condition <= 0xf);
3422
3423 uint32_t row3 = EpilogOffset;
3424 row3 |= Condition << 20;
3425 row3 |= (EpilogIndex & 0x3FF) << 24;
3426 if (MaybeEpilogOffset)
3427 streamer.emitInt32(Value: row3);
3428 else
3429 streamer.emitValue(
3430 Value: MCBinaryExpr::createOr(
3431 LHS: OffsetExpr, RHS: MCConstantExpr::create(Value: row3, Ctx&: context), Ctx&: context),
3432 Size: 4);
3433 }
3434 }
3435
3436 // Emit prolog unwind instructions (in reverse order).
3437 uint8_t numInst = info->Instructions.size();
3438 for (uint8_t c = 0; c < numInst; ++c) {
3439 WinEH::Instruction inst = info->Instructions.back();
3440 info->Instructions.pop_back();
3441 ARMEmitUnwindCode(streamer, inst);
3442 }
3443
3444 // Emit epilog unwind instructions
3445 for (auto &I : info->EpilogMap) {
3446 auto &EpilogInstrs = I.second.Instructions;
3447 for (const WinEH::Instruction &inst : EpilogInstrs)
3448 ARMEmitUnwindCode(streamer, inst);
3449 }
3450
3451 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes;
3452 assert(BytesMod >= 0);
3453 for (int i = 0; i < BytesMod; i++)
3454 streamer.emitInt8(Value: 0xFB);
3455
3456 if (info->HandlesExceptions)
3457 streamer.emitValue(
3458 Value: MCSymbolRefExpr::create(Symbol: info->ExceptionHandler,
3459 specifier: MCSymbolRefExpr::VK_COFF_IMGREL32, Ctx&: context),
3460 Size: 4);
3461}
3462
3463static void ARM64EmitRuntimeFunction(MCStreamer &streamer,
3464 const WinEH::FrameInfo *info) {
3465 MCContext &context = streamer.getContext();
3466
3467 streamer.emitValueToAlignment(Alignment: Align(4));
3468 for (const auto &S : info->Segments) {
3469 EmitSymbolRefWithOfs(streamer, Base: info->Begin, Offset: S.Offset);
3470 if (info->PackedInfo)
3471 streamer.emitInt32(Value: info->PackedInfo);
3472 else
3473 streamer.emitValue(
3474 Value: MCSymbolRefExpr::create(Symbol: S.Symbol, specifier: MCSymbolRefExpr::VK_COFF_IMGREL32,
3475 Ctx&: context),
3476 Size: 4);
3477 }
3478}
3479
3480
3481static void ARMEmitRuntimeFunction(MCStreamer &streamer,
3482 const WinEH::FrameInfo *info) {
3483 MCContext &context = streamer.getContext();
3484
3485 streamer.emitValueToAlignment(Alignment: Align(4));
3486 EmitSymbolRefWithOfs(streamer, Base: info->Begin, Other: info->Begin);
3487 if (info->PackedInfo)
3488 streamer.emitInt32(Value: info->PackedInfo);
3489 else
3490 streamer.emitValue(
3491 Value: MCSymbolRefExpr::create(Symbol: info->Symbol, specifier: MCSymbolRefExpr::VK_COFF_IMGREL32,
3492 Ctx&: context),
3493 Size: 4);
3494}
3495
3496void llvm::Win64EH::ARM64UnwindEmitter::Emit(MCStreamer &Streamer) const {
3497 // Emit the unwind info structs first.
3498 for (const auto &CFI : Streamer.getWinFrameInfos()) {
3499 WinEH::FrameInfo *Info = CFI.get();
3500 if (Info->empty())
3501 continue;
3502 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: CFI->TextSection);
3503 Streamer.switchSection(Section: XData);
3504 ARM64EmitUnwindInfo(streamer&: Streamer, info: Info);
3505 }
3506
3507 // Now emit RUNTIME_FUNCTION entries.
3508 for (const auto &CFI : Streamer.getWinFrameInfos()) {
3509 WinEH::FrameInfo *Info = CFI.get();
3510 // ARM64EmitUnwindInfo above clears the info struct, so we can't check
3511 // empty here. But if a Symbol is set, we should create the corresponding
3512 // pdata entry.
3513 if (!Info->Symbol)
3514 continue;
3515 MCSection *PData = Streamer.getAssociatedPDataSection(TextSec: CFI->TextSection);
3516 Streamer.switchSection(Section: PData);
3517 ARM64EmitRuntimeFunction(streamer&: Streamer, info: Info);
3518 }
3519}
3520
3521void llvm::Win64EH::ARM64UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer,
3522 WinEH::FrameInfo *info,
3523 bool HandlerData) const {
3524 // Called if there's an .seh_handlerdata directive before the end of the
3525 // function. This forces writing the xdata record already here - and
3526 // in this case, the function isn't actually ended already, but the xdata
3527 // record needs to know the function length. In these cases, if the funclet
3528 // end hasn't been marked yet, the xdata function length won't cover the
3529 // whole function, only up to this point.
3530 if (!info->FuncletOrFuncEnd) {
3531 Streamer.switchSection(Section: info->TextSection);
3532 info->FuncletOrFuncEnd = Streamer.emitCFILabel();
3533 }
3534 // Switch sections (the static function above is meant to be called from
3535 // here and from Emit().
3536 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: info->TextSection);
3537 Streamer.switchSection(Section: XData);
3538 ARM64EmitUnwindInfo(streamer&: Streamer, info, /* TryPacked = */ !HandlerData);
3539}
3540
3541void llvm::Win64EH::ARMUnwindEmitter::Emit(MCStreamer &Streamer) const {
3542 // Emit the unwind info structs first.
3543 for (const auto &CFI : Streamer.getWinFrameInfos()) {
3544 WinEH::FrameInfo *Info = CFI.get();
3545 if (Info->empty())
3546 continue;
3547 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: CFI->TextSection);
3548 Streamer.switchSection(Section: XData);
3549 ARMEmitUnwindInfo(streamer&: Streamer, info: Info);
3550 }
3551
3552 // Now emit RUNTIME_FUNCTION entries.
3553 for (const auto &CFI : Streamer.getWinFrameInfos()) {
3554 WinEH::FrameInfo *Info = CFI.get();
3555 // ARMEmitUnwindInfo above clears the info struct, so we can't check
3556 // empty here. But if a Symbol is set, we should create the corresponding
3557 // pdata entry.
3558 if (!Info->Symbol)
3559 continue;
3560 MCSection *PData = Streamer.getAssociatedPDataSection(TextSec: CFI->TextSection);
3561 Streamer.switchSection(Section: PData);
3562 ARMEmitRuntimeFunction(streamer&: Streamer, info: Info);
3563 }
3564}
3565
3566void llvm::Win64EH::ARMUnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer,
3567 WinEH::FrameInfo *info,
3568 bool HandlerData) const {
3569 // Called if there's an .seh_handlerdata directive before the end of the
3570 // function. This forces writing the xdata record already here - and
3571 // in this case, the function isn't actually ended already, but the xdata
3572 // record needs to know the function length. In these cases, if the funclet
3573 // end hasn't been marked yet, the xdata function length won't cover the
3574 // whole function, only up to this point.
3575 if (!info->FuncletOrFuncEnd) {
3576 Streamer.switchSection(Section: info->TextSection);
3577 info->FuncletOrFuncEnd = Streamer.emitCFILabel();
3578 }
3579 // Switch sections (the static function above is meant to be called from
3580 // here and from Emit().
3581 MCSection *XData = Streamer.getAssociatedXDataSection(TextSec: info->TextSection);
3582 Streamer.switchSection(Section: XData);
3583 ARMEmitUnwindInfo(streamer&: Streamer, info, /* TryPacked = */ !HandlerData);
3584}
3585