1//===- StackMaps.cpp ------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/CodeGen/StackMaps.h"
10#include "llvm/ADT/DenseMapInfo.h"
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/ADT/Twine.h"
13#include "llvm/CodeGen/AsmPrinter.h"
14#include "llvm/CodeGen/MachineFrameInfo.h"
15#include "llvm/CodeGen/MachineFunction.h"
16#include "llvm/CodeGen/MachineInstr.h"
17#include "llvm/CodeGen/MachineOperand.h"
18#include "llvm/CodeGen/TargetOpcodes.h"
19#include "llvm/CodeGen/TargetRegisterInfo.h"
20#include "llvm/CodeGen/TargetSubtargetInfo.h"
21#include "llvm/IR/DataLayout.h"
22#include "llvm/MC/MCContext.h"
23#include "llvm/MC/MCExpr.h"
24#include "llvm/MC/MCObjectFileInfo.h"
25#include "llvm/MC/MCStreamer.h"
26#include "llvm/Support/CommandLine.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/MathExtras.h"
30#include "llvm/Support/raw_ostream.h"
31#include <algorithm>
32#include <cassert>
33#include <cstdint>
34#include <iterator>
35#include <utility>
36
37using namespace llvm;
38
39#define DEBUG_TYPE "stackmaps"
40
41static cl::opt<int> StackMapVersion(
42 "stackmap-version", cl::init(Val: 3), cl::Hidden,
43 cl::desc("Specify the stackmap encoding version (default = 3)"));
44
45const char *StackMaps::WSMP = "Stack Maps: ";
46
47static uint64_t getConstMetaVal(const MachineInstr &MI, unsigned Idx) {
48 assert(MI.getOperand(Idx).isImm() &&
49 MI.getOperand(Idx).getImm() == StackMaps::ConstantOp);
50 const auto &MO = MI.getOperand(i: Idx + 1);
51 assert(MO.isImm());
52 return MO.getImm();
53}
54
55StackMapOpers::StackMapOpers(const MachineInstr *MI)
56 : MI(MI) {
57 assert(getVarIdx() <= MI->getNumOperands() &&
58 "invalid stackmap definition");
59}
60
61PatchPointOpers::PatchPointOpers(const MachineInstr *MI)
62 : MI(MI), HasDef(MI->getOperand(i: 0).isReg() && MI->getOperand(i: 0).isDef() &&
63 !MI->getOperand(i: 0).isImplicit()) {
64#ifndef NDEBUG
65 unsigned CheckStartIdx = 0, e = MI->getNumOperands();
66 while (CheckStartIdx < e && MI->getOperand(CheckStartIdx).isReg() &&
67 MI->getOperand(CheckStartIdx).isDef() &&
68 !MI->getOperand(CheckStartIdx).isImplicit())
69 ++CheckStartIdx;
70
71 assert(getMetaIdx() == CheckStartIdx &&
72 "Unexpected additional definition in Patchpoint intrinsic.");
73#endif
74}
75
76unsigned PatchPointOpers::getNextScratchIdx(unsigned StartIdx) const {
77 if (!StartIdx)
78 StartIdx = getVarIdx();
79
80 // Find the next scratch register (implicit def and early clobber)
81 unsigned ScratchIdx = StartIdx, e = MI->getNumOperands();
82 while (ScratchIdx < e &&
83 !(MI->getOperand(i: ScratchIdx).isReg() &&
84 MI->getOperand(i: ScratchIdx).isDef() &&
85 MI->getOperand(i: ScratchIdx).isImplicit() &&
86 MI->getOperand(i: ScratchIdx).isEarlyClobber()))
87 ++ScratchIdx;
88
89 assert(ScratchIdx != e && "No scratch register available");
90 return ScratchIdx;
91}
92
93unsigned StatepointOpers::getNumGcMapEntriesIdx() {
94 // Take index of num of allocas and skip all allocas records.
95 unsigned CurIdx = getNumAllocaIdx();
96 unsigned NumAllocas = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
97 CurIdx++;
98 while (NumAllocas--)
99 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
100 return CurIdx + 1; // skip <StackMaps::ConstantOp>
101}
102
103unsigned StatepointOpers::getNumAllocaIdx() {
104 // Take index of num of gc ptrs and skip all gc ptr records.
105 unsigned CurIdx = getNumGCPtrIdx();
106 unsigned NumGCPtrs = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
107 CurIdx++;
108 while (NumGCPtrs--)
109 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
110 return CurIdx + 1; // skip <StackMaps::ConstantOp>
111}
112
113unsigned StatepointOpers::getNumGCPtrIdx() {
114 // Take index of num of deopt args and skip all deopt records.
115 unsigned CurIdx = getNumDeoptArgsIdx();
116 unsigned NumDeoptArgs = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
117 CurIdx++;
118 while (NumDeoptArgs--) {
119 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
120 }
121 return CurIdx + 1; // skip <StackMaps::ConstantOp>
122}
123
124int StatepointOpers::getFirstGCPtrIdx() {
125 unsigned NumGCPtrsIdx = getNumGCPtrIdx();
126 unsigned NumGCPtrs = getConstMetaVal(MI: *MI, Idx: NumGCPtrsIdx - 1);
127 if (NumGCPtrs == 0)
128 return -1;
129 ++NumGCPtrsIdx; // skip <num gc ptrs>
130 assert(NumGCPtrsIdx < MI->getNumOperands());
131 return (int)NumGCPtrsIdx;
132}
133
134unsigned StatepointOpers::getGCPointerMap(
135 SmallVectorImpl<std::pair<unsigned, unsigned>> &GCMap) {
136 unsigned CurIdx = getNumGcMapEntriesIdx();
137 unsigned GCMapSize = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
138 CurIdx++;
139 for (unsigned N = 0; N < GCMapSize; ++N) {
140 unsigned B = MI->getOperand(i: CurIdx++).getImm();
141 unsigned D = MI->getOperand(i: CurIdx++).getImm();
142 GCMap.push_back(Elt: std::make_pair(x&: B, y&: D));
143 }
144
145 return GCMapSize;
146}
147
148bool StatepointOpers::isFoldableReg(Register Reg) const {
149 unsigned FoldableAreaStart = getVarIdx();
150 for (const MachineOperand &MO : MI->uses()) {
151 if (MO.getOperandNo() >= FoldableAreaStart)
152 break;
153 if (MO.isReg() && MO.getReg() == Reg)
154 return false;
155 }
156 return true;
157}
158
159bool StatepointOpers::isFoldableReg(const MachineInstr *MI, Register Reg) {
160 if (MI->getOpcode() != TargetOpcode::STATEPOINT)
161 return false;
162 return StatepointOpers(MI).isFoldableReg(Reg);
163}
164
165StackMaps::StackMaps(AsmPrinter &AP) : AP(AP) {
166 if (StackMapVersion != 3)
167 llvm_unreachable("Unsupported stackmap version!");
168}
169
170unsigned StackMaps::getNextMetaArgIdx(const MachineInstr *MI, unsigned CurIdx) {
171 assert(CurIdx < MI->getNumOperands() && "Bad meta arg index");
172 const auto &MO = MI->getOperand(i: CurIdx);
173 if (MO.isImm()) {
174 switch (MO.getImm()) {
175 default:
176 llvm_unreachable("Unrecognized operand type.");
177 case StackMaps::DirectMemRefOp:
178 CurIdx += 2;
179 break;
180 case StackMaps::IndirectMemRefOp:
181 CurIdx += 3;
182 break;
183 case StackMaps::ConstantOp:
184 ++CurIdx;
185 break;
186 }
187 }
188 ++CurIdx;
189 assert(CurIdx < MI->getNumOperands() && "points past operand list");
190 return CurIdx;
191}
192
193/// Go up the super-register chain until we hit a valid dwarf register number.
194static unsigned getDwarfRegNum(MCRegister Reg, const TargetRegisterInfo *TRI) {
195 int RegNum;
196 for (MCPhysReg SR : TRI->superregs_inclusive(Reg)) {
197 RegNum = TRI->getDwarfRegNum(Reg: SR, isEH: false);
198 if (RegNum >= 0)
199 break;
200 }
201
202 assert(RegNum >= 0 && isUInt<16>(RegNum) && "Invalid Dwarf register number.");
203 return (unsigned)RegNum;
204}
205
206MachineInstr::const_mop_iterator
207StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
208 MachineInstr::const_mop_iterator MOE, LocationVec &Locs,
209 LiveOutVec &LiveOuts) {
210 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo();
211 if (MOI->isImm()) {
212 switch (MOI->getImm()) {
213 default:
214 llvm_unreachable("Unrecognized operand type.");
215 case StackMaps::DirectMemRefOp: {
216 auto &DL = AP.MF->getDataLayout();
217
218 unsigned Size = DL.getPointerSizeInBits();
219 assert((Size % 8) == 0 && "Need pointer size in bytes.");
220 Size /= 8;
221 Register Reg = (++MOI)->getReg();
222 int64_t Imm = (++MOI)->getImm();
223 Locs.emplace_back(Args: StackMaps::Location::Direct, Args&: Size,
224 Args: getDwarfRegNum(Reg, TRI), Args&: Imm);
225 break;
226 }
227 case StackMaps::IndirectMemRefOp: {
228 int64_t Size = (++MOI)->getImm();
229 assert(Size > 0 && "Need a valid size for indirect memory locations.");
230 Register Reg = (++MOI)->getReg();
231 int64_t Imm = (++MOI)->getImm();
232 Locs.emplace_back(Args: StackMaps::Location::Indirect, Args&: Size,
233 Args: getDwarfRegNum(Reg, TRI), Args&: Imm);
234 break;
235 }
236 case StackMaps::ConstantOp: {
237 ++MOI;
238 assert(MOI->isImm() && "Expected constant operand.");
239 int64_t Imm = MOI->getImm();
240 if (isInt<32>(x: Imm)) {
241 Locs.emplace_back(Args: Location::Constant, Args: sizeof(int64_t), Args: 0, Args&: Imm);
242 } else {
243 // ConstPool is intentionally a MapVector of 'uint64_t's (as
244 // opposed to 'int64_t's). We should never be in a situation
245 // where we have to insert the empty key into a map, and for a
246 // DenseMap<uint64_t, T> this is (uint64_t)-1. It can be and is
247 // represented using 32 bit integers.
248 auto Result = ConstPool.insert(KV: std::make_pair(x&: Imm, y&: Imm));
249 Locs.emplace_back(Args: Location::ConstantIndex, Args: sizeof(int64_t), Args: 0,
250 Args: Result.first - ConstPool.begin());
251 }
252 break;
253 }
254 }
255 return ++MOI;
256 }
257
258 // The physical register number will ultimately be encoded as a DWARF regno.
259 // The stack map also records the size of a spill slot that can hold the
260 // register content. (The runtime can track the actual size of the data type
261 // if it needs to.)
262 if (MOI->isReg()) {
263 // Skip implicit registers (this includes our scratch registers)
264 if (MOI->isImplicit())
265 return ++MOI;
266
267 assert(MOI->getReg().isPhysical() &&
268 "Virtreg operands should have been rewritten before now.");
269 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg: MOI->getReg());
270 assert(!MOI->getSubReg() && "Physical subreg still around.");
271
272 unsigned Offset = 0;
273 unsigned DwarfRegNum = getDwarfRegNum(Reg: MOI->getReg(), TRI);
274 MCRegister LLVMRegNum = *TRI->getLLVMRegNum(RegNum: DwarfRegNum, isEH: false);
275 unsigned SubRegIdx = TRI->getSubRegIndex(RegNo: LLVMRegNum, SubRegNo: MOI->getReg());
276 if (SubRegIdx)
277 Offset = TRI->getSubRegIdxOffset(Idx: SubRegIdx);
278
279 Locs.emplace_back(Args: Location::Register, Args: TRI->getSpillSize(RC: *RC),
280 Args&: DwarfRegNum, Args&: Offset);
281 return ++MOI;
282 }
283
284 if (MOI->isRegLiveOut())
285 LiveOuts = parseRegisterLiveOutMask(Mask: MOI->getRegLiveOut());
286
287 return ++MOI;
288}
289
290void StackMaps::print(raw_ostream &OS) {
291 const TargetRegisterInfo *TRI =
292 AP.MF ? AP.MF->getSubtarget().getRegisterInfo() : nullptr;
293 OS << WSMP << "callsites:\n";
294 for (const auto &CSI : CSInfos) {
295 const LocationVec &CSLocs = CSI.Locations;
296 const LiveOutVec &LiveOuts = CSI.LiveOuts;
297
298 OS << WSMP << "callsite " << CSI.ID << "\n";
299 OS << WSMP << " has " << CSLocs.size() << " locations\n";
300
301 unsigned Idx = 0;
302 for (const auto &Loc : CSLocs) {
303 OS << WSMP << "\t\tLoc " << Idx << ": ";
304 switch (Loc.Type) {
305 case Location::Unprocessed:
306 OS << "<Unprocessed operand>";
307 break;
308 case Location::Register:
309 OS << "Register ";
310 if (TRI)
311 OS << printReg(Reg: Loc.Reg, TRI);
312 else
313 OS << Loc.Reg;
314 break;
315 case Location::Direct:
316 OS << "Direct ";
317 if (TRI)
318 OS << printReg(Reg: Loc.Reg, TRI);
319 else
320 OS << Loc.Reg;
321 if (Loc.Offset)
322 OS << " + " << Loc.Offset;
323 break;
324 case Location::Indirect:
325 OS << "Indirect ";
326 if (TRI)
327 OS << printReg(Reg: Loc.Reg, TRI);
328 else
329 OS << Loc.Reg;
330 OS << "+" << Loc.Offset;
331 break;
332 case Location::Constant:
333 OS << "Constant " << Loc.Offset;
334 break;
335 case Location::ConstantIndex:
336 OS << "Constant Index " << Loc.Offset;
337 break;
338 }
339 OS << "\t[encoding: .byte " << Loc.Type << ", .byte 0"
340 << ", .short " << Loc.Size << ", .short " << Loc.Reg << ", .short 0"
341 << ", .int " << Loc.Offset << "]\n";
342 Idx++;
343 }
344
345 OS << WSMP << "\thas " << LiveOuts.size() << " live-out registers\n";
346
347 Idx = 0;
348 for (const auto &LO : LiveOuts) {
349 OS << WSMP << "\t\tLO " << Idx << ": ";
350 if (TRI)
351 OS << printReg(Reg: LO.Reg, TRI);
352 else
353 OS << LO.Reg;
354 OS << "\t[encoding: .short " << LO.DwarfRegNum << ", .byte 0, .byte "
355 << LO.Size << "]\n";
356 Idx++;
357 }
358 }
359}
360
361/// Create a live-out register record for the given register Reg.
362StackMaps::LiveOutReg
363StackMaps::createLiveOutReg(unsigned Reg, const TargetRegisterInfo *TRI) const {
364 unsigned DwarfRegNum = getDwarfRegNum(Reg, TRI);
365 unsigned Size = TRI->getSpillSize(RC: *TRI->getMinimalPhysRegClass(Reg));
366 return LiveOutReg(Reg, DwarfRegNum, Size);
367}
368
369/// Parse the register live-out mask and return a vector of live-out registers
370/// that need to be recorded in the stackmap.
371StackMaps::LiveOutVec
372StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const {
373 assert(Mask && "No register mask specified");
374 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo();
375 LiveOutVec LiveOuts;
376
377 // Create a LiveOutReg for each bit that is set in the register mask.
378 for (unsigned Reg = 0, NumRegs = TRI->getNumRegs(); Reg != NumRegs; ++Reg)
379 if ((Mask[Reg / 32] >> (Reg % 32)) & 1)
380 LiveOuts.push_back(Elt: createLiveOutReg(Reg, TRI));
381
382 // We don't need to keep track of a register if its super-register is already
383 // in the list. Merge entries that refer to the same dwarf register and use
384 // the maximum size that needs to be spilled.
385
386 llvm::sort(C&: LiveOuts, Comp: [](const LiveOutReg &LHS, const LiveOutReg &RHS) {
387 // Only sort by the dwarf register number.
388 return LHS.DwarfRegNum < RHS.DwarfRegNum;
389 });
390
391 for (auto I = LiveOuts.begin(), E = LiveOuts.end(); I != E; ++I) {
392 for (auto *II = std::next(x: I); II != E; ++II) {
393 if (I->DwarfRegNum != II->DwarfRegNum) {
394 // Skip all the now invalid entries.
395 I = --II;
396 break;
397 }
398 I->Size = std::max(a: I->Size, b: II->Size);
399 if (I->Reg && TRI->isSuperRegister(RegA: I->Reg, RegB: II->Reg))
400 I->Reg = II->Reg;
401 II->Reg = 0; // mark for deletion.
402 }
403 }
404
405 llvm::erase_if(C&: LiveOuts, P: [](const LiveOutReg &LO) { return LO.Reg == 0; });
406
407 return LiveOuts;
408}
409
410// See statepoint MI format description in StatepointOpers' class comment
411// in include/llvm/CodeGen/StackMaps.h
412void StackMaps::parseStatepointOpers(const MachineInstr &MI,
413 MachineInstr::const_mop_iterator MOI,
414 MachineInstr::const_mop_iterator MOE,
415 LocationVec &Locations,
416 LiveOutVec &LiveOuts) {
417 LLVM_DEBUG(dbgs() << "record statepoint : " << MI << "\n");
418 StatepointOpers SO(&MI);
419 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts); // CC
420 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts); // Flags
421 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts); // Num Deopts
422
423 // Record Deopt Args.
424 unsigned NumDeoptArgs = Locations.back().Offset;
425 assert(Locations.back().Type == Location::Constant);
426 assert(NumDeoptArgs == SO.getNumDeoptArgs());
427
428 while (NumDeoptArgs--)
429 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts);
430
431 // Record gc base/derived pairs
432 assert(MOI->isImm() && MOI->getImm() == StackMaps::ConstantOp);
433 ++MOI;
434 assert(MOI->isImm());
435 unsigned NumGCPointers = MOI->getImm();
436 ++MOI;
437 if (NumGCPointers) {
438 // Map logical index of GC ptr to MI operand index.
439 SmallVector<unsigned, 8> GCPtrIndices;
440 unsigned GCPtrIdx = (unsigned)SO.getFirstGCPtrIdx();
441 assert((int)GCPtrIdx != -1);
442 assert(MOI - MI.operands_begin() == GCPtrIdx + 0LL);
443 while (NumGCPointers--) {
444 GCPtrIndices.push_back(Elt: GCPtrIdx);
445 GCPtrIdx = StackMaps::getNextMetaArgIdx(MI: &MI, CurIdx: GCPtrIdx);
446 }
447
448 SmallVector<std::pair<unsigned, unsigned>, 8> GCPairs;
449 unsigned NumGCPairs = SO.getGCPointerMap(GCMap&: GCPairs);
450 (void)NumGCPairs;
451 LLVM_DEBUG(dbgs() << "NumGCPairs = " << NumGCPairs << "\n");
452
453 auto MOB = MI.operands_begin();
454 for (auto &P : GCPairs) {
455 assert(P.first < GCPtrIndices.size() && "base pointer index not found");
456 assert(P.second < GCPtrIndices.size() &&
457 "derived pointer index not found");
458 unsigned BaseIdx = GCPtrIndices[P.first];
459 unsigned DerivedIdx = GCPtrIndices[P.second];
460 LLVM_DEBUG(dbgs() << "Base : " << BaseIdx << " Derived : " << DerivedIdx
461 << "\n");
462 (void)parseOperand(MOI: MOB + BaseIdx, MOE, Locs&: Locations, LiveOuts);
463 (void)parseOperand(MOI: MOB + DerivedIdx, MOE, Locs&: Locations, LiveOuts);
464 }
465
466 MOI = MOB + GCPtrIdx;
467 }
468
469 // Record gc allocas
470 assert(MOI < MOE);
471 assert(MOI->isImm() && MOI->getImm() == StackMaps::ConstantOp);
472 ++MOI;
473 unsigned NumAllocas = MOI->getImm();
474 ++MOI;
475 while (NumAllocas--) {
476 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts);
477 assert(MOI < MOE);
478 }
479}
480
481void StackMaps::recordStackMapOpers(const MCSymbol &MILabel,
482 const MachineInstr &MI, uint64_t ID,
483 MachineInstr::const_mop_iterator MOI,
484 MachineInstr::const_mop_iterator MOE,
485 bool recordResult) {
486 MCContext &OutContext = AP.OutStreamer->getContext();
487
488 LocationVec Locations;
489 LiveOutVec LiveOuts;
490
491 if (recordResult) {
492 assert(PatchPointOpers(&MI).hasDef() && "Stackmap has no return value.");
493 parseOperand(MOI: MI.operands_begin(), MOE: std::next(x: MI.operands_begin()), Locs&: Locations,
494 LiveOuts);
495 }
496
497 // Parse operands.
498 if (MI.getOpcode() == TargetOpcode::STATEPOINT)
499 parseStatepointOpers(MI, MOI, MOE, Locations, LiveOuts);
500 else
501 while (MOI != MOE)
502 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts);
503
504 // Create an expression to calculate the offset of the callsite from function
505 // entry.
506 const MCExpr *CSOffsetExpr = MCBinaryExpr::createSub(
507 LHS: MCSymbolRefExpr::create(Symbol: &MILabel, Ctx&: OutContext),
508 RHS: MCSymbolRefExpr::create(Symbol: AP.CurrentFnSymForSize, Ctx&: OutContext), Ctx&: OutContext);
509
510 CSInfos.emplace_back(args&: CSOffsetExpr, args&: ID, args: std::move(Locations),
511 args: std::move(LiveOuts));
512
513 // Record the stack size of the current function and update callsite count.
514 const MachineFrameInfo &MFI = AP.MF->getFrameInfo();
515 const TargetRegisterInfo *RegInfo = AP.MF->getSubtarget().getRegisterInfo();
516 bool HasDynamicFrameSize =
517 MFI.hasVarSizedObjects() || RegInfo->hasStackRealignment(MF: *(AP.MF));
518 uint64_t FrameSize = HasDynamicFrameSize ? UINT64_MAX : MFI.getStackSize();
519
520 auto [CurrentIt, Inserted] = FnInfos.try_emplace(Key: AP.CurrentFnSym, Args&: FrameSize);
521 if (!Inserted)
522 CurrentIt->second.RecordCount++;
523}
524
525void StackMaps::recordStackMap(const MCSymbol &L, const MachineInstr &MI) {
526 assert(MI.getOpcode() == TargetOpcode::STACKMAP && "expected stackmap");
527
528 StackMapOpers opers(&MI);
529 const int64_t ID = MI.getOperand(i: PatchPointOpers::IDPos).getImm();
530 recordStackMapOpers(MILabel: L, MI, ID, MOI: std::next(x: MI.operands_begin(),
531 n: opers.getVarIdx()),
532 MOE: MI.operands_end());
533}
534
535void StackMaps::recordPatchPoint(const MCSymbol &L, const MachineInstr &MI) {
536 assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "expected patchpoint");
537
538 PatchPointOpers opers(&MI);
539 const int64_t ID = opers.getID();
540 auto MOI = std::next(x: MI.operands_begin(), n: opers.getStackMapStartIdx());
541 recordStackMapOpers(MILabel: L, MI, ID, MOI, MOE: MI.operands_end(),
542 recordResult: opers.isAnyReg() && opers.hasDef());
543
544#ifndef NDEBUG
545 // verify anyregcc
546 auto &Locations = CSInfos.back().Locations;
547 if (opers.isAnyReg()) {
548 unsigned NArgs = opers.getNumCallArgs();
549 for (unsigned i = 0, e = (opers.hasDef() ? NArgs + 1 : NArgs); i != e; ++i)
550 assert(Locations[i].Type == Location::Register &&
551 "anyreg arg must be in reg.");
552 }
553#endif
554}
555
556void StackMaps::recordStatepoint(const MCSymbol &L, const MachineInstr &MI) {
557 assert(MI.getOpcode() == TargetOpcode::STATEPOINT && "expected statepoint");
558
559 StatepointOpers opers(&MI);
560 const unsigned StartIdx = opers.getVarIdx();
561 recordStackMapOpers(MILabel: L, MI, ID: opers.getID(), MOI: MI.operands_begin() + StartIdx,
562 MOE: MI.operands_end(), recordResult: false);
563}
564
565/// Emit the stackmap header.
566///
567/// Header {
568/// uint8 : Stack Map Version (currently 3)
569/// uint8 : Reserved (expected to be 0)
570/// uint16 : Reserved (expected to be 0)
571/// }
572/// uint32 : NumFunctions
573/// uint32 : NumConstants
574/// uint32 : NumRecords
575void StackMaps::emitStackmapHeader(MCStreamer &OS) {
576 // Header.
577 OS.emitIntValue(Value: StackMapVersion, Size: 1); // Version.
578 OS.emitIntValue(Value: 0, Size: 1); // Reserved.
579 OS.emitInt16(Value: 0); // Reserved.
580
581 // Num functions.
582 LLVM_DEBUG(dbgs() << WSMP << "#functions = " << FnInfos.size() << '\n');
583 OS.emitInt32(Value: FnInfos.size());
584 // Num constants.
585 LLVM_DEBUG(dbgs() << WSMP << "#constants = " << ConstPool.size() << '\n');
586 OS.emitInt32(Value: ConstPool.size());
587 // Num callsites.
588 LLVM_DEBUG(dbgs() << WSMP << "#callsites = " << CSInfos.size() << '\n');
589 OS.emitInt32(Value: CSInfos.size());
590}
591
592/// Emit the function frame record for each function.
593///
594/// StkSizeRecord[NumFunctions] {
595/// uint64 : Function Address
596/// uint64 : Stack Size
597/// uint64 : Record Count
598/// }
599void StackMaps::emitFunctionFrameRecords(MCStreamer &OS) {
600 // Function Frame records.
601 LLVM_DEBUG(dbgs() << WSMP << "functions:\n");
602 for (auto const &FR : FnInfos) {
603 LLVM_DEBUG(dbgs() << WSMP << "function addr: " << FR.first
604 << " frame size: " << FR.second.StackSize
605 << " callsite count: " << FR.second.RecordCount << '\n');
606 OS.emitSymbolValue(Sym: FR.first, Size: 8);
607 OS.emitIntValue(Value: FR.second.StackSize, Size: 8);
608 OS.emitIntValue(Value: FR.second.RecordCount, Size: 8);
609 }
610}
611
612/// Emit the constant pool.
613///
614/// int64 : Constants[NumConstants]
615void StackMaps::emitConstantPoolEntries(MCStreamer &OS) {
616 // Constant pool entries.
617 LLVM_DEBUG(dbgs() << WSMP << "constants:\n");
618 for (const auto &ConstEntry : ConstPool) {
619 LLVM_DEBUG(dbgs() << WSMP << ConstEntry.second << '\n');
620 OS.emitIntValue(Value: ConstEntry.second, Size: 8);
621 }
622}
623
624/// Emit the callsite info for each callsite.
625///
626/// StkMapRecord[NumRecords] {
627/// uint64 : PatchPoint ID
628/// uint32 : Instruction Offset
629/// uint16 : Reserved (record flags)
630/// uint16 : NumLocations
631/// Location[NumLocations] {
632/// uint8 : Register | Direct | Indirect | Constant | ConstantIndex
633/// uint8 : Size in Bytes
634/// uint16 : Dwarf RegNum
635/// int32 : Offset
636/// }
637/// uint16 : Padding
638/// uint16 : NumLiveOuts
639/// LiveOuts[NumLiveOuts] {
640/// uint16 : Dwarf RegNum
641/// uint8 : Reserved
642/// uint8 : Size in Bytes
643/// }
644/// uint32 : Padding (only if required to align to 8 byte)
645/// }
646///
647/// Location Encoding, Type, Value:
648/// 0x1, Register, Reg (value in register)
649/// 0x2, Direct, Reg + Offset (frame index)
650/// 0x3, Indirect, [Reg + Offset] (spilled value)
651/// 0x4, Constant, Offset (small constant)
652/// 0x5, ConstIndex, Constants[Offset] (large constant)
653void StackMaps::emitCallsiteEntries(MCStreamer &OS) {
654 LLVM_DEBUG(print(dbgs()));
655 // Callsite entries.
656 for (const auto &CSI : CSInfos) {
657 const LocationVec &CSLocs = CSI.Locations;
658 const LiveOutVec &LiveOuts = CSI.LiveOuts;
659
660 // Verify stack map entry. It's better to communicate a problem to the
661 // runtime than crash in case of in-process compilation. Currently, we do
662 // simple overflow checks, but we may eventually communicate other
663 // compilation errors this way.
664 if (CSLocs.size() > UINT16_MAX || LiveOuts.size() > UINT16_MAX) {
665 OS.emitIntValue(UINT64_MAX, Size: 8); // Invalid ID.
666 OS.emitValue(Value: CSI.CSOffsetExpr, Size: 4);
667 OS.emitInt16(Value: 0); // Reserved.
668 OS.emitInt16(Value: 0); // 0 locations.
669 OS.emitInt16(Value: 0); // padding.
670 OS.emitInt16(Value: 0); // 0 live-out registers.
671 OS.emitInt32(Value: 0); // padding.
672 continue;
673 }
674
675 OS.emitIntValue(Value: CSI.ID, Size: 8);
676 OS.emitValue(Value: CSI.CSOffsetExpr, Size: 4);
677
678 // Reserved for flags.
679 OS.emitInt16(Value: 0);
680 OS.emitInt16(Value: CSLocs.size());
681
682 for (const auto &Loc : CSLocs) {
683 OS.emitIntValue(Value: Loc.Type, Size: 1);
684 OS.emitIntValue(Value: 0, Size: 1); // Reserved
685 OS.emitInt16(Value: Loc.Size);
686 OS.emitInt16(Value: Loc.Reg);
687 OS.emitInt16(Value: 0); // Reserved
688 OS.emitInt32(Value: Loc.Offset);
689 }
690
691 // Emit alignment to 8 byte.
692 OS.emitValueToAlignment(Alignment: Align(8));
693
694 // Num live-out registers and padding to align to 4 byte.
695 OS.emitInt16(Value: 0);
696 OS.emitInt16(Value: LiveOuts.size());
697
698 for (const auto &LO : LiveOuts) {
699 OS.emitInt16(Value: LO.DwarfRegNum);
700 OS.emitIntValue(Value: 0, Size: 1);
701 OS.emitIntValue(Value: LO.Size, Size: 1);
702 }
703 // Emit alignment to 8 byte.
704 OS.emitValueToAlignment(Alignment: Align(8));
705 }
706}
707
708/// Serialize the stackmap data.
709void StackMaps::serializeToStackMapSection() {
710 (void)WSMP;
711 // Bail out if there's no stack map data.
712 assert((!CSInfos.empty() || ConstPool.empty()) &&
713 "Expected empty constant pool too!");
714 assert((!CSInfos.empty() || FnInfos.empty()) &&
715 "Expected empty function record too!");
716 if (CSInfos.empty())
717 return;
718
719 MCContext &OutContext = AP.OutStreamer->getContext();
720 MCStreamer &OS = *AP.OutStreamer;
721
722 // Create the section.
723 MCSection *StackMapSection =
724 OutContext.getObjectFileInfo()->getStackMapSection();
725 OS.switchSection(Section: StackMapSection);
726
727 // Emit a dummy symbol to force section inclusion.
728 OS.emitLabel(Symbol: OutContext.getOrCreateSymbol(Name: Twine("__LLVM_StackMaps")));
729
730 // Serialize data.
731 LLVM_DEBUG(dbgs() << "********** Stack Map Output **********\n");
732 emitStackmapHeader(OS);
733 emitFunctionFrameRecords(OS);
734 emitConstantPoolEntries(OS);
735 emitCallsiteEntries(OS);
736 OS.addBlankLine();
737
738 // Clean up.
739 CSInfos.clear();
740 ConstPool.clear();
741}
742