1//===- StackMaps.cpp ------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/CodeGen/StackMaps.h"
10#include "llvm/ADT/STLExtras.h"
11#include "llvm/ADT/Twine.h"
12#include "llvm/CodeGen/AsmPrinter.h"
13#include "llvm/CodeGen/MachineFrameInfo.h"
14#include "llvm/CodeGen/MachineFunction.h"
15#include "llvm/CodeGen/MachineInstr.h"
16#include "llvm/CodeGen/MachineOperand.h"
17#include "llvm/CodeGen/TargetOpcodes.h"
18#include "llvm/CodeGen/TargetRegisterInfo.h"
19#include "llvm/CodeGen/TargetSubtargetInfo.h"
20#include "llvm/IR/DataLayout.h"
21#include "llvm/MC/MCContext.h"
22#include "llvm/MC/MCExpr.h"
23#include "llvm/MC/MCObjectFileInfo.h"
24#include "llvm/MC/MCStreamer.h"
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/Debug.h"
27#include "llvm/Support/ErrorHandling.h"
28#include "llvm/Support/MathExtras.h"
29#include "llvm/Support/raw_ostream.h"
30#include <algorithm>
31#include <cassert>
32#include <cstdint>
33#include <iterator>
34#include <utility>
35
36using namespace llvm;
37
38#define DEBUG_TYPE "stackmaps"
39
40static cl::opt<int> StackMapVersion(
41 "stackmap-version", cl::init(Val: 3), cl::Hidden,
42 cl::desc("Specify the stackmap encoding version (default = 3)"));
43
44const char *StackMaps::WSMP = "Stack Maps: ";
45
46static uint64_t getConstMetaVal(const MachineInstr &MI, unsigned Idx) {
47 assert(MI.getOperand(Idx).isImm() &&
48 MI.getOperand(Idx).getImm() == StackMaps::ConstantOp);
49 const auto &MO = MI.getOperand(i: Idx + 1);
50 assert(MO.isImm());
51 return MO.getImm();
52}
53
54StackMapOpers::StackMapOpers(const MachineInstr *MI)
55 : MI(MI) {
56 assert(getVarIdx() <= MI->getNumOperands() &&
57 "invalid stackmap definition");
58}
59
60PatchPointOpers::PatchPointOpers(const MachineInstr *MI)
61 : MI(MI), HasDef(MI->getOperand(i: 0).isReg() && MI->getOperand(i: 0).isDef() &&
62 !MI->getOperand(i: 0).isImplicit()) {
63#ifndef NDEBUG
64 unsigned CheckStartIdx = 0, e = MI->getNumOperands();
65 while (CheckStartIdx < e && MI->getOperand(CheckStartIdx).isReg() &&
66 MI->getOperand(CheckStartIdx).isDef() &&
67 !MI->getOperand(CheckStartIdx).isImplicit())
68 ++CheckStartIdx;
69
70 assert(getMetaIdx() == CheckStartIdx &&
71 "Unexpected additional definition in Patchpoint intrinsic.");
72#endif
73}
74
75unsigned PatchPointOpers::getNextScratchIdx(unsigned StartIdx) const {
76 if (!StartIdx)
77 StartIdx = getVarIdx();
78
79 // Find the next scratch register (implicit def and early clobber)
80 unsigned ScratchIdx = StartIdx, e = MI->getNumOperands();
81 while (ScratchIdx < e &&
82 !(MI->getOperand(i: ScratchIdx).isReg() &&
83 MI->getOperand(i: ScratchIdx).isDef() &&
84 MI->getOperand(i: ScratchIdx).isImplicit() &&
85 MI->getOperand(i: ScratchIdx).isEarlyClobber()))
86 ++ScratchIdx;
87
88 assert(ScratchIdx != e && "No scratch register available");
89 return ScratchIdx;
90}
91
92unsigned StatepointOpers::getNumGcMapEntriesIdx() {
93 // Take index of num of allocas and skip all allocas records.
94 unsigned CurIdx = getNumAllocaIdx();
95 unsigned NumAllocas = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
96 CurIdx++;
97 while (NumAllocas--)
98 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
99 return CurIdx + 1; // skip <StackMaps::ConstantOp>
100}
101
102unsigned StatepointOpers::getNumAllocaIdx() {
103 // Take index of num of gc ptrs and skip all gc ptr records.
104 unsigned CurIdx = getNumGCPtrIdx();
105 unsigned NumGCPtrs = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
106 CurIdx++;
107 while (NumGCPtrs--)
108 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
109 return CurIdx + 1; // skip <StackMaps::ConstantOp>
110}
111
112unsigned StatepointOpers::getNumGCPtrIdx() {
113 // Take index of num of deopt args and skip all deopt records.
114 unsigned CurIdx = getNumDeoptArgsIdx();
115 unsigned NumDeoptArgs = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
116 CurIdx++;
117 while (NumDeoptArgs--) {
118 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
119 }
120 return CurIdx + 1; // skip <StackMaps::ConstantOp>
121}
122
123int StatepointOpers::getFirstGCPtrIdx() {
124 unsigned NumGCPtrsIdx = getNumGCPtrIdx();
125 unsigned NumGCPtrs = getConstMetaVal(MI: *MI, Idx: NumGCPtrsIdx - 1);
126 if (NumGCPtrs == 0)
127 return -1;
128 ++NumGCPtrsIdx; // skip <num gc ptrs>
129 assert(NumGCPtrsIdx < MI->getNumOperands());
130 return (int)NumGCPtrsIdx;
131}
132
133unsigned StatepointOpers::getGCPointerMap(
134 SmallVectorImpl<std::pair<unsigned, unsigned>> &GCMap) {
135 unsigned CurIdx = getNumGcMapEntriesIdx();
136 unsigned GCMapSize = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
137 CurIdx++;
138 for (unsigned N = 0; N < GCMapSize; ++N) {
139 unsigned B = MI->getOperand(i: CurIdx++).getImm();
140 unsigned D = MI->getOperand(i: CurIdx++).getImm();
141 GCMap.push_back(Elt: std::make_pair(x&: B, y&: D));
142 }
143
144 return GCMapSize;
145}
146
147bool StatepointOpers::isFoldableReg(Register Reg) const {
148 unsigned FoldableAreaStart = getVarIdx();
149 for (const MachineOperand &MO : MI->uses()) {
150 if (MO.getOperandNo() >= FoldableAreaStart)
151 break;
152 if (MO.isReg() && MO.getReg() == Reg)
153 return false;
154 }
155 return true;
156}
157
158bool StatepointOpers::isFoldableReg(const MachineInstr *MI, Register Reg) {
159 if (MI->getOpcode() != TargetOpcode::STATEPOINT)
160 return false;
161 return StatepointOpers(MI).isFoldableReg(Reg);
162}
163
164StackMaps::StackMaps(AsmPrinter &AP) : AP(AP) {
165 if (StackMapVersion != 3)
166 llvm_unreachable("Unsupported stackmap version!");
167}
168
169unsigned StackMaps::getNextMetaArgIdx(const MachineInstr *MI, unsigned CurIdx) {
170 assert(CurIdx < MI->getNumOperands() && "Bad meta arg index");
171 const auto &MO = MI->getOperand(i: CurIdx);
172 if (MO.isImm()) {
173 switch (MO.getImm()) {
174 default:
175 llvm_unreachable("Unrecognized operand type.");
176 case StackMaps::DirectMemRefOp:
177 CurIdx += 2;
178 break;
179 case StackMaps::IndirectMemRefOp:
180 CurIdx += 3;
181 break;
182 case StackMaps::ConstantOp:
183 ++CurIdx;
184 break;
185 }
186 }
187 ++CurIdx;
188 assert(CurIdx < MI->getNumOperands() && "points past operand list");
189 return CurIdx;
190}
191
192/// Go up the super-register chain until we hit a valid dwarf register number.
193static unsigned getDwarfRegNum(MCRegister Reg, const TargetRegisterInfo *TRI) {
194 int RegNum;
195 for (MCPhysReg SR : TRI->superregs_inclusive(Reg)) {
196 RegNum = TRI->getDwarfRegNum(Reg: SR, isEH: false);
197 if (RegNum >= 0)
198 break;
199 }
200
201 assert(RegNum >= 0 && isUInt<16>(RegNum) && "Invalid Dwarf register number.");
202 return (unsigned)RegNum;
203}
204
205MachineInstr::const_mop_iterator
206StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
207 MachineInstr::const_mop_iterator MOE, LocationVec &Locs,
208 LiveOutVec &LiveOuts) {
209 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo();
210 if (MOI->isImm()) {
211 switch (MOI->getImm()) {
212 default:
213 llvm_unreachable("Unrecognized operand type.");
214 case StackMaps::DirectMemRefOp: {
215 auto &DL = AP.MF->getDataLayout();
216
217 unsigned Size = DL.getPointerSizeInBits();
218 assert((Size % 8) == 0 && "Need pointer size in bytes.");
219 Size /= 8;
220 Register Reg = (++MOI)->getReg();
221 int64_t Imm = (++MOI)->getImm();
222 Locs.emplace_back(Args: StackMaps::Location::Direct, Args&: Size,
223 Args: getDwarfRegNum(Reg, TRI), Args&: Imm);
224 break;
225 }
226 case StackMaps::IndirectMemRefOp: {
227 int64_t Size = (++MOI)->getImm();
228 assert(Size > 0 && "Need a valid size for indirect memory locations.");
229 Register Reg = (++MOI)->getReg();
230 int64_t Imm = (++MOI)->getImm();
231 Locs.emplace_back(Args: StackMaps::Location::Indirect, Args&: Size,
232 Args: getDwarfRegNum(Reg, TRI), Args&: Imm);
233 break;
234 }
235 case StackMaps::ConstantOp: {
236 ++MOI;
237 assert(MOI->isImm() && "Expected constant operand.");
238 int64_t Imm = MOI->getImm();
239 if (isInt<32>(x: Imm)) {
240 Locs.emplace_back(Args: Location::Constant, Args: sizeof(int64_t), Args: 0, Args&: Imm);
241 } else {
242 auto Result = ConstPool.insert(KV: std::make_pair(x&: Imm, y&: Imm));
243 Locs.emplace_back(Args: Location::ConstantIndex, Args: sizeof(int64_t), Args: 0,
244 Args: Result.first - ConstPool.begin());
245 }
246 break;
247 }
248 }
249 return ++MOI;
250 }
251
252 // The physical register number will ultimately be encoded as a DWARF regno.
253 // The stack map also records the size of a spill slot that can hold the
254 // register content. (The runtime can track the actual size of the data type
255 // if it needs to.)
256 if (MOI->isReg()) {
257 // Skip implicit registers (this includes our scratch registers)
258 if (MOI->isImplicit())
259 return ++MOI;
260
261 assert(MOI->getReg().isPhysical() &&
262 "Virtreg operands should have been rewritten before now.");
263 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg: MOI->getReg());
264 assert(!MOI->getSubReg() && "Physical subreg still around.");
265
266 unsigned Offset = 0;
267 unsigned DwarfRegNum = getDwarfRegNum(Reg: MOI->getReg(), TRI);
268 MCRegister LLVMRegNum = *TRI->getLLVMRegNum(RegNum: DwarfRegNum, isEH: false);
269 unsigned SubRegIdx = TRI->getSubRegIndex(RegNo: LLVMRegNum, SubRegNo: MOI->getReg());
270 if (SubRegIdx)
271 Offset = TRI->getSubRegIdxOffset(Idx: SubRegIdx);
272
273 Locs.emplace_back(Args: Location::Register, Args: TRI->getSpillSize(RC: *RC),
274 Args&: DwarfRegNum, Args&: Offset);
275 return ++MOI;
276 }
277
278 if (MOI->isRegLiveOut())
279 LiveOuts = parseRegisterLiveOutMask(Mask: MOI->getRegLiveOut());
280
281 return ++MOI;
282}
283
284void StackMaps::print(raw_ostream &OS) {
285 const TargetRegisterInfo *TRI =
286 AP.MF ? AP.MF->getSubtarget().getRegisterInfo() : nullptr;
287 OS << WSMP << "callsites:\n";
288 for (const auto &CSI : CSInfos) {
289 const LocationVec &CSLocs = CSI.Locations;
290 const LiveOutVec &LiveOuts = CSI.LiveOuts;
291
292 OS << WSMP << "callsite " << CSI.ID << "\n";
293 OS << WSMP << " has " << CSLocs.size() << " locations\n";
294
295 unsigned Idx = 0;
296 for (const auto &Loc : CSLocs) {
297 OS << WSMP << "\t\tLoc " << Idx << ": ";
298 switch (Loc.Type) {
299 case Location::Unprocessed:
300 OS << "<Unprocessed operand>";
301 break;
302 case Location::Register:
303 OS << "Register ";
304 if (TRI)
305 OS << printReg(Reg: Loc.Reg, TRI);
306 else
307 OS << Loc.Reg;
308 break;
309 case Location::Direct:
310 OS << "Direct ";
311 if (TRI)
312 OS << printReg(Reg: Loc.Reg, TRI);
313 else
314 OS << Loc.Reg;
315 if (Loc.Offset)
316 OS << " + " << Loc.Offset;
317 break;
318 case Location::Indirect:
319 OS << "Indirect ";
320 if (TRI)
321 OS << printReg(Reg: Loc.Reg, TRI);
322 else
323 OS << Loc.Reg;
324 OS << "+" << Loc.Offset;
325 break;
326 case Location::Constant:
327 OS << "Constant " << Loc.Offset;
328 break;
329 case Location::ConstantIndex:
330 OS << "Constant Index " << Loc.Offset;
331 break;
332 }
333 OS << "\t[encoding: .byte " << Loc.Type << ", .byte 0"
334 << ", .short " << Loc.Size << ", .short " << Loc.Reg << ", .short 0"
335 << ", .int " << Loc.Offset << "]\n";
336 Idx++;
337 }
338
339 OS << WSMP << "\thas " << LiveOuts.size() << " live-out registers\n";
340
341 Idx = 0;
342 for (const auto &LO : LiveOuts) {
343 OS << WSMP << "\t\tLO " << Idx << ": ";
344 if (TRI)
345 OS << printReg(Reg: LO.Reg, TRI);
346 else
347 OS << LO.Reg;
348 OS << "\t[encoding: .short " << LO.DwarfRegNum << ", .byte 0, .byte "
349 << LO.Size << "]\n";
350 Idx++;
351 }
352 }
353}
354
355/// Create a live-out register record for the given register Reg.
356StackMaps::LiveOutReg
357StackMaps::createLiveOutReg(unsigned Reg, const TargetRegisterInfo *TRI) const {
358 unsigned DwarfRegNum = getDwarfRegNum(Reg, TRI);
359 unsigned Size = TRI->getSpillSize(RC: *TRI->getMinimalPhysRegClass(Reg));
360 return LiveOutReg(Reg, DwarfRegNum, Size);
361}
362
363/// Parse the register live-out mask and return a vector of live-out registers
364/// that need to be recorded in the stackmap.
365StackMaps::LiveOutVec
366StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const {
367 assert(Mask && "No register mask specified");
368 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo();
369 LiveOutVec LiveOuts;
370
371 // Create a LiveOutReg for each bit that is set in the register mask.
372 for (unsigned Reg = 0, NumRegs = TRI->getNumRegs(); Reg != NumRegs; ++Reg)
373 if ((Mask[Reg / 32] >> (Reg % 32)) & 1)
374 LiveOuts.push_back(Elt: createLiveOutReg(Reg, TRI));
375
376 // We don't need to keep track of a register if its super-register is already
377 // in the list. Merge entries that refer to the same dwarf register and use
378 // the maximum size that needs to be spilled.
379
380 llvm::sort(C&: LiveOuts, Comp: [](const LiveOutReg &LHS, const LiveOutReg &RHS) {
381 // Only sort by the dwarf register number.
382 return LHS.DwarfRegNum < RHS.DwarfRegNum;
383 });
384
385 for (auto I = LiveOuts.begin(), E = LiveOuts.end(); I != E; ++I) {
386 for (auto *II = std::next(x: I); II != E; ++II) {
387 if (I->DwarfRegNum != II->DwarfRegNum) {
388 // Skip all the now invalid entries.
389 I = --II;
390 break;
391 }
392 I->Size = std::max(a: I->Size, b: II->Size);
393 if (I->Reg && TRI->isSuperRegister(RegA: I->Reg, RegB: II->Reg))
394 I->Reg = II->Reg;
395 II->Reg = 0; // mark for deletion.
396 }
397 }
398
399 llvm::erase_if(C&: LiveOuts, P: [](const LiveOutReg &LO) { return LO.Reg == 0; });
400
401 return LiveOuts;
402}
403
404// See statepoint MI format description in StatepointOpers' class comment
405// in include/llvm/CodeGen/StackMaps.h
406void StackMaps::parseStatepointOpers(const MachineInstr &MI,
407 MachineInstr::const_mop_iterator MOI,
408 MachineInstr::const_mop_iterator MOE,
409 LocationVec &Locations,
410 LiveOutVec &LiveOuts) {
411 LLVM_DEBUG(dbgs() << "record statepoint : " << MI << "\n");
412 StatepointOpers SO(&MI);
413 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts); // CC
414 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts); // Flags
415 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts); // Num Deopts
416
417 // Record Deopt Args.
418 unsigned NumDeoptArgs = Locations.back().Offset;
419 assert(Locations.back().Type == Location::Constant);
420 assert(NumDeoptArgs == SO.getNumDeoptArgs());
421
422 while (NumDeoptArgs--)
423 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts);
424
425 // Record gc base/derived pairs
426 assert(MOI->isImm() && MOI->getImm() == StackMaps::ConstantOp);
427 ++MOI;
428 assert(MOI->isImm());
429 unsigned NumGCPointers = MOI->getImm();
430 ++MOI;
431 if (NumGCPointers) {
432 // Map logical index of GC ptr to MI operand index.
433 SmallVector<unsigned, 8> GCPtrIndices;
434 unsigned GCPtrIdx = (unsigned)SO.getFirstGCPtrIdx();
435 assert((int)GCPtrIdx != -1);
436 assert(MOI - MI.operands_begin() == GCPtrIdx + 0LL);
437 while (NumGCPointers--) {
438 GCPtrIndices.push_back(Elt: GCPtrIdx);
439 GCPtrIdx = StackMaps::getNextMetaArgIdx(MI: &MI, CurIdx: GCPtrIdx);
440 }
441
442 SmallVector<std::pair<unsigned, unsigned>, 8> GCPairs;
443 unsigned NumGCPairs = SO.getGCPointerMap(GCMap&: GCPairs);
444 (void)NumGCPairs;
445 LLVM_DEBUG(dbgs() << "NumGCPairs = " << NumGCPairs << "\n");
446
447 auto MOB = MI.operands_begin();
448 for (auto &P : GCPairs) {
449 assert(P.first < GCPtrIndices.size() && "base pointer index not found");
450 assert(P.second < GCPtrIndices.size() &&
451 "derived pointer index not found");
452 unsigned BaseIdx = GCPtrIndices[P.first];
453 unsigned DerivedIdx = GCPtrIndices[P.second];
454 LLVM_DEBUG(dbgs() << "Base : " << BaseIdx << " Derived : " << DerivedIdx
455 << "\n");
456 (void)parseOperand(MOI: MOB + BaseIdx, MOE, Locs&: Locations, LiveOuts);
457 (void)parseOperand(MOI: MOB + DerivedIdx, MOE, Locs&: Locations, LiveOuts);
458 }
459
460 MOI = MOB + GCPtrIdx;
461 }
462
463 // Record gc allocas
464 assert(MOI < MOE);
465 assert(MOI->isImm() && MOI->getImm() == StackMaps::ConstantOp);
466 ++MOI;
467 unsigned NumAllocas = MOI->getImm();
468 ++MOI;
469 while (NumAllocas--) {
470 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts);
471 assert(MOI < MOE);
472 }
473}
474
475void StackMaps::recordStackMapOpers(const MCSymbol &MILabel,
476 const MachineInstr &MI, uint64_t ID,
477 MachineInstr::const_mop_iterator MOI,
478 MachineInstr::const_mop_iterator MOE,
479 bool recordResult) {
480 MCContext &OutContext = AP.OutStreamer->getContext();
481
482 LocationVec Locations;
483 LiveOutVec LiveOuts;
484
485 if (recordResult) {
486 assert(PatchPointOpers(&MI).hasDef() && "Stackmap has no return value.");
487 parseOperand(MOI: MI.operands_begin(), MOE: std::next(x: MI.operands_begin()), Locs&: Locations,
488 LiveOuts);
489 }
490
491 // Parse operands.
492 if (MI.getOpcode() == TargetOpcode::STATEPOINT)
493 parseStatepointOpers(MI, MOI, MOE, Locations, LiveOuts);
494 else
495 while (MOI != MOE)
496 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts);
497
498 // Create an expression to calculate the offset of the callsite from function
499 // entry.
500 const MCExpr *CSOffsetExpr = MCBinaryExpr::createSub(
501 LHS: MCSymbolRefExpr::create(Symbol: &MILabel, Ctx&: OutContext),
502 RHS: MCSymbolRefExpr::create(Symbol: AP.CurrentFnSymForSize, Ctx&: OutContext), Ctx&: OutContext);
503
504 CSInfos.emplace_back(args&: CSOffsetExpr, args&: ID, args: std::move(Locations),
505 args: std::move(LiveOuts));
506
507 // Record the stack size of the current function and update callsite count.
508 const MachineFrameInfo &MFI = AP.MF->getFrameInfo();
509 const TargetRegisterInfo *RegInfo = AP.MF->getSubtarget().getRegisterInfo();
510 bool HasDynamicFrameSize =
511 MFI.hasVarSizedObjects() || RegInfo->hasStackRealignment(MF: *(AP.MF));
512 uint64_t FrameSize = HasDynamicFrameSize ? UINT64_MAX : MFI.getStackSize();
513
514 auto [CurrentIt, Inserted] = FnInfos.try_emplace(Key: AP.CurrentFnSym, Args&: FrameSize);
515 if (!Inserted)
516 CurrentIt->second.RecordCount++;
517}
518
519void StackMaps::recordStackMap(const MCSymbol &L, const MachineInstr &MI) {
520 assert(MI.getOpcode() == TargetOpcode::STACKMAP && "expected stackmap");
521
522 StackMapOpers opers(&MI);
523 const int64_t ID = MI.getOperand(i: PatchPointOpers::IDPos).getImm();
524 recordStackMapOpers(MILabel: L, MI, ID, MOI: std::next(x: MI.operands_begin(),
525 n: opers.getVarIdx()),
526 MOE: MI.operands_end());
527}
528
529void StackMaps::recordPatchPoint(const MCSymbol &L, const MachineInstr &MI) {
530 assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "expected patchpoint");
531
532 PatchPointOpers opers(&MI);
533 const int64_t ID = opers.getID();
534 auto MOI = std::next(x: MI.operands_begin(), n: opers.getStackMapStartIdx());
535 recordStackMapOpers(MILabel: L, MI, ID, MOI, MOE: MI.operands_end(),
536 recordResult: opers.isAnyReg() && opers.hasDef());
537
538#ifndef NDEBUG
539 // verify anyregcc
540 auto &Locations = CSInfos.back().Locations;
541 if (opers.isAnyReg()) {
542 unsigned NArgs = opers.getNumCallArgs();
543 for (unsigned i = 0, e = (opers.hasDef() ? NArgs + 1 : NArgs); i != e; ++i)
544 assert(Locations[i].Type == Location::Register &&
545 "anyreg arg must be in reg.");
546 }
547#endif
548}
549
550void StackMaps::recordStatepoint(const MCSymbol &L, const MachineInstr &MI) {
551 assert(MI.getOpcode() == TargetOpcode::STATEPOINT && "expected statepoint");
552
553 StatepointOpers opers(&MI);
554 const unsigned StartIdx = opers.getVarIdx();
555 recordStackMapOpers(MILabel: L, MI, ID: opers.getID(), MOI: MI.operands_begin() + StartIdx,
556 MOE: MI.operands_end(), recordResult: false);
557}
558
559/// Emit the stackmap header.
560///
561/// Header {
562/// uint8 : Stack Map Version (currently 3)
563/// uint8 : Reserved (expected to be 0)
564/// uint16 : Reserved (expected to be 0)
565/// }
566/// uint32 : NumFunctions
567/// uint32 : NumConstants
568/// uint32 : NumRecords
569void StackMaps::emitStackmapHeader(MCStreamer &OS) {
570 // Header.
571 OS.emitIntValue(Value: StackMapVersion, Size: 1); // Version.
572 OS.emitIntValue(Value: 0, Size: 1); // Reserved.
573 OS.emitInt16(Value: 0); // Reserved.
574
575 // Num functions.
576 LLVM_DEBUG(dbgs() << WSMP << "#functions = " << FnInfos.size() << '\n');
577 OS.emitInt32(Value: FnInfos.size());
578 // Num constants.
579 LLVM_DEBUG(dbgs() << WSMP << "#constants = " << ConstPool.size() << '\n');
580 OS.emitInt32(Value: ConstPool.size());
581 // Num callsites.
582 LLVM_DEBUG(dbgs() << WSMP << "#callsites = " << CSInfos.size() << '\n');
583 OS.emitInt32(Value: CSInfos.size());
584}
585
586/// Emit the function frame record for each function.
587///
588/// StkSizeRecord[NumFunctions] {
589/// uint64 : Function Address
590/// uint64 : Stack Size
591/// uint64 : Record Count
592/// }
593void StackMaps::emitFunctionFrameRecords(MCStreamer &OS) {
594 // Function Frame records.
595 LLVM_DEBUG(dbgs() << WSMP << "functions:\n");
596 for (auto const &FR : FnInfos) {
597 LLVM_DEBUG(dbgs() << WSMP << "function addr: " << FR.first
598 << " frame size: " << FR.second.StackSize
599 << " callsite count: " << FR.second.RecordCount << '\n');
600 OS.emitSymbolValue(Sym: FR.first, Size: 8);
601 OS.emitIntValue(Value: FR.second.StackSize, Size: 8);
602 OS.emitIntValue(Value: FR.second.RecordCount, Size: 8);
603 }
604}
605
606/// Emit the constant pool.
607///
608/// int64 : Constants[NumConstants]
609void StackMaps::emitConstantPoolEntries(MCStreamer &OS) {
610 // Constant pool entries.
611 LLVM_DEBUG(dbgs() << WSMP << "constants:\n");
612 for (const auto &ConstEntry : ConstPool) {
613 LLVM_DEBUG(dbgs() << WSMP << ConstEntry.second << '\n');
614 OS.emitIntValue(Value: ConstEntry.second, Size: 8);
615 }
616}
617
618/// Emit the callsite info for each callsite.
619///
620/// StkMapRecord[NumRecords] {
621/// uint64 : PatchPoint ID
622/// uint32 : Instruction Offset
623/// uint16 : Reserved (record flags)
624/// uint16 : NumLocations
625/// Location[NumLocations] {
626/// uint8 : Register | Direct | Indirect | Constant | ConstantIndex
627/// uint8 : Size in Bytes
628/// uint16 : Dwarf RegNum
629/// int32 : Offset
630/// }
631/// uint16 : Padding
632/// uint16 : NumLiveOuts
633/// LiveOuts[NumLiveOuts] {
634/// uint16 : Dwarf RegNum
635/// uint8 : Reserved
636/// uint8 : Size in Bytes
637/// }
638/// uint32 : Padding (only if required to align to 8 byte)
639/// }
640///
641/// Location Encoding, Type, Value:
642/// 0x1, Register, Reg (value in register)
643/// 0x2, Direct, Reg + Offset (frame index)
644/// 0x3, Indirect, [Reg + Offset] (spilled value)
645/// 0x4, Constant, Offset (small constant)
646/// 0x5, ConstIndex, Constants[Offset] (large constant)
647void StackMaps::emitCallsiteEntries(MCStreamer &OS) {
648 LLVM_DEBUG(print(dbgs()));
649 // Callsite entries.
650 for (const auto &CSI : CSInfos) {
651 const LocationVec &CSLocs = CSI.Locations;
652 const LiveOutVec &LiveOuts = CSI.LiveOuts;
653
654 // Verify stack map entry. It's better to communicate a problem to the
655 // runtime than crash in case of in-process compilation. Currently, we do
656 // simple overflow checks, but we may eventually communicate other
657 // compilation errors this way.
658 if (CSLocs.size() > UINT16_MAX || LiveOuts.size() > UINT16_MAX) {
659 OS.emitIntValue(UINT64_MAX, Size: 8); // Invalid ID.
660 OS.emitValue(Value: CSI.CSOffsetExpr, Size: 4);
661 OS.emitInt16(Value: 0); // Reserved.
662 OS.emitInt16(Value: 0); // 0 locations.
663 OS.emitInt16(Value: 0); // padding.
664 OS.emitInt16(Value: 0); // 0 live-out registers.
665 OS.emitInt32(Value: 0); // padding.
666 continue;
667 }
668
669 OS.emitIntValue(Value: CSI.ID, Size: 8);
670 OS.emitValue(Value: CSI.CSOffsetExpr, Size: 4);
671
672 // Reserved for flags.
673 OS.emitInt16(Value: 0);
674 OS.emitInt16(Value: CSLocs.size());
675
676 for (const auto &Loc : CSLocs) {
677 OS.emitIntValue(Value: Loc.Type, Size: 1);
678 OS.emitIntValue(Value: 0, Size: 1); // Reserved
679 OS.emitInt16(Value: Loc.Size);
680 OS.emitInt16(Value: Loc.Reg);
681 OS.emitInt16(Value: 0); // Reserved
682 OS.emitInt32(Value: Loc.Offset);
683 }
684
685 // Emit alignment to 8 byte.
686 OS.emitValueToAlignment(Alignment: Align(8));
687
688 // Num live-out registers and padding to align to 4 byte.
689 OS.emitInt16(Value: 0);
690 OS.emitInt16(Value: LiveOuts.size());
691
692 for (const auto &LO : LiveOuts) {
693 OS.emitInt16(Value: LO.DwarfRegNum);
694 OS.emitIntValue(Value: 0, Size: 1);
695 OS.emitIntValue(Value: LO.Size, Size: 1);
696 }
697 // Emit alignment to 8 byte.
698 OS.emitValueToAlignment(Alignment: Align(8));
699 }
700}
701
702/// Serialize the stackmap data.
703void StackMaps::serializeToStackMapSection() {
704 (void)WSMP;
705 // Bail out if there's no stack map data.
706 assert((!CSInfos.empty() || ConstPool.empty()) &&
707 "Expected empty constant pool too!");
708 assert((!CSInfos.empty() || FnInfos.empty()) &&
709 "Expected empty function record too!");
710 if (CSInfos.empty())
711 return;
712
713 MCContext &OutContext = AP.OutStreamer->getContext();
714 MCStreamer &OS = *AP.OutStreamer;
715
716 // Create the section.
717 MCSection *StackMapSection =
718 OutContext.getObjectFileInfo()->getStackMapSection();
719 OS.switchSection(Section: StackMapSection);
720
721 // Emit a dummy symbol to force section inclusion.
722 OS.emitLabel(Symbol: OutContext.getOrCreateSymbol(Name: Twine("__LLVM_StackMaps")));
723
724 // Serialize data.
725 LLVM_DEBUG(dbgs() << "********** Stack Map Output **********\n");
726 emitStackmapHeader(OS);
727 emitFunctionFrameRecords(OS);
728 emitConstantPoolEntries(OS);
729 emitCallsiteEntries(OS);
730 OS.addBlankLine();
731
732 // Clean up.
733 CSInfos.clear();
734 ConstPool.clear();
735}
736