1//===- StackMaps.cpp ------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/CodeGen/StackMaps.h"
10#include "llvm/ADT/DenseMapInfo.h"
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/ADT/Twine.h"
13#include "llvm/CodeGen/AsmPrinter.h"
14#include "llvm/CodeGen/MachineFrameInfo.h"
15#include "llvm/CodeGen/MachineFunction.h"
16#include "llvm/CodeGen/MachineInstr.h"
17#include "llvm/CodeGen/MachineOperand.h"
18#include "llvm/CodeGen/TargetOpcodes.h"
19#include "llvm/CodeGen/TargetRegisterInfo.h"
20#include "llvm/CodeGen/TargetSubtargetInfo.h"
21#include "llvm/IR/DataLayout.h"
22#include "llvm/MC/MCContext.h"
23#include "llvm/MC/MCExpr.h"
24#include "llvm/MC/MCObjectFileInfo.h"
25#include "llvm/MC/MCStreamer.h"
26#include "llvm/Support/CommandLine.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/MathExtras.h"
30#include "llvm/Support/raw_ostream.h"
31#include <algorithm>
32#include <cassert>
33#include <cstdint>
34#include <iterator>
35#include <utility>
36
37using namespace llvm;
38
39#define DEBUG_TYPE "stackmaps"
40
41static cl::opt<int> StackMapVersion(
42 "stackmap-version", cl::init(Val: 3), cl::Hidden,
43 cl::desc("Specify the stackmap encoding version (default = 3)"));
44
45const char *StackMaps::WSMP = "Stack Maps: ";
46
47static uint64_t getConstMetaVal(const MachineInstr &MI, unsigned Idx) {
48 assert(MI.getOperand(Idx).isImm() &&
49 MI.getOperand(Idx).getImm() == StackMaps::ConstantOp);
50 const auto &MO = MI.getOperand(i: Idx + 1);
51 assert(MO.isImm());
52 return MO.getImm();
53}
54
55StackMapOpers::StackMapOpers(const MachineInstr *MI)
56 : MI(MI) {
57 assert(getVarIdx() <= MI->getNumOperands() &&
58 "invalid stackmap definition");
59}
60
61PatchPointOpers::PatchPointOpers(const MachineInstr *MI)
62 : MI(MI), HasDef(MI->getOperand(i: 0).isReg() && MI->getOperand(i: 0).isDef() &&
63 !MI->getOperand(i: 0).isImplicit()) {
64#ifndef NDEBUG
65 unsigned CheckStartIdx = 0, e = MI->getNumOperands();
66 while (CheckStartIdx < e && MI->getOperand(CheckStartIdx).isReg() &&
67 MI->getOperand(CheckStartIdx).isDef() &&
68 !MI->getOperand(CheckStartIdx).isImplicit())
69 ++CheckStartIdx;
70
71 assert(getMetaIdx() == CheckStartIdx &&
72 "Unexpected additional definition in Patchpoint intrinsic.");
73#endif
74}
75
76unsigned PatchPointOpers::getNextScratchIdx(unsigned StartIdx) const {
77 if (!StartIdx)
78 StartIdx = getVarIdx();
79
80 // Find the next scratch register (implicit def and early clobber)
81 unsigned ScratchIdx = StartIdx, e = MI->getNumOperands();
82 while (ScratchIdx < e &&
83 !(MI->getOperand(i: ScratchIdx).isReg() &&
84 MI->getOperand(i: ScratchIdx).isDef() &&
85 MI->getOperand(i: ScratchIdx).isImplicit() &&
86 MI->getOperand(i: ScratchIdx).isEarlyClobber()))
87 ++ScratchIdx;
88
89 assert(ScratchIdx != e && "No scratch register available");
90 return ScratchIdx;
91}
92
93unsigned StatepointOpers::getNumGcMapEntriesIdx() {
94 // Take index of num of allocas and skip all allocas records.
95 unsigned CurIdx = getNumAllocaIdx();
96 unsigned NumAllocas = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
97 CurIdx++;
98 while (NumAllocas--)
99 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
100 return CurIdx + 1; // skip <StackMaps::ConstantOp>
101}
102
103unsigned StatepointOpers::getNumAllocaIdx() {
104 // Take index of num of gc ptrs and skip all gc ptr records.
105 unsigned CurIdx = getNumGCPtrIdx();
106 unsigned NumGCPtrs = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
107 CurIdx++;
108 while (NumGCPtrs--)
109 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
110 return CurIdx + 1; // skip <StackMaps::ConstantOp>
111}
112
113unsigned StatepointOpers::getNumGCPtrIdx() {
114 // Take index of num of deopt args and skip all deopt records.
115 unsigned CurIdx = getNumDeoptArgsIdx();
116 unsigned NumDeoptArgs = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
117 CurIdx++;
118 while (NumDeoptArgs--) {
119 CurIdx = StackMaps::getNextMetaArgIdx(MI, CurIdx);
120 }
121 return CurIdx + 1; // skip <StackMaps::ConstantOp>
122}
123
124int StatepointOpers::getFirstGCPtrIdx() {
125 unsigned NumGCPtrsIdx = getNumGCPtrIdx();
126 unsigned NumGCPtrs = getConstMetaVal(MI: *MI, Idx: NumGCPtrsIdx - 1);
127 if (NumGCPtrs == 0)
128 return -1;
129 ++NumGCPtrsIdx; // skip <num gc ptrs>
130 assert(NumGCPtrsIdx < MI->getNumOperands());
131 return (int)NumGCPtrsIdx;
132}
133
134unsigned StatepointOpers::getGCPointerMap(
135 SmallVectorImpl<std::pair<unsigned, unsigned>> &GCMap) {
136 unsigned CurIdx = getNumGcMapEntriesIdx();
137 unsigned GCMapSize = getConstMetaVal(MI: *MI, Idx: CurIdx - 1);
138 CurIdx++;
139 for (unsigned N = 0; N < GCMapSize; ++N) {
140 unsigned B = MI->getOperand(i: CurIdx++).getImm();
141 unsigned D = MI->getOperand(i: CurIdx++).getImm();
142 GCMap.push_back(Elt: std::make_pair(x&: B, y&: D));
143 }
144
145 return GCMapSize;
146}
147
148bool StatepointOpers::isFoldableReg(Register Reg) const {
149 unsigned FoldableAreaStart = getVarIdx();
150 for (const MachineOperand &MO : MI->uses()) {
151 if (MO.getOperandNo() >= FoldableAreaStart)
152 break;
153 if (MO.isReg() && MO.getReg() == Reg)
154 return false;
155 }
156 return true;
157}
158
159bool StatepointOpers::isFoldableReg(const MachineInstr *MI, Register Reg) {
160 if (MI->getOpcode() != TargetOpcode::STATEPOINT)
161 return false;
162 return StatepointOpers(MI).isFoldableReg(Reg);
163}
164
165StackMaps::StackMaps(AsmPrinter &AP) : AP(AP) {
166 if (StackMapVersion != 3)
167 llvm_unreachable("Unsupported stackmap version!");
168}
169
170unsigned StackMaps::getNextMetaArgIdx(const MachineInstr *MI, unsigned CurIdx) {
171 assert(CurIdx < MI->getNumOperands() && "Bad meta arg index");
172 const auto &MO = MI->getOperand(i: CurIdx);
173 if (MO.isImm()) {
174 switch (MO.getImm()) {
175 default:
176 llvm_unreachable("Unrecognized operand type.");
177 case StackMaps::DirectMemRefOp:
178 CurIdx += 2;
179 break;
180 case StackMaps::IndirectMemRefOp:
181 CurIdx += 3;
182 break;
183 case StackMaps::ConstantOp:
184 ++CurIdx;
185 break;
186 }
187 }
188 ++CurIdx;
189 assert(CurIdx < MI->getNumOperands() && "points past operand list");
190 return CurIdx;
191}
192
193/// Go up the super-register chain until we hit a valid dwarf register number.
194static unsigned getDwarfRegNum(MCRegister Reg, const TargetRegisterInfo *TRI) {
195 int RegNum;
196 for (MCPhysReg SR : TRI->superregs_inclusive(Reg)) {
197 RegNum = TRI->getDwarfRegNum(Reg: SR, isEH: false);
198 if (RegNum >= 0)
199 break;
200 }
201
202 assert(RegNum >= 0 && isUInt<16>(RegNum) && "Invalid Dwarf register number.");
203 return (unsigned)RegNum;
204}
205
206MachineInstr::const_mop_iterator
207StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
208 MachineInstr::const_mop_iterator MOE, LocationVec &Locs,
209 LiveOutVec &LiveOuts) {
210 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo();
211 if (MOI->isImm()) {
212 switch (MOI->getImm()) {
213 default:
214 llvm_unreachable("Unrecognized operand type.");
215 case StackMaps::DirectMemRefOp: {
216 auto &DL = AP.MF->getDataLayout();
217
218 unsigned Size = DL.getPointerSizeInBits();
219 assert((Size % 8) == 0 && "Need pointer size in bytes.");
220 Size /= 8;
221 Register Reg = (++MOI)->getReg();
222 int64_t Imm = (++MOI)->getImm();
223 Locs.emplace_back(Args: StackMaps::Location::Direct, Args&: Size,
224 Args: getDwarfRegNum(Reg, TRI), Args&: Imm);
225 break;
226 }
227 case StackMaps::IndirectMemRefOp: {
228 int64_t Size = (++MOI)->getImm();
229 assert(Size > 0 && "Need a valid size for indirect memory locations.");
230 Register Reg = (++MOI)->getReg();
231 int64_t Imm = (++MOI)->getImm();
232 Locs.emplace_back(Args: StackMaps::Location::Indirect, Args&: Size,
233 Args: getDwarfRegNum(Reg, TRI), Args&: Imm);
234 break;
235 }
236 case StackMaps::ConstantOp: {
237 ++MOI;
238 assert(MOI->isImm() && "Expected constant operand.");
239 int64_t Imm = MOI->getImm();
240 if (isInt<32>(x: Imm)) {
241 Locs.emplace_back(Args: Location::Constant, Args: sizeof(int64_t), Args: 0, Args&: Imm);
242 } else {
243 auto Result = ConstPool.insert(KV: std::make_pair(x&: Imm, y&: Imm));
244 Locs.emplace_back(Args: Location::ConstantIndex, Args: sizeof(int64_t), Args: 0,
245 Args: Result.first - ConstPool.begin());
246 }
247 break;
248 }
249 }
250 return ++MOI;
251 }
252
253 // The physical register number will ultimately be encoded as a DWARF regno.
254 // The stack map also records the size of a spill slot that can hold the
255 // register content. (The runtime can track the actual size of the data type
256 // if it needs to.)
257 if (MOI->isReg()) {
258 // Skip implicit registers (this includes our scratch registers)
259 if (MOI->isImplicit())
260 return ++MOI;
261
262 assert(MOI->getReg().isPhysical() &&
263 "Virtreg operands should have been rewritten before now.");
264 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg: MOI->getReg());
265 assert(!MOI->getSubReg() && "Physical subreg still around.");
266
267 unsigned Offset = 0;
268 unsigned DwarfRegNum = getDwarfRegNum(Reg: MOI->getReg(), TRI);
269 MCRegister LLVMRegNum = *TRI->getLLVMRegNum(RegNum: DwarfRegNum, isEH: false);
270 unsigned SubRegIdx = TRI->getSubRegIndex(RegNo: LLVMRegNum, SubRegNo: MOI->getReg());
271 if (SubRegIdx)
272 Offset = TRI->getSubRegIdxOffset(Idx: SubRegIdx);
273
274 Locs.emplace_back(Args: Location::Register, Args: TRI->getSpillSize(RC: *RC),
275 Args&: DwarfRegNum, Args&: Offset);
276 return ++MOI;
277 }
278
279 if (MOI->isRegLiveOut())
280 LiveOuts = parseRegisterLiveOutMask(Mask: MOI->getRegLiveOut());
281
282 return ++MOI;
283}
284
285void StackMaps::print(raw_ostream &OS) {
286 const TargetRegisterInfo *TRI =
287 AP.MF ? AP.MF->getSubtarget().getRegisterInfo() : nullptr;
288 OS << WSMP << "callsites:\n";
289 for (const auto &CSI : CSInfos) {
290 const LocationVec &CSLocs = CSI.Locations;
291 const LiveOutVec &LiveOuts = CSI.LiveOuts;
292
293 OS << WSMP << "callsite " << CSI.ID << "\n";
294 OS << WSMP << " has " << CSLocs.size() << " locations\n";
295
296 unsigned Idx = 0;
297 for (const auto &Loc : CSLocs) {
298 OS << WSMP << "\t\tLoc " << Idx << ": ";
299 switch (Loc.Type) {
300 case Location::Unprocessed:
301 OS << "<Unprocessed operand>";
302 break;
303 case Location::Register:
304 OS << "Register ";
305 if (TRI)
306 OS << printReg(Reg: Loc.Reg, TRI);
307 else
308 OS << Loc.Reg;
309 break;
310 case Location::Direct:
311 OS << "Direct ";
312 if (TRI)
313 OS << printReg(Reg: Loc.Reg, TRI);
314 else
315 OS << Loc.Reg;
316 if (Loc.Offset)
317 OS << " + " << Loc.Offset;
318 break;
319 case Location::Indirect:
320 OS << "Indirect ";
321 if (TRI)
322 OS << printReg(Reg: Loc.Reg, TRI);
323 else
324 OS << Loc.Reg;
325 OS << "+" << Loc.Offset;
326 break;
327 case Location::Constant:
328 OS << "Constant " << Loc.Offset;
329 break;
330 case Location::ConstantIndex:
331 OS << "Constant Index " << Loc.Offset;
332 break;
333 }
334 OS << "\t[encoding: .byte " << Loc.Type << ", .byte 0"
335 << ", .short " << Loc.Size << ", .short " << Loc.Reg << ", .short 0"
336 << ", .int " << Loc.Offset << "]\n";
337 Idx++;
338 }
339
340 OS << WSMP << "\thas " << LiveOuts.size() << " live-out registers\n";
341
342 Idx = 0;
343 for (const auto &LO : LiveOuts) {
344 OS << WSMP << "\t\tLO " << Idx << ": ";
345 if (TRI)
346 OS << printReg(Reg: LO.Reg, TRI);
347 else
348 OS << LO.Reg;
349 OS << "\t[encoding: .short " << LO.DwarfRegNum << ", .byte 0, .byte "
350 << LO.Size << "]\n";
351 Idx++;
352 }
353 }
354}
355
356/// Create a live-out register record for the given register Reg.
357StackMaps::LiveOutReg
358StackMaps::createLiveOutReg(unsigned Reg, const TargetRegisterInfo *TRI) const {
359 unsigned DwarfRegNum = getDwarfRegNum(Reg, TRI);
360 unsigned Size = TRI->getSpillSize(RC: *TRI->getMinimalPhysRegClass(Reg));
361 return LiveOutReg(Reg, DwarfRegNum, Size);
362}
363
364/// Parse the register live-out mask and return a vector of live-out registers
365/// that need to be recorded in the stackmap.
366StackMaps::LiveOutVec
367StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const {
368 assert(Mask && "No register mask specified");
369 const TargetRegisterInfo *TRI = AP.MF->getSubtarget().getRegisterInfo();
370 LiveOutVec LiveOuts;
371
372 // Create a LiveOutReg for each bit that is set in the register mask.
373 for (unsigned Reg = 0, NumRegs = TRI->getNumRegs(); Reg != NumRegs; ++Reg)
374 if ((Mask[Reg / 32] >> (Reg % 32)) & 1)
375 LiveOuts.push_back(Elt: createLiveOutReg(Reg, TRI));
376
377 // We don't need to keep track of a register if its super-register is already
378 // in the list. Merge entries that refer to the same dwarf register and use
379 // the maximum size that needs to be spilled.
380
381 llvm::sort(C&: LiveOuts, Comp: [](const LiveOutReg &LHS, const LiveOutReg &RHS) {
382 // Only sort by the dwarf register number.
383 return LHS.DwarfRegNum < RHS.DwarfRegNum;
384 });
385
386 for (auto I = LiveOuts.begin(), E = LiveOuts.end(); I != E; ++I) {
387 for (auto *II = std::next(x: I); II != E; ++II) {
388 if (I->DwarfRegNum != II->DwarfRegNum) {
389 // Skip all the now invalid entries.
390 I = --II;
391 break;
392 }
393 I->Size = std::max(a: I->Size, b: II->Size);
394 if (I->Reg && TRI->isSuperRegister(RegA: I->Reg, RegB: II->Reg))
395 I->Reg = II->Reg;
396 II->Reg = 0; // mark for deletion.
397 }
398 }
399
400 llvm::erase_if(C&: LiveOuts, P: [](const LiveOutReg &LO) { return LO.Reg == 0; });
401
402 return LiveOuts;
403}
404
405// See statepoint MI format description in StatepointOpers' class comment
406// in include/llvm/CodeGen/StackMaps.h
407void StackMaps::parseStatepointOpers(const MachineInstr &MI,
408 MachineInstr::const_mop_iterator MOI,
409 MachineInstr::const_mop_iterator MOE,
410 LocationVec &Locations,
411 LiveOutVec &LiveOuts) {
412 LLVM_DEBUG(dbgs() << "record statepoint : " << MI << "\n");
413 StatepointOpers SO(&MI);
414 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts); // CC
415 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts); // Flags
416 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts); // Num Deopts
417
418 // Record Deopt Args.
419 unsigned NumDeoptArgs = Locations.back().Offset;
420 assert(Locations.back().Type == Location::Constant);
421 assert(NumDeoptArgs == SO.getNumDeoptArgs());
422
423 while (NumDeoptArgs--)
424 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts);
425
426 // Record gc base/derived pairs
427 assert(MOI->isImm() && MOI->getImm() == StackMaps::ConstantOp);
428 ++MOI;
429 assert(MOI->isImm());
430 unsigned NumGCPointers = MOI->getImm();
431 ++MOI;
432 if (NumGCPointers) {
433 // Map logical index of GC ptr to MI operand index.
434 SmallVector<unsigned, 8> GCPtrIndices;
435 unsigned GCPtrIdx = (unsigned)SO.getFirstGCPtrIdx();
436 assert((int)GCPtrIdx != -1);
437 assert(MOI - MI.operands_begin() == GCPtrIdx + 0LL);
438 while (NumGCPointers--) {
439 GCPtrIndices.push_back(Elt: GCPtrIdx);
440 GCPtrIdx = StackMaps::getNextMetaArgIdx(MI: &MI, CurIdx: GCPtrIdx);
441 }
442
443 SmallVector<std::pair<unsigned, unsigned>, 8> GCPairs;
444 unsigned NumGCPairs = SO.getGCPointerMap(GCMap&: GCPairs);
445 (void)NumGCPairs;
446 LLVM_DEBUG(dbgs() << "NumGCPairs = " << NumGCPairs << "\n");
447
448 auto MOB = MI.operands_begin();
449 for (auto &P : GCPairs) {
450 assert(P.first < GCPtrIndices.size() && "base pointer index not found");
451 assert(P.second < GCPtrIndices.size() &&
452 "derived pointer index not found");
453 unsigned BaseIdx = GCPtrIndices[P.first];
454 unsigned DerivedIdx = GCPtrIndices[P.second];
455 LLVM_DEBUG(dbgs() << "Base : " << BaseIdx << " Derived : " << DerivedIdx
456 << "\n");
457 (void)parseOperand(MOI: MOB + BaseIdx, MOE, Locs&: Locations, LiveOuts);
458 (void)parseOperand(MOI: MOB + DerivedIdx, MOE, Locs&: Locations, LiveOuts);
459 }
460
461 MOI = MOB + GCPtrIdx;
462 }
463
464 // Record gc allocas
465 assert(MOI < MOE);
466 assert(MOI->isImm() && MOI->getImm() == StackMaps::ConstantOp);
467 ++MOI;
468 unsigned NumAllocas = MOI->getImm();
469 ++MOI;
470 while (NumAllocas--) {
471 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts);
472 assert(MOI < MOE);
473 }
474}
475
476void StackMaps::recordStackMapOpers(const MCSymbol &MILabel,
477 const MachineInstr &MI, uint64_t ID,
478 MachineInstr::const_mop_iterator MOI,
479 MachineInstr::const_mop_iterator MOE,
480 bool recordResult) {
481 MCContext &OutContext = AP.OutStreamer->getContext();
482
483 LocationVec Locations;
484 LiveOutVec LiveOuts;
485
486 if (recordResult) {
487 assert(PatchPointOpers(&MI).hasDef() && "Stackmap has no return value.");
488 parseOperand(MOI: MI.operands_begin(), MOE: std::next(x: MI.operands_begin()), Locs&: Locations,
489 LiveOuts);
490 }
491
492 // Parse operands.
493 if (MI.getOpcode() == TargetOpcode::STATEPOINT)
494 parseStatepointOpers(MI, MOI, MOE, Locations, LiveOuts);
495 else
496 while (MOI != MOE)
497 MOI = parseOperand(MOI, MOE, Locs&: Locations, LiveOuts);
498
499 // Create an expression to calculate the offset of the callsite from function
500 // entry.
501 const MCExpr *CSOffsetExpr = MCBinaryExpr::createSub(
502 LHS: MCSymbolRefExpr::create(Symbol: &MILabel, Ctx&: OutContext),
503 RHS: MCSymbolRefExpr::create(Symbol: AP.CurrentFnSymForSize, Ctx&: OutContext), Ctx&: OutContext);
504
505 CSInfos.emplace_back(args&: CSOffsetExpr, args&: ID, args: std::move(Locations),
506 args: std::move(LiveOuts));
507
508 // Record the stack size of the current function and update callsite count.
509 const MachineFrameInfo &MFI = AP.MF->getFrameInfo();
510 const TargetRegisterInfo *RegInfo = AP.MF->getSubtarget().getRegisterInfo();
511 bool HasDynamicFrameSize =
512 MFI.hasVarSizedObjects() || RegInfo->hasStackRealignment(MF: *(AP.MF));
513 uint64_t FrameSize = HasDynamicFrameSize ? UINT64_MAX : MFI.getStackSize();
514
515 auto [CurrentIt, Inserted] = FnInfos.try_emplace(Key: AP.CurrentFnSym, Args&: FrameSize);
516 if (!Inserted)
517 CurrentIt->second.RecordCount++;
518}
519
520void StackMaps::recordStackMap(const MCSymbol &L, const MachineInstr &MI) {
521 assert(MI.getOpcode() == TargetOpcode::STACKMAP && "expected stackmap");
522
523 StackMapOpers opers(&MI);
524 const int64_t ID = MI.getOperand(i: PatchPointOpers::IDPos).getImm();
525 recordStackMapOpers(MILabel: L, MI, ID, MOI: std::next(x: MI.operands_begin(),
526 n: opers.getVarIdx()),
527 MOE: MI.operands_end());
528}
529
530void StackMaps::recordPatchPoint(const MCSymbol &L, const MachineInstr &MI) {
531 assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "expected patchpoint");
532
533 PatchPointOpers opers(&MI);
534 const int64_t ID = opers.getID();
535 auto MOI = std::next(x: MI.operands_begin(), n: opers.getStackMapStartIdx());
536 recordStackMapOpers(MILabel: L, MI, ID, MOI, MOE: MI.operands_end(),
537 recordResult: opers.isAnyReg() && opers.hasDef());
538
539#ifndef NDEBUG
540 // verify anyregcc
541 auto &Locations = CSInfos.back().Locations;
542 if (opers.isAnyReg()) {
543 unsigned NArgs = opers.getNumCallArgs();
544 for (unsigned i = 0, e = (opers.hasDef() ? NArgs + 1 : NArgs); i != e; ++i)
545 assert(Locations[i].Type == Location::Register &&
546 "anyreg arg must be in reg.");
547 }
548#endif
549}
550
551void StackMaps::recordStatepoint(const MCSymbol &L, const MachineInstr &MI) {
552 assert(MI.getOpcode() == TargetOpcode::STATEPOINT && "expected statepoint");
553
554 StatepointOpers opers(&MI);
555 const unsigned StartIdx = opers.getVarIdx();
556 recordStackMapOpers(MILabel: L, MI, ID: opers.getID(), MOI: MI.operands_begin() + StartIdx,
557 MOE: MI.operands_end(), recordResult: false);
558}
559
560/// Emit the stackmap header.
561///
562/// Header {
563/// uint8 : Stack Map Version (currently 3)
564/// uint8 : Reserved (expected to be 0)
565/// uint16 : Reserved (expected to be 0)
566/// }
567/// uint32 : NumFunctions
568/// uint32 : NumConstants
569/// uint32 : NumRecords
570void StackMaps::emitStackmapHeader(MCStreamer &OS) {
571 // Header.
572 OS.emitIntValue(Value: StackMapVersion, Size: 1); // Version.
573 OS.emitIntValue(Value: 0, Size: 1); // Reserved.
574 OS.emitInt16(Value: 0); // Reserved.
575
576 // Num functions.
577 LLVM_DEBUG(dbgs() << WSMP << "#functions = " << FnInfos.size() << '\n');
578 OS.emitInt32(Value: FnInfos.size());
579 // Num constants.
580 LLVM_DEBUG(dbgs() << WSMP << "#constants = " << ConstPool.size() << '\n');
581 OS.emitInt32(Value: ConstPool.size());
582 // Num callsites.
583 LLVM_DEBUG(dbgs() << WSMP << "#callsites = " << CSInfos.size() << '\n');
584 OS.emitInt32(Value: CSInfos.size());
585}
586
587/// Emit the function frame record for each function.
588///
589/// StkSizeRecord[NumFunctions] {
590/// uint64 : Function Address
591/// uint64 : Stack Size
592/// uint64 : Record Count
593/// }
594void StackMaps::emitFunctionFrameRecords(MCStreamer &OS) {
595 // Function Frame records.
596 LLVM_DEBUG(dbgs() << WSMP << "functions:\n");
597 for (auto const &FR : FnInfos) {
598 LLVM_DEBUG(dbgs() << WSMP << "function addr: " << FR.first
599 << " frame size: " << FR.second.StackSize
600 << " callsite count: " << FR.second.RecordCount << '\n');
601 OS.emitSymbolValue(Sym: FR.first, Size: 8);
602 OS.emitIntValue(Value: FR.second.StackSize, Size: 8);
603 OS.emitIntValue(Value: FR.second.RecordCount, Size: 8);
604 }
605}
606
607/// Emit the constant pool.
608///
609/// int64 : Constants[NumConstants]
610void StackMaps::emitConstantPoolEntries(MCStreamer &OS) {
611 // Constant pool entries.
612 LLVM_DEBUG(dbgs() << WSMP << "constants:\n");
613 for (const auto &ConstEntry : ConstPool) {
614 LLVM_DEBUG(dbgs() << WSMP << ConstEntry.second << '\n');
615 OS.emitIntValue(Value: ConstEntry.second, Size: 8);
616 }
617}
618
619/// Emit the callsite info for each callsite.
620///
621/// StkMapRecord[NumRecords] {
622/// uint64 : PatchPoint ID
623/// uint32 : Instruction Offset
624/// uint16 : Reserved (record flags)
625/// uint16 : NumLocations
626/// Location[NumLocations] {
627/// uint8 : Register | Direct | Indirect | Constant | ConstantIndex
628/// uint8 : Size in Bytes
629/// uint16 : Dwarf RegNum
630/// int32 : Offset
631/// }
632/// uint16 : Padding
633/// uint16 : NumLiveOuts
634/// LiveOuts[NumLiveOuts] {
635/// uint16 : Dwarf RegNum
636/// uint8 : Reserved
637/// uint8 : Size in Bytes
638/// }
639/// uint32 : Padding (only if required to align to 8 byte)
640/// }
641///
642/// Location Encoding, Type, Value:
643/// 0x1, Register, Reg (value in register)
644/// 0x2, Direct, Reg + Offset (frame index)
645/// 0x3, Indirect, [Reg + Offset] (spilled value)
646/// 0x4, Constant, Offset (small constant)
647/// 0x5, ConstIndex, Constants[Offset] (large constant)
648void StackMaps::emitCallsiteEntries(MCStreamer &OS) {
649 LLVM_DEBUG(print(dbgs()));
650 // Callsite entries.
651 for (const auto &CSI : CSInfos) {
652 const LocationVec &CSLocs = CSI.Locations;
653 const LiveOutVec &LiveOuts = CSI.LiveOuts;
654
655 // Verify stack map entry. It's better to communicate a problem to the
656 // runtime than crash in case of in-process compilation. Currently, we do
657 // simple overflow checks, but we may eventually communicate other
658 // compilation errors this way.
659 if (CSLocs.size() > UINT16_MAX || LiveOuts.size() > UINT16_MAX) {
660 OS.emitIntValue(UINT64_MAX, Size: 8); // Invalid ID.
661 OS.emitValue(Value: CSI.CSOffsetExpr, Size: 4);
662 OS.emitInt16(Value: 0); // Reserved.
663 OS.emitInt16(Value: 0); // 0 locations.
664 OS.emitInt16(Value: 0); // padding.
665 OS.emitInt16(Value: 0); // 0 live-out registers.
666 OS.emitInt32(Value: 0); // padding.
667 continue;
668 }
669
670 OS.emitIntValue(Value: CSI.ID, Size: 8);
671 OS.emitValue(Value: CSI.CSOffsetExpr, Size: 4);
672
673 // Reserved for flags.
674 OS.emitInt16(Value: 0);
675 OS.emitInt16(Value: CSLocs.size());
676
677 for (const auto &Loc : CSLocs) {
678 OS.emitIntValue(Value: Loc.Type, Size: 1);
679 OS.emitIntValue(Value: 0, Size: 1); // Reserved
680 OS.emitInt16(Value: Loc.Size);
681 OS.emitInt16(Value: Loc.Reg);
682 OS.emitInt16(Value: 0); // Reserved
683 OS.emitInt32(Value: Loc.Offset);
684 }
685
686 // Emit alignment to 8 byte.
687 OS.emitValueToAlignment(Alignment: Align(8));
688
689 // Num live-out registers and padding to align to 4 byte.
690 OS.emitInt16(Value: 0);
691 OS.emitInt16(Value: LiveOuts.size());
692
693 for (const auto &LO : LiveOuts) {
694 OS.emitInt16(Value: LO.DwarfRegNum);
695 OS.emitIntValue(Value: 0, Size: 1);
696 OS.emitIntValue(Value: LO.Size, Size: 1);
697 }
698 // Emit alignment to 8 byte.
699 OS.emitValueToAlignment(Alignment: Align(8));
700 }
701}
702
703/// Serialize the stackmap data.
704void StackMaps::serializeToStackMapSection() {
705 (void)WSMP;
706 // Bail out if there's no stack map data.
707 assert((!CSInfos.empty() || ConstPool.empty()) &&
708 "Expected empty constant pool too!");
709 assert((!CSInfos.empty() || FnInfos.empty()) &&
710 "Expected empty function record too!");
711 if (CSInfos.empty())
712 return;
713
714 MCContext &OutContext = AP.OutStreamer->getContext();
715 MCStreamer &OS = *AP.OutStreamer;
716
717 // Create the section.
718 MCSection *StackMapSection =
719 OutContext.getObjectFileInfo()->getStackMapSection();
720 OS.switchSection(Section: StackMapSection);
721
722 // Emit a dummy symbol to force section inclusion.
723 OS.emitLabel(Symbol: OutContext.getOrCreateSymbol(Name: Twine("__LLVM_StackMaps")));
724
725 // Serialize data.
726 LLVM_DEBUG(dbgs() << "********** Stack Map Output **********\n");
727 emitStackmapHeader(OS);
728 emitFunctionFrameRecords(OS);
729 emitConstantPoolEntries(OS);
730 emitCallsiteEntries(OS);
731 OS.addBlankLine();
732
733 // Clean up.
734 CSInfos.clear();
735 ConstPool.clear();
736}
737