1//===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains code to lower X86 MachineInstrs to their corresponding
10// MCInst records.
11//
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/X86ATTInstPrinter.h"
15#include "MCTargetDesc/X86BaseInfo.h"
16#include "MCTargetDesc/X86EncodingOptimization.h"
17#include "MCTargetDesc/X86InstComments.h"
18#include "MCTargetDesc/X86MCAsmInfo.h"
19#include "MCTargetDesc/X86ShuffleDecode.h"
20#include "MCTargetDesc/X86TargetStreamer.h"
21#include "X86AsmPrinter.h"
22#include "X86MachineFunctionInfo.h"
23#include "X86RegisterInfo.h"
24#include "X86ShuffleDecodeConstantPool.h"
25#include "X86Subtarget.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SmallString.h"
28#include "llvm/ADT/StringExtras.h"
29#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
30#include "llvm/CodeGen/MachineConstantPool.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineModuleInfoImpls.h"
33#include "llvm/CodeGen/MachineOperand.h"
34#include "llvm/CodeGen/StackMaps.h"
35#include "llvm/CodeGen/WinEHFuncInfo.h"
36#include "llvm/IR/DataLayout.h"
37#include "llvm/IR/GlobalValue.h"
38#include "llvm/IR/Mangler.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCCodeEmitter.h"
41#include "llvm/MC/MCContext.h"
42#include "llvm/MC/MCExpr.h"
43#include "llvm/MC/MCFixup.h"
44#include "llvm/MC/MCInst.h"
45#include "llvm/MC/MCInstBuilder.h"
46#include "llvm/MC/MCSection.h"
47#include "llvm/MC/MCStreamer.h"
48#include "llvm/MC/MCSymbol.h"
49#include "llvm/MC/TargetRegistry.h"
50#include "llvm/Target/TargetLoweringObjectFile.h"
51#include "llvm/Target/TargetMachine.h"
52#include "llvm/Transforms/CFGuard.h"
53#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
54#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
55#include <string>
56
57using namespace llvm;
58
59static cl::opt<bool> EnableBranchHint("enable-branch-hint",
60 cl::desc("Enable branch hint."),
61 cl::init(Val: false), cl::Hidden);
62static cl::opt<unsigned> BranchHintProbabilityThreshold(
63 "branch-hint-probability-threshold",
64 cl::desc("The probability threshold of enabling branch hint."),
65 cl::init(Val: 50), cl::Hidden);
66
67namespace {
68
69/// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.
70class X86MCInstLower {
71 MCContext &Ctx;
72 const MachineFunction &MF;
73 const TargetMachine &TM;
74 const MCAsmInfo &MAI;
75 X86AsmPrinter &AsmPrinter;
76
77public:
78 X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter);
79
80 MCOperand LowerMachineOperand(const MachineInstr *MI,
81 const MachineOperand &MO) const;
82 void Lower(const MachineInstr *MI, MCInst &OutMI) const;
83
84 MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const;
85 MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const;
86
87private:
88 MachineModuleInfoMachO &getMachOMMI() const;
89};
90
91} // end anonymous namespace
92
93/// A RAII helper which defines a region of instructions which can't have
94/// padding added between them for correctness.
95struct NoAutoPaddingScope {
96 MCStreamer &OS;
97 const bool OldAllowAutoPadding;
98 NoAutoPaddingScope(MCStreamer &OS)
99 : OS(OS), OldAllowAutoPadding(OS.getAllowAutoPadding()) {
100 changeAndComment(b: false);
101 }
102 ~NoAutoPaddingScope() { changeAndComment(b: OldAllowAutoPadding); }
103 void changeAndComment(bool b) {
104 if (b == OS.getAllowAutoPadding())
105 return;
106 OS.setAllowAutoPadding(b);
107 if (b)
108 OS.emitRawComment(T: "autopadding");
109 else
110 OS.emitRawComment(T: "noautopadding");
111 }
112};
113
114// Emit a minimal sequence of nops spanning NumBytes bytes.
115static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
116 const X86Subtarget *Subtarget);
117
118void X86AsmPrinter::StackMapShadowTracker::count(const MCInst &Inst,
119 const MCSubtargetInfo &STI,
120 MCCodeEmitter *CodeEmitter) {
121 if (InShadow) {
122 SmallString<256> Code;
123 SmallVector<MCFixup, 4> Fixups;
124 CodeEmitter->encodeInstruction(Inst, CB&: Code, Fixups, STI);
125 CurrentShadowSize += Code.size();
126 if (CurrentShadowSize >= RequiredShadowSize)
127 InShadow = false; // The shadow is big enough. Stop counting.
128 }
129}
130
131void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding(
132 MCStreamer &OutStreamer, const MCSubtargetInfo &STI) {
133 if (InShadow && CurrentShadowSize < RequiredShadowSize) {
134 InShadow = false;
135 emitX86Nops(OS&: OutStreamer, NumBytes: RequiredShadowSize - CurrentShadowSize,
136 Subtarget: &MF->getSubtarget<X86Subtarget>());
137 }
138}
139
140void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {
141 OutStreamer->emitInstruction(Inst, STI: getSubtargetInfo());
142 SMShadowTracker.count(Inst, STI: getSubtargetInfo(), CodeEmitter: CodeEmitter.get());
143}
144
145X86MCInstLower::X86MCInstLower(const MachineFunction &mf,
146 X86AsmPrinter &asmprinter)
147 : Ctx(asmprinter.OutContext), MF(mf), TM(mf.getTarget()),
148 MAI(TM.getMCAsmInfo()), AsmPrinter(asmprinter) {}
149
150MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {
151 return AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoMachO>();
152}
153
154/// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol
155/// operand to an MCSymbol.
156MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const {
157 const Triple &TT = TM.getTargetTriple();
158 if (MO.isGlobal() && TT.isOSBinFormatELF())
159 return AsmPrinter.getSymbolPreferLocal(GV: *MO.getGlobal());
160
161 const DataLayout &DL = MF.getDataLayout();
162 assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) &&
163 "Isn't a symbol reference");
164
165 MCSymbol *Sym = nullptr;
166 SmallString<128> Name;
167 StringRef Suffix;
168
169 switch (MO.getTargetFlags()) {
170 case X86II::MO_DLLIMPORT:
171 // Handle dllimport linkage.
172 Name += "__imp_";
173 break;
174 case X86II::MO_COFFSTUB:
175 Name += ".refptr.";
176 break;
177 case X86II::MO_DARWIN_NONLAZY:
178 case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
179 Suffix = "$non_lazy_ptr";
180 break;
181 }
182
183 if (!Suffix.empty())
184 Name += DL.getInternalSymbolPrefix();
185
186 if (MO.isGlobal()) {
187 const GlobalValue *GV = MO.getGlobal();
188 AsmPrinter.getNameWithPrefix(Name, GV);
189 } else if (MO.isSymbol()) {
190 Mangler::getNameWithPrefix(OutName&: Name, GVName: MO.getSymbolName(), DL);
191 } else if (MO.isMBB()) {
192 assert(Suffix.empty());
193 Sym = MO.getMBB()->getSymbol();
194 }
195
196 Name += Suffix;
197 if (!Sym)
198 Sym = Ctx.getOrCreateSymbol(Name);
199
200 // If the target flags on the operand changes the name of the symbol, do that
201 // before we return the symbol.
202 switch (MO.getTargetFlags()) {
203 default:
204 break;
205 case X86II::MO_COFFSTUB: {
206 MachineModuleInfoCOFF &MMICOFF =
207 AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoCOFF>();
208 MachineModuleInfoImpl::StubValueTy &StubSym = MMICOFF.getGVStubEntry(Sym);
209 if (!StubSym.getPointer()) {
210 assert(MO.isGlobal() && "Extern symbol not handled yet");
211 StubSym = MachineModuleInfoImpl::StubValueTy(
212 AsmPrinter.getSymbol(GV: MO.getGlobal()), true);
213 }
214 break;
215 }
216 case X86II::MO_DARWIN_NONLAZY:
217 case X86II::MO_DARWIN_NONLAZY_PIC_BASE: {
218 MachineModuleInfoImpl::StubValueTy &StubSym =
219 getMachOMMI().getGVStubEntry(Sym);
220 if (!StubSym.getPointer()) {
221 assert(MO.isGlobal() && "Extern symbol not handled yet");
222 StubSym = MachineModuleInfoImpl::StubValueTy(
223 AsmPrinter.getSymbol(GV: MO.getGlobal()),
224 !MO.getGlobal()->hasInternalLinkage());
225 }
226 break;
227 }
228 }
229
230 return Sym;
231}
232
233MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
234 MCSymbol *Sym) const {
235 // FIXME: We would like an efficient form for this, so we don't have to do a
236 // lot of extra uniquing.
237 const MCExpr *Expr = nullptr;
238 uint16_t Specifier = X86::S_None;
239
240 switch (MO.getTargetFlags()) {
241 default:
242 llvm_unreachable("Unknown target flag on GV operand");
243 case X86II::MO_NO_FLAG: // No flag.
244 // These affect the name of the symbol, not any suffix.
245 case X86II::MO_DARWIN_NONLAZY:
246 case X86II::MO_DLLIMPORT:
247 case X86II::MO_COFFSTUB:
248 break;
249
250 case X86II::MO_TLVP:
251 Specifier = X86::S_TLVP;
252 break;
253 case X86II::MO_TLVP_PIC_BASE:
254 Expr = MCSymbolRefExpr::create(Symbol: Sym, specifier: X86::S_TLVP, Ctx);
255 // Subtract the pic base.
256 Expr = MCBinaryExpr::createSub(
257 LHS: Expr, RHS: MCSymbolRefExpr::create(Symbol: MF.getPICBaseSymbol(), Ctx), Ctx);
258 break;
259 case X86II::MO_SECREL:
260 Specifier = uint16_t(X86::S_COFF_SECREL);
261 break;
262 case X86II::MO_TLSGD:
263 Specifier = X86::S_TLSGD;
264 break;
265 case X86II::MO_TLSLD:
266 Specifier = X86::S_TLSLD;
267 break;
268 case X86II::MO_TLSLDM:
269 Specifier = X86::S_TLSLDM;
270 break;
271 case X86II::MO_GOTTPOFF:
272 Specifier = X86::S_GOTTPOFF;
273 break;
274 case X86II::MO_INDNTPOFF:
275 Specifier = X86::S_INDNTPOFF;
276 break;
277 case X86II::MO_TPOFF:
278 Specifier = X86::S_TPOFF;
279 break;
280 case X86II::MO_DTPOFF:
281 Specifier = X86::S_DTPOFF;
282 break;
283 case X86II::MO_NTPOFF:
284 Specifier = X86::S_NTPOFF;
285 break;
286 case X86II::MO_GOTNTPOFF:
287 Specifier = X86::S_GOTNTPOFF;
288 break;
289 case X86II::MO_GOTPCREL:
290 Specifier = X86::S_GOTPCREL;
291 break;
292 case X86II::MO_GOTPCREL_NORELAX:
293 Specifier = X86::S_GOTPCREL_NORELAX;
294 break;
295 case X86II::MO_GOT:
296 Specifier = X86::S_GOT;
297 break;
298 case X86II::MO_GOTOFF:
299 Specifier = X86::S_GOTOFF;
300 break;
301 case X86II::MO_PLT:
302 Specifier = X86::S_PLT;
303 break;
304 case X86II::MO_ABS8:
305 Specifier = X86::S_ABS8;
306 break;
307 case X86II::MO_PIC_BASE_OFFSET:
308 case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
309 Expr = MCSymbolRefExpr::create(Symbol: Sym, Ctx);
310 // Subtract the pic base.
311 Expr = MCBinaryExpr::createSub(
312 LHS: Expr, RHS: MCSymbolRefExpr::create(Symbol: MF.getPICBaseSymbol(), Ctx), Ctx);
313 if (MO.isJTI()) {
314 assert(MAI.doesSetDirectiveSuppressReloc());
315 // If .set directive is supported, use it to reduce the number of
316 // relocations the assembler will generate for differences between
317 // local labels. This is only safe when the symbols are in the same
318 // section so we are restricting it to jumptable references.
319 MCSymbol *Label = Ctx.createTempSymbol();
320 AsmPrinter.OutStreamer->emitAssignment(Symbol: Label, Value: Expr);
321 Expr = MCSymbolRefExpr::create(Symbol: Label, Ctx);
322 }
323 break;
324 }
325
326 if (!Expr)
327 Expr = MCSymbolRefExpr::create(Symbol: Sym, specifier: Specifier, Ctx);
328
329 if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
330 Expr = MCBinaryExpr::createAdd(
331 LHS: Expr, RHS: MCConstantExpr::create(Value: MO.getOffset(), Ctx), Ctx);
332 return MCOperand::createExpr(Val: Expr);
333}
334
335static unsigned getRetOpcode(const X86Subtarget &Subtarget) {
336 return Subtarget.is64Bit() ? X86::RET64 : X86::RET32;
337}
338
339MCOperand X86MCInstLower::LowerMachineOperand(const MachineInstr *MI,
340 const MachineOperand &MO) const {
341 switch (MO.getType()) {
342 default:
343 MI->print(OS&: errs());
344 llvm_unreachable("unknown operand type");
345 case MachineOperand::MO_Register:
346 // Ignore all implicit register operands.
347 if (MO.isImplicit())
348 return MCOperand();
349 return MCOperand::createReg(Reg: MO.getReg());
350 case MachineOperand::MO_Immediate:
351 return MCOperand::createImm(Val: MO.getImm());
352 case MachineOperand::MO_MachineBasicBlock:
353 case MachineOperand::MO_GlobalAddress:
354 case MachineOperand::MO_ExternalSymbol:
355 return LowerSymbolOperand(MO, Sym: GetSymbolFromOperand(MO));
356 case MachineOperand::MO_MCSymbol:
357 return LowerSymbolOperand(MO, Sym: MO.getMCSymbol());
358 case MachineOperand::MO_JumpTableIndex:
359 return LowerSymbolOperand(MO, Sym: AsmPrinter.GetJTISymbol(JTID: MO.getIndex()));
360 case MachineOperand::MO_ConstantPoolIndex:
361 return LowerSymbolOperand(MO, Sym: AsmPrinter.GetCPISymbol(CPID: MO.getIndex()));
362 case MachineOperand::MO_BlockAddress:
363 return LowerSymbolOperand(
364 MO, Sym: AsmPrinter.GetBlockAddressSymbol(BA: MO.getBlockAddress()));
365 case MachineOperand::MO_RegisterMask:
366 // Ignore call clobbers.
367 return MCOperand();
368 }
369}
370
371// Replace TAILJMP opcodes with their equivalent opcodes that have encoding
372// information.
373static unsigned convertTailJumpOpcode(unsigned Opcode, bool IsLarge = false) {
374 switch (Opcode) {
375 case X86::TAILJMPr:
376 Opcode = X86::JMP32r;
377 break;
378 case X86::TAILJMPm:
379 Opcode = X86::JMP32m;
380 break;
381 case X86::TAILJMPr64:
382 Opcode = X86::JMP64r;
383 break;
384 case X86::TAILJMPm64:
385 Opcode = X86::JMP64m;
386 break;
387 case X86::TAILJMPr64_REX:
388 Opcode = X86::JMP64r_REX;
389 break;
390 case X86::TAILJMPm64_REX:
391 Opcode = X86::JMP64m_REX;
392 break;
393 case X86::TAILJMPd:
394 case X86::TAILJMPd64:
395 Opcode = IsLarge ? X86::JMPABS64i : X86::JMP_1;
396 break;
397 case X86::TAILJMPd_CC:
398 case X86::TAILJMPd64_CC:
399 Opcode = X86::JCC_1;
400 break;
401 }
402
403 return Opcode;
404}
405
406void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
407 OutMI.setOpcode(MI->getOpcode());
408
409 for (const MachineOperand &MO : MI->operands())
410 if (auto Op = LowerMachineOperand(MI, MO); Op.isValid())
411 OutMI.addOperand(Op);
412
413 bool In64BitMode = AsmPrinter.getSubtarget().is64Bit();
414 if (X86::optimizeInstFromVEX3ToVEX2(MI&: OutMI, Desc: MI->getDesc()) ||
415 X86::optimizeShiftRotateWithImmediateOne(MI&: OutMI) ||
416 X86::optimizeVPCMPWithImmediateOneOrSix(MI&: OutMI) ||
417 X86::optimizeMOVSX(MI&: OutMI) || X86::optimizeINCDEC(MI&: OutMI, In64BitMode) ||
418 X86::optimizeMOV(MI&: OutMI, In64BitMode) ||
419 X86::optimizeToFixedRegisterOrShortImmediateForm(MI&: OutMI))
420 return;
421
422 // Handle a few special cases to eliminate operand modifiers.
423 switch (OutMI.getOpcode()) {
424 case X86::LEA64_32r:
425 case X86::LEA64r:
426 case X86::LEA16r:
427 case X86::LEA32r:
428 // LEA should have a segment register, but it must be empty.
429 assert(OutMI.getNumOperands() == 1 + X86::AddrNumOperands &&
430 "Unexpected # of LEA operands");
431 assert(OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 &&
432 "LEA has segment specified!");
433 break;
434 case X86::MULX32Hrr:
435 case X86::MULX32Hrm:
436 case X86::MULX64Hrr:
437 case X86::MULX64Hrm: {
438 // Turn into regular MULX by duplicating the destination.
439 unsigned NewOpc;
440 switch (OutMI.getOpcode()) {
441 default: llvm_unreachable("Invalid opcode");
442 case X86::MULX32Hrr: NewOpc = X86::MULX32rr; break;
443 case X86::MULX32Hrm: NewOpc = X86::MULX32rm; break;
444 case X86::MULX64Hrr: NewOpc = X86::MULX64rr; break;
445 case X86::MULX64Hrm: NewOpc = X86::MULX64rm; break;
446 }
447 OutMI.setOpcode(NewOpc);
448 // Duplicate the destination.
449 MCRegister DestReg = OutMI.getOperand(i: 0).getReg();
450 OutMI.insert(I: OutMI.begin(), Op: MCOperand::createReg(Reg: DestReg));
451 break;
452 }
453 // CALL64r, CALL64pcrel32 - These instructions used to have
454 // register inputs modeled as normal uses instead of implicit uses. As such,
455 // they we used to truncate off all but the first operand (the callee). This
456 // issue seems to have been fixed at some point. This assert verifies that.
457 case X86::CALL64r:
458 case X86::CALL64pcrel32:
459 assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
460 break;
461 case X86::EH_RETURN:
462 case X86::EH_RETURN64: {
463 OutMI = MCInst();
464 OutMI.setOpcode(getRetOpcode(Subtarget: AsmPrinter.getSubtarget()));
465 break;
466 }
467 case X86::CLEANUPRET: {
468 // Replace CLEANUPRET with the appropriate RET.
469 OutMI = MCInst();
470 OutMI.setOpcode(getRetOpcode(Subtarget: AsmPrinter.getSubtarget()));
471 break;
472 }
473 case X86::CATCHRET: {
474 // Replace CATCHRET with the appropriate RET.
475 const X86Subtarget &Subtarget = AsmPrinter.getSubtarget();
476 unsigned ReturnReg = In64BitMode ? X86::RAX : X86::EAX;
477 OutMI = MCInst();
478 OutMI.setOpcode(getRetOpcode(Subtarget));
479 OutMI.addOperand(Op: MCOperand::createReg(Reg: ReturnReg));
480 break;
481 }
482 // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump
483 // instruction.
484 case X86::TAILJMPr:
485 case X86::TAILJMPr64:
486 case X86::TAILJMPr64_REX:
487 case X86::TAILJMPd:
488 assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
489 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode()));
490 break;
491 case X86::TAILJMPd64: {
492 assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
493 bool IsLarge = TM.getCodeModel() == CodeModel::Large;
494 assert((!IsLarge || AsmPrinter.getSubtarget().hasJMPABS()) &&
495 "Unexpected TAILJMPd64 in large code model without JMPABS");
496 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode(), IsLarge));
497 break;
498 }
499 case X86::TAILJMPd_CC:
500 case X86::TAILJMPd64_CC:
501 assert(OutMI.getNumOperands() == 2 && "Unexpected number of operands!");
502 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode()));
503 break;
504 case X86::TAILJMPm:
505 case X86::TAILJMPm64:
506 case X86::TAILJMPm64_REX:
507 assert(OutMI.getNumOperands() == X86::AddrNumOperands &&
508 "Unexpected number of operands!");
509 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode()));
510 break;
511 case X86::MASKMOVDQU:
512 case X86::VMASKMOVDQU:
513 if (In64BitMode)
514 OutMI.setFlags(X86::IP_HAS_AD_SIZE);
515 break;
516 case X86::BSF16rm:
517 case X86::BSF16rr:
518 case X86::BSF32rm:
519 case X86::BSF32rr:
520 case X86::BSF64rm:
521 case X86::BSF64rr: {
522 // Add an REP prefix to BSF instructions so that new processors can
523 // recognize as TZCNT, which has better performance than BSF.
524 // BSF and TZCNT have different interpretations on ZF bit. So make sure
525 // it won't be used later.
526 const MachineOperand *FlagDef =
527 MI->findRegisterDefOperand(Reg: X86::EFLAGS, /*TRI=*/nullptr);
528 if (!MF.getFunction().hasOptSize() && FlagDef && FlagDef->isDead())
529 OutMI.setFlags(X86::IP_HAS_REPEAT);
530 break;
531 }
532 default:
533 break;
534 }
535}
536
537void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
538 const MachineInstr &MI) {
539 NoAutoPaddingScope NoPadScope(*OutStreamer);
540 bool Is64Bits = getSubtarget().is64Bit();
541 bool Is64BitsLP64 = getSubtarget().isTarget64BitLP64();
542 MCContext &Ctx = OutStreamer->getContext();
543
544 X86::Specifier Specifier;
545 switch (MI.getOpcode()) {
546 case X86::TLS_addr32:
547 case X86::TLS_addr64:
548 case X86::TLS_addrX32:
549 Specifier = X86::S_TLSGD;
550 break;
551 case X86::TLS_base_addr32:
552 Specifier = X86::S_TLSLDM;
553 break;
554 case X86::TLS_base_addr64:
555 case X86::TLS_base_addrX32:
556 Specifier = X86::S_TLSLD;
557 break;
558 case X86::TLS_desc32:
559 case X86::TLS_desc64:
560 Specifier = X86::S_TLSDESC;
561 break;
562 default:
563 llvm_unreachable("unexpected opcode");
564 }
565
566 const MCSymbolRefExpr *Sym = MCSymbolRefExpr::create(
567 Symbol: MCInstLowering.GetSymbolFromOperand(MO: MI.getOperand(i: 3)), specifier: Specifier, Ctx);
568
569 // Before binutils 2.41, ld has a bogus TLS relaxation error when the GD/LD
570 // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is
571 // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by
572 // only using GOT when GOTPCRELX is enabled.
573 // TODO Delete the workaround when rustc no longer relies on the hack
574 bool UseGot = MMI->getModule()->getRtLibUseGOT() &&
575 Ctx.getTargetOptions().X86RelaxRelocations;
576
577 if (Specifier == X86::S_TLSDESC) {
578 const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(
579 Symbol: MCInstLowering.GetSymbolFromOperand(MO: MI.getOperand(i: 3)), specifier: X86::S_TLSCALL,
580 Ctx);
581 EmitAndCountInstruction(
582 Inst&: MCInstBuilder(Is64BitsLP64 ? X86::LEA64r : X86::LEA32r)
583 .addReg(Reg: Is64BitsLP64 ? X86::RAX : X86::EAX)
584 .addReg(Reg: Is64Bits ? X86::RIP : X86::EBX)
585 .addImm(Val: 1)
586 .addReg(Reg: 0)
587 .addExpr(Val: Sym)
588 .addReg(Reg: 0));
589 EmitAndCountInstruction(
590 Inst&: MCInstBuilder(Is64Bits ? X86::CALL64m : X86::CALL32m)
591 .addReg(Reg: Is64BitsLP64 ? X86::RAX : X86::EAX)
592 .addImm(Val: 1)
593 .addReg(Reg: 0)
594 .addExpr(Val: Expr)
595 .addReg(Reg: 0));
596 } else if (Is64Bits) {
597 bool NeedsPadding = Specifier == X86::S_TLSGD;
598 if (NeedsPadding && Is64BitsLP64)
599 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DATA16_PREFIX));
600 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::LEA64r)
601 .addReg(Reg: X86::RDI)
602 .addReg(Reg: X86::RIP)
603 .addImm(Val: 1)
604 .addReg(Reg: 0)
605 .addExpr(Val: Sym)
606 .addReg(Reg: 0));
607 const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol(Name: "__tls_get_addr");
608 if (NeedsPadding) {
609 if (!UseGot)
610 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DATA16_PREFIX));
611 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DATA16_PREFIX));
612 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::REX64_PREFIX));
613 }
614 if (UseGot) {
615 const MCExpr *Expr =
616 MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_GOTPCREL, Ctx);
617 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64m)
618 .addReg(Reg: X86::RIP)
619 .addImm(Val: 1)
620 .addReg(Reg: 0)
621 .addExpr(Val: Expr)
622 .addReg(Reg: 0));
623 } else {
624 EmitAndCountInstruction(
625 Inst&: MCInstBuilder(X86::CALL64pcrel32)
626 .addExpr(Val: MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_PLT, Ctx)));
627 }
628 } else {
629 if (Specifier == X86::S_TLSGD && !UseGot) {
630 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::LEA32r)
631 .addReg(Reg: X86::EAX)
632 .addReg(Reg: 0)
633 .addImm(Val: 1)
634 .addReg(Reg: X86::EBX)
635 .addExpr(Val: Sym)
636 .addReg(Reg: 0));
637 } else {
638 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::LEA32r)
639 .addReg(Reg: X86::EAX)
640 .addReg(Reg: X86::EBX)
641 .addImm(Val: 1)
642 .addReg(Reg: 0)
643 .addExpr(Val: Sym)
644 .addReg(Reg: 0));
645 }
646
647 const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol(Name: "___tls_get_addr");
648 if (UseGot) {
649 const MCExpr *Expr = MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_GOT, Ctx);
650 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL32m)
651 .addReg(Reg: X86::EBX)
652 .addImm(Val: 1)
653 .addReg(Reg: 0)
654 .addExpr(Val: Expr)
655 .addReg(Reg: 0));
656 } else {
657 EmitAndCountInstruction(
658 Inst&: MCInstBuilder(X86::CALLpcrel32)
659 .addExpr(Val: MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_PLT, Ctx)));
660 }
661 }
662}
663
664/// Emit the largest nop instruction smaller than or equal to \p NumBytes
665/// bytes. Return the size of nop emitted.
666static unsigned emitNop(MCStreamer &OS, unsigned NumBytes,
667 const X86Subtarget *Subtarget) {
668 // Determine the longest nop which can be efficiently decoded for the given
669 // target cpu. 15-bytes is the longest single NOP instruction, but some
670 // platforms can't decode the longest forms efficiently.
671 unsigned MaxNopLength = 1;
672 if (Subtarget->is64Bit()) {
673 // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the
674 // IndexReg/BaseReg below need to be updated.
675 if (Subtarget->hasFeature(Feature: X86::TuningFast7ByteNOP))
676 MaxNopLength = 7;
677 else if (Subtarget->hasFeature(Feature: X86::TuningFast15ByteNOP))
678 MaxNopLength = 15;
679 else if (Subtarget->hasFeature(Feature: X86::TuningFast11ByteNOP))
680 MaxNopLength = 11;
681 else
682 MaxNopLength = 10;
683 } if (Subtarget->is32Bit())
684 MaxNopLength = 2;
685
686 // Cap a single nop emission at the profitable value for the target
687 NumBytes = std::min(a: NumBytes, b: MaxNopLength);
688
689 unsigned NopSize;
690 unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg;
691 IndexReg = Displacement = SegmentReg = 0;
692 BaseReg = X86::RAX;
693 ScaleVal = 1;
694 switch (NumBytes) {
695 case 0:
696 llvm_unreachable("Zero nops?");
697 break;
698 case 1:
699 NopSize = 1;
700 Opc = X86::NOOP;
701 break;
702 case 2:
703 NopSize = 2;
704 Opc = X86::XCHG16ar;
705 break;
706 case 3:
707 NopSize = 3;
708 Opc = X86::NOOPL;
709 break;
710 case 4:
711 NopSize = 4;
712 Opc = X86::NOOPL;
713 Displacement = 8;
714 break;
715 case 5:
716 NopSize = 5;
717 Opc = X86::NOOPL;
718 Displacement = 8;
719 IndexReg = X86::RAX;
720 break;
721 case 6:
722 NopSize = 6;
723 Opc = X86::NOOPW;
724 Displacement = 8;
725 IndexReg = X86::RAX;
726 break;
727 case 7:
728 NopSize = 7;
729 Opc = X86::NOOPL;
730 Displacement = 512;
731 break;
732 case 8:
733 NopSize = 8;
734 Opc = X86::NOOPL;
735 Displacement = 512;
736 IndexReg = X86::RAX;
737 break;
738 case 9:
739 NopSize = 9;
740 Opc = X86::NOOPW;
741 Displacement = 512;
742 IndexReg = X86::RAX;
743 break;
744 default:
745 NopSize = 10;
746 Opc = X86::NOOPW;
747 Displacement = 512;
748 IndexReg = X86::RAX;
749 SegmentReg = X86::CS;
750 break;
751 }
752
753 unsigned NumPrefixes = std::min(a: NumBytes - NopSize, b: 5U);
754 NopSize += NumPrefixes;
755 for (unsigned i = 0; i != NumPrefixes; ++i)
756 OS.emitBytes(Data: "\x66");
757
758 switch (Opc) {
759 default: llvm_unreachable("Unexpected opcode");
760 case X86::NOOP:
761 OS.emitInstruction(Inst: MCInstBuilder(Opc), STI: *Subtarget);
762 break;
763 case X86::XCHG16ar:
764 OS.emitInstruction(Inst: MCInstBuilder(Opc).addReg(Reg: X86::AX).addReg(Reg: X86::AX),
765 STI: *Subtarget);
766 break;
767 case X86::NOOPL:
768 case X86::NOOPW:
769 OS.emitInstruction(Inst: MCInstBuilder(Opc)
770 .addReg(Reg: BaseReg)
771 .addImm(Val: ScaleVal)
772 .addReg(Reg: IndexReg)
773 .addImm(Val: Displacement)
774 .addReg(Reg: SegmentReg),
775 STI: *Subtarget);
776 break;
777 }
778 assert(NopSize <= NumBytes && "We overemitted?");
779 return NopSize;
780}
781
782/// Emit the optimal amount of multi-byte nops on X86.
783static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
784 const X86Subtarget *Subtarget) {
785 unsigned NopsToEmit = NumBytes;
786 (void)NopsToEmit;
787 while (NumBytes) {
788 NumBytes -= emitNop(OS, NumBytes, Subtarget);
789 assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!");
790 }
791}
792
793void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
794 X86MCInstLower &MCIL) {
795 assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
796
797 NoAutoPaddingScope NoPadScope(*OutStreamer);
798
799 StatepointOpers SOpers(&MI);
800 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
801 emitX86Nops(OS&: *OutStreamer, NumBytes: PatchBytes, Subtarget);
802 } else {
803 // Lower call target and choose correct opcode
804 const MachineOperand &CallTarget = SOpers.getCallTarget();
805 MCOperand CallTargetMCOp;
806 unsigned CallOpcode;
807 switch (CallTarget.getType()) {
808 case MachineOperand::MO_GlobalAddress:
809 case MachineOperand::MO_ExternalSymbol:
810 CallTargetMCOp = MCIL.LowerSymbolOperand(
811 MO: CallTarget, Sym: MCIL.GetSymbolFromOperand(MO: CallTarget));
812 CallOpcode = X86::CALL64pcrel32;
813 // Currently, we only support relative addressing with statepoints.
814 // Otherwise, we'll need a scratch register to hold the target
815 // address. You'll fail asserts during load & relocation if this
816 // symbol is to far away. (TODO: support non-relative addressing)
817 break;
818 case MachineOperand::MO_Immediate:
819 CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm());
820 CallOpcode = X86::CALL64pcrel32;
821 // Currently, we only support relative addressing with statepoints.
822 // Otherwise, we'll need a scratch register to hold the target
823 // immediate. You'll fail asserts during load & relocation if this
824 // address is to far away. (TODO: support non-relative addressing)
825 break;
826 case MachineOperand::MO_Register:
827 // FIXME: Add retpoline support and remove this.
828 if (Subtarget->useIndirectThunkCalls())
829 report_fatal_error(reason: "Lowering register statepoints with thunks not "
830 "yet implemented.");
831 CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg());
832 CallOpcode = X86::CALL64r;
833 break;
834 default:
835 llvm_unreachable("Unsupported operand type in statepoint call target");
836 break;
837 }
838
839 // Emit call
840 MCInst CallInst;
841 CallInst.setOpcode(CallOpcode);
842 CallInst.addOperand(Op: CallTargetMCOp);
843 OutStreamer->emitInstruction(Inst: CallInst, STI: getSubtargetInfo());
844 maybeEmitNopAfterCallForWindowsEH(MI: &MI);
845 }
846
847 // Record our statepoint node in the same section used by STACKMAP
848 // and PATCHPOINT
849 auto &Ctx = OutStreamer->getContext();
850 MCSymbol *MILabel = Ctx.createTempSymbol();
851 OutStreamer->emitLabel(Symbol: MILabel);
852 SM.recordStatepoint(L: *MILabel, MI);
853}
854
855void X86AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI,
856 X86MCInstLower &MCIL) {
857 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
858 // <opcode>, <operands>
859
860 NoAutoPaddingScope NoPadScope(*OutStreamer);
861
862 Register DefRegister = FaultingMI.getOperand(i: 0).getReg();
863 FaultMaps::FaultKind FK =
864 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm());
865 MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol();
866 unsigned Opcode = FaultingMI.getOperand(i: 3).getImm();
867 unsigned OperandsBeginIdx = 4;
868
869 auto &Ctx = OutStreamer->getContext();
870 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
871 OutStreamer->emitLabel(Symbol: FaultingLabel);
872
873 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
874 FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel);
875
876 MCInst MI;
877 MI.setOpcode(Opcode);
878
879 if (DefRegister != X86::NoRegister)
880 MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister));
881
882 for (const MachineOperand &MO :
883 llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx))
884 if (auto Op = MCIL.LowerMachineOperand(MI: &FaultingMI, MO); Op.isValid())
885 MI.addOperand(Op);
886
887 OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName());
888 OutStreamer->emitInstruction(Inst: MI, STI: getSubtargetInfo());
889}
890
891void X86AsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI,
892 X86MCInstLower &MCIL) {
893 bool Is64Bits = Subtarget->is64Bit();
894 MCContext &Ctx = OutStreamer->getContext();
895 MCSymbol *fentry = Ctx.getOrCreateSymbol(Name: "__fentry__");
896 const MCSymbolRefExpr *Op = MCSymbolRefExpr::create(Symbol: fentry, Ctx);
897
898 EmitAndCountInstruction(
899 Inst&: MCInstBuilder(Is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32)
900 .addExpr(Val: Op));
901}
902
903void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
904 assert(std::next(MI.getIterator())->isCall() &&
905 "KCFI_CHECK not followed by a call instruction");
906
907 // Adjust the offset for patchable-function-prefix. X86InstrInfo::getNop()
908 // returns a 1-byte X86::NOOP, which means the offset is the same in
909 // bytes. This assumes that patchable-function-prefix is the same for all
910 // functions.
911 const MachineFunction &MF = *MI.getMF();
912 int64_t PrefixNops = MF.getFunction().getFnAttributeAsParsedInteger(
913 Kind: "patchable-function-prefix");
914
915 // KCFI allows indirect calls to any location that's preceded by a valid
916 // type identifier. To avoid encoding the full constant into an instruction,
917 // and thus emitting potential call target gadgets at each indirect call
918 // site, load a negated constant to a register and compare that to the
919 // expected value at the call target.
920 const Register AddrReg = MI.getOperand(i: 0).getReg();
921 const uint32_t Type = MI.getOperand(i: 1).getImm();
922 // The check is immediately before the call. If the call target is in R10,
923 // we can clobber R11 for the check instead.
924 unsigned TempReg = AddrReg == X86::R10 ? X86::R11D : X86::R10D;
925 EmitAndCountInstruction(
926 Inst&: MCInstBuilder(X86::MOV32ri).addReg(Reg: TempReg).addImm(Val: -MaskKCFIType(Value: Type)));
927 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::ADD32rm)
928 .addReg(Reg: X86::NoRegister)
929 .addReg(Reg: TempReg)
930 .addReg(Reg: AddrReg)
931 .addImm(Val: 1)
932 .addReg(Reg: X86::NoRegister)
933 .addImm(Val: -(PrefixNops + 4))
934 .addReg(Reg: X86::NoRegister));
935
936 MCSymbol *Pass = OutContext.createTempSymbol();
937 EmitAndCountInstruction(
938 Inst&: MCInstBuilder(X86::JCC_1)
939 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Pass, Ctx&: OutContext))
940 .addImm(Val: X86::COND_E));
941
942 MCSymbol *Trap = OutContext.createTempSymbol();
943 OutStreamer->emitLabel(Symbol: Trap);
944 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::TRAP));
945 emitKCFITrapEntry(MF, Symbol: Trap);
946 OutStreamer->emitLabel(Symbol: Pass);
947}
948
949void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
950 // FIXME: Make this work on non-ELF.
951 if (!TM.getTargetTriple().isOSBinFormatELF()) {
952 report_fatal_error(reason: "llvm.asan.check.memaccess only supported on ELF");
953 return;
954 }
955
956 const auto &Reg = MI.getOperand(i: 0).getReg();
957 ASanAccessInfo AccessInfo(MI.getOperand(i: 1).getImm());
958
959 uint64_t ShadowBase;
960 int MappingScale;
961 bool OrShadowOffset;
962 getAddressSanitizerParams(TargetTriple: TM.getTargetTriple(), LongSize: 64, IsKasan: AccessInfo.CompileKernel,
963 ShadowBase: &ShadowBase, MappingScale: &MappingScale, OrShadowOffset: &OrShadowOffset);
964
965 StringRef Name = AccessInfo.IsWrite ? "store" : "load";
966 StringRef Op = OrShadowOffset ? "or" : "add";
967 std::string SymName = ("__asan_check_" + Name + "_" + Op + "_" +
968 Twine(1ULL << AccessInfo.AccessSizeIndex) + "_" +
969 TM.getMCRegisterInfo().getName(RegNo: Reg.asMCReg()))
970 .str();
971 if (OrShadowOffset)
972 report_fatal_error(
973 reason: "OrShadowOffset is not supported with optimized callbacks");
974
975 EmitAndCountInstruction(
976 Inst&: MCInstBuilder(X86::CALL64pcrel32)
977 .addExpr(Val: MCSymbolRefExpr::create(
978 Symbol: OutContext.getOrCreateSymbol(Name: SymName), Ctx&: OutContext)));
979}
980
981void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI,
982 X86MCInstLower &MCIL) {
983 // PATCHABLE_OP minsize
984
985 NoAutoPaddingScope NoPadScope(*OutStreamer);
986
987 auto NextMI = std::find_if(first: std::next(x: MI.getIterator()),
988 last: MI.getParent()->end().getInstrIterator(),
989 pred: [](auto &II) { return !II.isMetaInstruction(); });
990
991 SmallString<256> Code;
992 unsigned MinSize = MI.getOperand(i: 0).getImm();
993
994 if (NextMI != MI.getParent()->end() && !NextMI->isInlineAsm()) {
995 // Lower the next MachineInstr to find its byte size.
996 // If the next instruction is inline assembly, we skip lowering it for now,
997 // and assume we should always generate NOPs.
998 MCInst MCI;
999 MCIL.Lower(MI: &*NextMI, OutMI&: MCI);
1000
1001 SmallVector<MCFixup, 4> Fixups;
1002 CodeEmitter->encodeInstruction(Inst: MCI, CB&: Code, Fixups, STI: getSubtargetInfo());
1003 }
1004
1005 if (Code.size() < MinSize) {
1006 if (MinSize == 2 && Subtarget->is32Bit() &&
1007 Subtarget->isTargetWindowsMSVC() &&
1008 (Subtarget->getCPU().empty() || Subtarget->getCPU() == "pentium3")) {
1009 // For compatibility reasons, when targetting MSVC, it is important to
1010 // generate a 'legacy' NOP in the form of a 8B FF MOV EDI, EDI. Some tools
1011 // rely specifically on this pattern to be able to patch a function.
1012 // This is only for 32-bit targets, when using /arch:IA32 or /arch:SSE.
1013 OutStreamer->emitInstruction(
1014 Inst: MCInstBuilder(X86::MOV32rr_REV).addReg(Reg: X86::EDI).addReg(Reg: X86::EDI),
1015 STI: *Subtarget);
1016 } else {
1017 unsigned NopSize = emitNop(OS&: *OutStreamer, NumBytes: MinSize, Subtarget);
1018 assert(NopSize == MinSize && "Could not implement MinSize!");
1019 (void)NopSize;
1020 }
1021 }
1022}
1023
1024// Lower a stackmap of the form:
1025// <id>, <shadowBytes>, ...
1026void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
1027 SMShadowTracker.emitShadowPadding(OutStreamer&: *OutStreamer, STI: getSubtargetInfo());
1028
1029 auto &Ctx = OutStreamer->getContext();
1030 MCSymbol *MILabel = Ctx.createTempSymbol();
1031 OutStreamer->emitLabel(Symbol: MILabel);
1032
1033 SM.recordStackMap(L: *MILabel, MI);
1034 unsigned NumShadowBytes = MI.getOperand(i: 1).getImm();
1035 SMShadowTracker.reset(RequiredSize: NumShadowBytes);
1036}
1037
1038// Lower a patchpoint of the form:
1039// [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ...
1040void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
1041 X86MCInstLower &MCIL) {
1042 assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64");
1043
1044 SMShadowTracker.emitShadowPadding(OutStreamer&: *OutStreamer, STI: getSubtargetInfo());
1045
1046 NoAutoPaddingScope NoPadScope(*OutStreamer);
1047
1048 auto &Ctx = OutStreamer->getContext();
1049 MCSymbol *MILabel = Ctx.createTempSymbol();
1050 OutStreamer->emitLabel(Symbol: MILabel);
1051 SM.recordPatchPoint(L: *MILabel, MI);
1052
1053 PatchPointOpers opers(&MI);
1054 unsigned ScratchIdx = opers.getNextScratchIdx();
1055 unsigned EncodedBytes = 0;
1056 const MachineOperand &CalleeMO = opers.getCallTarget();
1057
1058 // Check for null target. If target is non-null (i.e. is non-zero or is
1059 // symbolic) then emit a call.
1060 if (!(CalleeMO.isImm() && !CalleeMO.getImm())) {
1061 MCOperand CalleeMCOp;
1062 switch (CalleeMO.getType()) {
1063 default:
1064 /// FIXME: Add a verifier check for bad callee types.
1065 llvm_unreachable("Unrecognized callee operand type.");
1066 case MachineOperand::MO_Immediate:
1067 if (CalleeMO.getImm())
1068 CalleeMCOp = MCOperand::createImm(Val: CalleeMO.getImm());
1069 break;
1070 case MachineOperand::MO_ExternalSymbol:
1071 case MachineOperand::MO_GlobalAddress:
1072 CalleeMCOp = MCIL.LowerSymbolOperand(MO: CalleeMO,
1073 Sym: MCIL.GetSymbolFromOperand(MO: CalleeMO));
1074 break;
1075 }
1076
1077 // Emit MOV to materialize the target address and the CALL to target.
1078 // This is encoded with 12-13 bytes, depending on which register is used.
1079 Register ScratchReg = MI.getOperand(i: ScratchIdx).getReg();
1080 if (X86II::isX86_64ExtendedReg(Reg: ScratchReg))
1081 EncodedBytes = 13;
1082 else
1083 EncodedBytes = 12;
1084
1085 EmitAndCountInstruction(
1086 Inst&: MCInstBuilder(X86::MOV64ri).addReg(Reg: ScratchReg).addOperand(Op: CalleeMCOp));
1087 // FIXME: Add retpoline support and remove this.
1088 if (Subtarget->useIndirectThunkCalls())
1089 report_fatal_error(
1090 reason: "Lowering patchpoint with thunks not yet implemented.");
1091 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64r).addReg(Reg: ScratchReg));
1092 }
1093
1094 // Emit padding.
1095 unsigned NumBytes = opers.getNumPatchBytes();
1096 assert(NumBytes >= EncodedBytes &&
1097 "Patchpoint can't request size less than the length of a call.");
1098
1099 emitX86Nops(OS&: *OutStreamer, NumBytes: NumBytes - EncodedBytes, Subtarget);
1100}
1101
1102void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
1103 X86MCInstLower &MCIL) {
1104 assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64");
1105
1106 NoAutoPaddingScope NoPadScope(*OutStreamer);
1107
1108 // We want to emit the following pattern, which follows the x86 calling
1109 // convention to prepare for the trampoline call to be patched in.
1110 //
1111 // .p2align 1, ...
1112 // .Lxray_event_sled_N:
1113 // jmp +N // jump across the instrumentation sled
1114 // ... // set up arguments in register
1115 // callq __xray_CustomEvent@plt // force dependency to symbol
1116 // ...
1117 // <jump here>
1118 //
1119 // After patching, it would look something like:
1120 //
1121 // nopw (2-byte nop)
1122 // ...
1123 // callq __xrayCustomEvent // already lowered
1124 // ...
1125 //
1126 // ---
1127 // First we emit the label and the jump.
1128 auto CurSled = OutContext.createTempSymbol(Name: "xray_event_sled_", AlwaysAddSuffix: true);
1129 OutStreamer->AddComment(T: "# XRay Custom Event Log");
1130 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1131 OutStreamer->emitLabel(Symbol: CurSled);
1132
1133 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1134 // an operand (computed as an offset from the jmp instruction).
1135 // FIXME: Find another less hacky way do force the relative jump.
1136 OutStreamer->emitBinaryData(Data: "\xeb\x0f");
1137
1138 // The default C calling convention will place two arguments into %rcx and
1139 // %rdx -- so we only work with those.
1140 const Register DestRegs[] = {X86::RDI, X86::RSI};
1141 bool UsedMask[] = {false, false};
1142 // Filled out in loop.
1143 Register SrcRegs[] = {0, 0};
1144
1145 // Then we put the operands in the %rdi and %rsi registers. We spill the
1146 // values in the register before we clobber them, and mark them as used in
1147 // UsedMask. In case the arguments are already in the correct register, we use
1148 // emit nops appropriately sized to keep the sled the same size in every
1149 // situation.
1150 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1151 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO: MI.getOperand(i: I));
1152 Op.isValid()) {
1153 assert(Op.isReg() && "Only support arguments in registers");
1154 SrcRegs[I] = getX86SubSuperRegister(Reg: Op.getReg(), Size: 64);
1155 assert(SrcRegs[I].isValid() && "Invalid operand");
1156 if (SrcRegs[I] != DestRegs[I]) {
1157 UsedMask[I] = true;
1158 EmitAndCountInstruction(
1159 Inst&: MCInstBuilder(X86::PUSH64r).addReg(Reg: DestRegs[I]));
1160 } else {
1161 emitX86Nops(OS&: *OutStreamer, NumBytes: 4, Subtarget);
1162 }
1163 }
1164
1165 // Now that the register values are stashed, mov arguments into place.
1166 // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1167 // earlier DestReg. We will have already overwritten over the register before
1168 // we can copy from it.
1169 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1170 if (SrcRegs[I] != DestRegs[I])
1171 EmitAndCountInstruction(
1172 Inst&: MCInstBuilder(X86::MOV64rr).addReg(Reg: DestRegs[I]).addReg(Reg: SrcRegs[I]));
1173
1174 // We emit a hard dependency on the __xray_CustomEvent symbol, which is the
1175 // name of the trampoline to be implemented by the XRay runtime.
1176 auto TSym = OutContext.getOrCreateSymbol(Name: "__xray_CustomEvent");
1177 MachineOperand TOp = MachineOperand::CreateMCSymbol(Sym: TSym);
1178 if (isPositionIndependent())
1179 TOp.setTargetFlags(X86II::MO_PLT);
1180
1181 // Emit the call instruction.
1182 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64pcrel32)
1183 .addOperand(Op: MCIL.LowerSymbolOperand(MO: TOp, Sym: TSym)));
1184
1185 // Restore caller-saved and used registers.
1186 for (unsigned I = sizeof UsedMask; I-- > 0;)
1187 if (UsedMask[I])
1188 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::POP64r).addReg(Reg: DestRegs[I]));
1189 else
1190 emitX86Nops(OS&: *OutStreamer, NumBytes: 1, Subtarget);
1191
1192 OutStreamer->AddComment(T: "xray custom event end.");
1193
1194 // Record the sled version. Version 0 of this sled was spelled differently, so
1195 // we let the runtime handle the different offsets we're using. Version 2
1196 // changed the absolute address to a PC-relative address.
1197 recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2);
1198}
1199
1200void X86AsmPrinter::LowerPATCHABLE_TYPED_EVENT_CALL(const MachineInstr &MI,
1201 X86MCInstLower &MCIL) {
1202 assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64");
1203
1204 NoAutoPaddingScope NoPadScope(*OutStreamer);
1205
1206 // We want to emit the following pattern, which follows the x86 calling
1207 // convention to prepare for the trampoline call to be patched in.
1208 //
1209 // .p2align 1, ...
1210 // .Lxray_event_sled_N:
1211 // jmp +N // jump across the instrumentation sled
1212 // ... // set up arguments in register
1213 // callq __xray_TypedEvent@plt // force dependency to symbol
1214 // ...
1215 // <jump here>
1216 //
1217 // After patching, it would look something like:
1218 //
1219 // nopw (2-byte nop)
1220 // ...
1221 // callq __xrayTypedEvent // already lowered
1222 // ...
1223 //
1224 // ---
1225 // First we emit the label and the jump.
1226 auto CurSled = OutContext.createTempSymbol(Name: "xray_typed_event_sled_", AlwaysAddSuffix: true);
1227 OutStreamer->AddComment(T: "# XRay Typed Event Log");
1228 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1229 OutStreamer->emitLabel(Symbol: CurSled);
1230
1231 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1232 // an operand (computed as an offset from the jmp instruction).
1233 // FIXME: Find another less hacky way do force the relative jump.
1234 OutStreamer->emitBinaryData(Data: "\xeb\x14");
1235
1236 // An x86-64 convention may place three arguments into %rcx, %rdx, and R8,
1237 // so we'll work with those. Or we may be called via SystemV, in which case
1238 // we don't have to do any translation.
1239 const Register DestRegs[] = {X86::RDI, X86::RSI, X86::RDX};
1240 bool UsedMask[] = {false, false, false};
1241
1242 // Will fill out src regs in the loop.
1243 Register SrcRegs[] = {0, 0, 0};
1244
1245 // Then we put the operands in the SystemV registers. We spill the values in
1246 // the registers before we clobber them, and mark them as used in UsedMask.
1247 // In case the arguments are already in the correct register, we emit nops
1248 // appropriately sized to keep the sled the same size in every situation.
1249 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1250 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO: MI.getOperand(i: I));
1251 Op.isValid()) {
1252 // TODO: Is register only support adequate?
1253 assert(Op.isReg() && "Only supports arguments in registers");
1254 SrcRegs[I] = getX86SubSuperRegister(Reg: Op.getReg(), Size: 64);
1255 assert(SrcRegs[I].isValid() && "Invalid operand");
1256 if (SrcRegs[I] != DestRegs[I]) {
1257 UsedMask[I] = true;
1258 EmitAndCountInstruction(
1259 Inst&: MCInstBuilder(X86::PUSH64r).addReg(Reg: DestRegs[I]));
1260 } else {
1261 emitX86Nops(OS&: *OutStreamer, NumBytes: 4, Subtarget);
1262 }
1263 }
1264
1265 // In the above loop we only stash all of the destination registers or emit
1266 // nops if the arguments are already in the right place. Doing the actually
1267 // moving is postponed until after all the registers are stashed so nothing
1268 // is clobbers. We've already added nops to account for the size of mov and
1269 // push if the register is in the right place, so we only have to worry about
1270 // emitting movs.
1271 // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1272 // earlier DestReg. We will have already overwritten over the register before
1273 // we can copy from it.
1274 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1275 if (UsedMask[I])
1276 EmitAndCountInstruction(
1277 Inst&: MCInstBuilder(X86::MOV64rr).addReg(Reg: DestRegs[I]).addReg(Reg: SrcRegs[I]));
1278
1279 // We emit a hard dependency on the __xray_TypedEvent symbol, which is the
1280 // name of the trampoline to be implemented by the XRay runtime.
1281 auto TSym = OutContext.getOrCreateSymbol(Name: "__xray_TypedEvent");
1282 MachineOperand TOp = MachineOperand::CreateMCSymbol(Sym: TSym);
1283 if (isPositionIndependent())
1284 TOp.setTargetFlags(X86II::MO_PLT);
1285
1286 // Emit the call instruction.
1287 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64pcrel32)
1288 .addOperand(Op: MCIL.LowerSymbolOperand(MO: TOp, Sym: TSym)));
1289
1290 // Restore caller-saved and used registers.
1291 for (unsigned I = sizeof UsedMask; I-- > 0;)
1292 if (UsedMask[I])
1293 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::POP64r).addReg(Reg: DestRegs[I]));
1294 else
1295 emitX86Nops(OS&: *OutStreamer, NumBytes: 1, Subtarget);
1296
1297 OutStreamer->AddComment(T: "xray typed event end.");
1298
1299 // Record the sled version.
1300 recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2);
1301}
1302
1303void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
1304 X86MCInstLower &MCIL) {
1305
1306 NoAutoPaddingScope NoPadScope(*OutStreamer);
1307
1308 const Function &F = MF->getFunction();
1309 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
1310 unsigned Num = F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry");
1311 emitX86Nops(OS&: *OutStreamer, NumBytes: Num, Subtarget);
1312 return;
1313 }
1314 // We want to emit the following pattern:
1315 //
1316 // .p2align 1, ...
1317 // .Lxray_sled_N:
1318 // jmp .tmpN
1319 // # 9 bytes worth of noops
1320 //
1321 // We need the 9 bytes because at runtime, we'd be patching over the full 11
1322 // bytes with the following pattern:
1323 //
1324 // mov %r10, <function id, 32-bit> // 6 bytes
1325 // call <relative offset, 32-bits> // 5 bytes
1326 //
1327 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
1328 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1329 OutStreamer->emitLabel(Symbol: CurSled);
1330
1331 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1332 // an operand (computed as an offset from the jmp instruction).
1333 // FIXME: Find another less hacky way do force the relative jump.
1334 OutStreamer->emitBytes(Data: "\xeb\x09");
1335 emitX86Nops(OS&: *OutStreamer, NumBytes: 9, Subtarget);
1336 recordSled(Sled: CurSled, MI, Kind: SledKind::FUNCTION_ENTER, Version: 2);
1337}
1338
1339void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
1340 X86MCInstLower &MCIL) {
1341 NoAutoPaddingScope NoPadScope(*OutStreamer);
1342
1343 // Since PATCHABLE_RET takes the opcode of the return statement as an
1344 // argument, we use that to emit the correct form of the RET that we want.
1345 // i.e. when we see this:
1346 //
1347 // PATCHABLE_RET X86::RET ...
1348 //
1349 // We should emit the RET followed by sleds.
1350 //
1351 // .p2align 1, ...
1352 // .Lxray_sled_N:
1353 // ret # or equivalent instruction
1354 // # 10 bytes worth of noops
1355 //
1356 // This just makes sure that the alignment for the next instruction is 2.
1357 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
1358 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1359 OutStreamer->emitLabel(Symbol: CurSled);
1360 unsigned OpCode = MI.getOperand(i: 0).getImm();
1361 MCInst Ret;
1362 Ret.setOpcode(OpCode);
1363 for (auto &MO : drop_begin(RangeOrContainer: MI.operands()))
1364 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO); Op.isValid())
1365 Ret.addOperand(Op);
1366 OutStreamer->emitInstruction(Inst: Ret, STI: getSubtargetInfo());
1367 emitX86Nops(OS&: *OutStreamer, NumBytes: 10, Subtarget);
1368 recordSled(Sled: CurSled, MI, Kind: SledKind::FUNCTION_EXIT, Version: 2);
1369}
1370
1371void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI,
1372 X86MCInstLower &MCIL) {
1373 MCInst TC;
1374 TC.setOpcode(convertTailJumpOpcode(Opcode: MI.getOperand(i: 0).getImm()));
1375 // Drop the tail jump opcode.
1376 auto TCOperands = drop_begin(RangeOrContainer: MI.operands());
1377 bool IsConditional = TC.getOpcode() == X86::JCC_1;
1378 MCSymbol *FallthroughLabel;
1379 if (IsConditional) {
1380 // Rewrite:
1381 // je target
1382 //
1383 // To:
1384 // jne .fallthrough
1385 // .p2align 1, ...
1386 // .Lxray_sled_N:
1387 // SLED_CODE
1388 // jmp target
1389 // .fallthrough:
1390 FallthroughLabel = OutContext.createTempSymbol();
1391 EmitToStreamer(
1392 S&: *OutStreamer,
1393 Inst: MCInstBuilder(X86::JCC_1)
1394 .addExpr(Val: MCSymbolRefExpr::create(Symbol: FallthroughLabel, Ctx&: OutContext))
1395 .addImm(Val: X86::GetOppositeBranchCondition(
1396 CC: static_cast<X86::CondCode>(MI.getOperand(i: 2).getImm()))));
1397 TC.setOpcode(X86::JMP_1);
1398 // Drop the condition code.
1399 TCOperands = drop_end(RangeOrContainer&: TCOperands);
1400 }
1401
1402 NoAutoPaddingScope NoPadScope(*OutStreamer);
1403
1404 // Like PATCHABLE_RET, we have the actual instruction in the operands to this
1405 // instruction so we lower that particular instruction and its operands.
1406 // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how
1407 // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to
1408 // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual
1409 // tail call much like how we have it in PATCHABLE_RET.
1410 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
1411 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1412 OutStreamer->emitLabel(Symbol: CurSled);
1413 auto Target = OutContext.createTempSymbol();
1414
1415 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1416 // an operand (computed as an offset from the jmp instruction).
1417 // FIXME: Find another less hacky way do force the relative jump.
1418 OutStreamer->emitBytes(Data: "\xeb\x09");
1419 emitX86Nops(OS&: *OutStreamer, NumBytes: 9, Subtarget);
1420 OutStreamer->emitLabel(Symbol: Target);
1421 recordSled(Sled: CurSled, MI, Kind: SledKind::TAIL_CALL, Version: 2);
1422
1423 // Before emitting the instruction, add a comment to indicate that this is
1424 // indeed a tail call.
1425 OutStreamer->AddComment(T: "TAILCALL");
1426 for (auto &MO : TCOperands)
1427 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO); Op.isValid())
1428 TC.addOperand(Op);
1429 OutStreamer->emitInstruction(Inst: TC, STI: getSubtargetInfo());
1430
1431 if (IsConditional)
1432 OutStreamer->emitLabel(Symbol: FallthroughLabel);
1433}
1434
1435static unsigned getSrcIdx(const MachineInstr* MI, unsigned SrcIdx) {
1436 if (X86II::isKMasked(TSFlags: MI->getDesc().TSFlags)) {
1437 // Skip mask operand.
1438 ++SrcIdx;
1439 if (X86II::isKMergeMasked(TSFlags: MI->getDesc().TSFlags)) {
1440 // Skip passthru operand.
1441 ++SrcIdx;
1442 }
1443 }
1444 return SrcIdx;
1445}
1446
1447static void printDstRegisterName(raw_ostream &CS, const MachineInstr *MI,
1448 unsigned SrcOpIdx) {
1449 const MachineOperand &DstOp = MI->getOperand(i: 0);
1450 CS << X86ATTInstPrinter::getRegisterName(Reg: DstOp.getReg());
1451
1452 // Handle AVX512 MASK/MASXZ write mask comments.
1453 // MASK: zmmX {%kY}
1454 // MASKZ: zmmX {%kY} {z}
1455 if (X86II::isKMasked(TSFlags: MI->getDesc().TSFlags)) {
1456 const MachineOperand &WriteMaskOp = MI->getOperand(i: SrcOpIdx - 1);
1457 StringRef Mask = X86ATTInstPrinter::getRegisterName(Reg: WriteMaskOp.getReg());
1458 CS << " {%" << Mask << "}";
1459 if (!X86II::isKMergeMasked(TSFlags: MI->getDesc().TSFlags)) {
1460 CS << " {z}";
1461 }
1462 }
1463}
1464
1465static void printShuffleMask(raw_ostream &CS, StringRef Src1Name,
1466 StringRef Src2Name, ArrayRef<int> Mask) {
1467 // One source operand, fix the mask to print all elements in one span.
1468 SmallVector<int, 8> ShuffleMask(Mask);
1469 if (Src1Name == Src2Name)
1470 for (int i = 0, e = ShuffleMask.size(); i != e; ++i)
1471 if (ShuffleMask[i] >= e)
1472 ShuffleMask[i] -= e;
1473
1474 for (int i = 0, e = ShuffleMask.size(); i != e; ++i) {
1475 if (i != 0)
1476 CS << ",";
1477 if (ShuffleMask[i] == SM_SentinelZero) {
1478 CS << "zero";
1479 continue;
1480 }
1481
1482 // Otherwise, it must come from src1 or src2. Print the span of elements
1483 // that comes from this src.
1484 bool isSrc1 = ShuffleMask[i] < (int)e;
1485 CS << (isSrc1 ? Src1Name : Src2Name) << '[';
1486
1487 bool IsFirst = true;
1488 while (i != e && ShuffleMask[i] != SM_SentinelZero &&
1489 (ShuffleMask[i] < (int)e) == isSrc1) {
1490 if (!IsFirst)
1491 CS << ',';
1492 else
1493 IsFirst = false;
1494 if (ShuffleMask[i] == SM_SentinelUndef)
1495 CS << "u";
1496 else
1497 CS << ShuffleMask[i] % (int)e;
1498 ++i;
1499 }
1500 CS << ']';
1501 --i; // For loop increments element #.
1502 }
1503}
1504
1505static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx,
1506 unsigned SrcOp2Idx, ArrayRef<int> Mask) {
1507 std::string Comment;
1508
1509 const MachineOperand &SrcOp1 = MI->getOperand(i: SrcOp1Idx);
1510 const MachineOperand &SrcOp2 = MI->getOperand(i: SrcOp2Idx);
1511 StringRef Src1Name = SrcOp1.isReg()
1512 ? X86ATTInstPrinter::getRegisterName(Reg: SrcOp1.getReg())
1513 : "mem";
1514 StringRef Src2Name = SrcOp2.isReg()
1515 ? X86ATTInstPrinter::getRegisterName(Reg: SrcOp2.getReg())
1516 : "mem";
1517
1518 raw_string_ostream CS(Comment);
1519 printDstRegisterName(CS, MI, SrcOpIdx: SrcOp1Idx);
1520 CS << " = ";
1521 printShuffleMask(CS, Src1Name, Src2Name, Mask);
1522
1523 return Comment;
1524}
1525
1526static void printConstant(const APInt &Val, raw_ostream &CS,
1527 bool PrintZero = false) {
1528 if (Val.getBitWidth() <= 64) {
1529 CS << (PrintZero ? 0ULL : Val.getZExtValue());
1530 } else {
1531 // print multi-word constant as (w0,w1)
1532 CS << "(";
1533 for (int i = 0, N = Val.getNumWords(); i < N; ++i) {
1534 if (i > 0)
1535 CS << ",";
1536 CS << (PrintZero ? 0ULL : Val.getRawData()[i]);
1537 }
1538 CS << ")";
1539 }
1540}
1541
1542static void printConstant(const APFloat &Flt, raw_ostream &CS,
1543 bool PrintZero = false) {
1544 SmallString<32> Str;
1545 // Force scientific notation to distinguish from integers.
1546 if (PrintZero)
1547 APFloat::getZero(Sem: Flt.getSemantics()).toString(Str, FormatPrecision: 0, FormatMaxPadding: 0);
1548 else
1549 Flt.toString(Str, FormatPrecision: 0, FormatMaxPadding: 0);
1550 CS << Str;
1551}
1552
1553static void printConstant(const Constant *COp, unsigned BitWidth,
1554 raw_ostream &CS, bool PrintZero = false) {
1555 if (isa<UndefValue>(Val: COp)) {
1556 CS << "u";
1557 } else if (auto *CI = dyn_cast<ConstantInt>(Val: COp)) {
1558 if (auto VTy = dyn_cast<FixedVectorType>(Val: CI->getType())) {
1559 for (unsigned I = 0, E = VTy->getNumElements(); I != E; ++I) {
1560 if (I != 0)
1561 CS << ',';
1562 printConstant(Val: CI->getValue(), CS, PrintZero);
1563 }
1564 } else
1565 printConstant(Val: CI->getValue(), CS, PrintZero);
1566 } else if (auto *CF = dyn_cast<ConstantFP>(Val: COp)) {
1567 if (auto VTy = dyn_cast<FixedVectorType>(Val: CF->getType())) {
1568 unsigned EltBits = VTy->getScalarSizeInBits();
1569 unsigned E = std::min(a: BitWidth / EltBits, b: VTy->getNumElements());
1570 if ((BitWidth % EltBits) == 0) {
1571 for (unsigned I = 0; I != E; ++I) {
1572 if (I != 0)
1573 CS << ",";
1574 printConstant(Flt: CF->getValueAPF(), CS, PrintZero);
1575 }
1576 } else {
1577 CS << "?";
1578 }
1579 } else
1580 printConstant(Flt: CF->getValueAPF(), CS, PrintZero);
1581 } else if (auto *CDS = dyn_cast<ConstantDataSequential>(Val: COp)) {
1582 Type *EltTy = CDS->getElementType();
1583 bool IsInteger = EltTy->isIntegerTy();
1584 bool IsFP = EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy();
1585 unsigned EltBits = EltTy->getPrimitiveSizeInBits();
1586 unsigned E = std::min(a: BitWidth / EltBits, b: (unsigned)CDS->getNumElements());
1587 if ((BitWidth % EltBits) == 0) {
1588 for (unsigned I = 0; I != E; ++I) {
1589 if (I != 0)
1590 CS << ",";
1591 if (IsInteger)
1592 printConstant(Val: CDS->getElementAsAPInt(i: I), CS, PrintZero);
1593 else if (IsFP)
1594 printConstant(Flt: CDS->getElementAsAPFloat(i: I), CS, PrintZero);
1595 else
1596 CS << "?";
1597 }
1598 } else {
1599 CS << "?";
1600 }
1601 } else if (auto *CV = dyn_cast<ConstantVector>(Val: COp)) {
1602 unsigned EltBits = CV->getType()->getScalarSizeInBits();
1603 unsigned E = std::min(a: BitWidth / EltBits, b: CV->getNumOperands());
1604 if ((BitWidth % EltBits) == 0) {
1605 for (unsigned I = 0; I != E; ++I) {
1606 if (I != 0)
1607 CS << ",";
1608 printConstant(COp: CV->getOperand(i_nocapture: I), BitWidth: EltBits, CS, PrintZero);
1609 }
1610 } else {
1611 CS << "?";
1612 }
1613 } else {
1614 CS << "?";
1615 }
1616}
1617
1618static void printZeroUpperMove(const MachineInstr *MI, MCStreamer &OutStreamer,
1619 int SclWidth, int VecWidth,
1620 const char *ShuffleComment) {
1621 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1622
1623 std::string Comment;
1624 raw_string_ostream CS(Comment);
1625 printDstRegisterName(CS, MI, SrcOpIdx: SrcIdx);
1626 CS << " = ";
1627
1628 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx)) {
1629 CS << "[";
1630 printConstant(COp: C, BitWidth: SclWidth, CS);
1631 for (int I = 1, E = VecWidth / SclWidth; I < E; ++I) {
1632 CS << ",";
1633 printConstant(COp: C, BitWidth: SclWidth, CS, PrintZero: true);
1634 }
1635 CS << "]";
1636 OutStreamer.AddComment(T: CS.str());
1637 return; // early-out
1638 }
1639
1640 // We didn't find a constant load, fallback to a shuffle mask decode.
1641 CS << ShuffleComment;
1642 OutStreamer.AddComment(T: CS.str());
1643}
1644
1645static void printBroadcast(const MachineInstr *MI, MCStreamer &OutStreamer,
1646 int Repeats, int BitWidth) {
1647 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1648 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx)) {
1649 std::string Comment;
1650 raw_string_ostream CS(Comment);
1651 printDstRegisterName(CS, MI, SrcOpIdx: SrcIdx);
1652 CS << " = [";
1653 for (int l = 0; l != Repeats; ++l) {
1654 if (l != 0)
1655 CS << ",";
1656 printConstant(COp: C, BitWidth, CS);
1657 }
1658 CS << "]";
1659 OutStreamer.AddComment(T: CS.str());
1660 }
1661}
1662
1663static void addConstantComment(const MachineInstr *MI, MCStreamer &OutStreamer,
1664 unsigned OpNo, int BitWidth, int Repeats = 1) {
1665 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo)) {
1666 std::string Comment;
1667 raw_string_ostream CS(Comment);
1668 CS << "[";
1669 for (int I = 0; I != Repeats; ++I) {
1670 if (I != 0)
1671 CS << ",";
1672 printConstant(COp: C, BitWidth, CS);
1673 }
1674 CS << "]";
1675 OutStreamer.AddComment(T: CS.str());
1676 }
1677}
1678
1679static bool printExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1680 int SrcEltBits, int DstEltBits, bool IsSext) {
1681 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1682 auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx);
1683 if (C && C->getType()->getScalarSizeInBits() == unsigned(SrcEltBits)) {
1684 if (auto *CDS = dyn_cast<ConstantDataSequential>(Val: C)) {
1685 int NumElts = CDS->getNumElements();
1686 std::string Comment;
1687 raw_string_ostream CS(Comment);
1688 printDstRegisterName(CS, MI, SrcOpIdx: SrcIdx);
1689 CS << " = [";
1690 for (int i = 0; i != NumElts; ++i) {
1691 if (i != 0)
1692 CS << ",";
1693 if (CDS->getElementType()->isIntegerTy()) {
1694 APInt Elt = CDS->getElementAsAPInt(i);
1695 Elt = IsSext ? Elt.sext(width: DstEltBits) : Elt.zext(width: DstEltBits);
1696 printConstant(Val: Elt, CS);
1697 } else
1698 CS << "?";
1699 }
1700 CS << "]";
1701 OutStreamer.AddComment(T: CS.str());
1702 return true;
1703 }
1704 }
1705
1706 return false;
1707}
1708static void printSignExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1709 int SrcEltBits, int DstEltBits) {
1710 printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, IsSext: true);
1711}
1712static void printZeroExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1713 int SrcEltBits, int DstEltBits) {
1714 if (printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, IsSext: false))
1715 return;
1716
1717 // We didn't find a constant load, fallback to a shuffle mask decode.
1718 std::string Comment;
1719 raw_string_ostream CS(Comment);
1720 printDstRegisterName(CS, MI, SrcOpIdx: getSrcIdx(MI, SrcIdx: 1));
1721 CS << " = ";
1722
1723 SmallVector<int> Mask;
1724 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1725 assert((Width % DstEltBits) == 0 && (DstEltBits % SrcEltBits) == 0 &&
1726 "Illegal extension ratio");
1727 DecodeZeroExtendMask(SrcScalarBits: SrcEltBits, DstScalarBits: DstEltBits, NumDstElts: Width / DstEltBits, IsAnyExtend: false, ShuffleMask&: Mask);
1728 printShuffleMask(CS, Src1Name: "mem", Src2Name: "", Mask);
1729
1730 OutStreamer.AddComment(T: CS.str());
1731}
1732
1733void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) {
1734 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
1735 assert((getSubtarget().isOSWindows() || getSubtarget().isUEFI()) &&
1736 "SEH_ instruction Windows and UEFI only");
1737
1738 // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86.
1739 if (EmitFPOData) {
1740 X86TargetStreamer *XTS =
1741 static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer());
1742 switch (MI->getOpcode()) {
1743 case X86::SEH_PushReg:
1744 XTS->emitFPOPushReg(Reg: MI->getOperand(i: 0).getImm());
1745 break;
1746 case X86::SEH_StackAlloc:
1747 XTS->emitFPOStackAlloc(StackAlloc: MI->getOperand(i: 0).getImm());
1748 break;
1749 case X86::SEH_StackAlign:
1750 XTS->emitFPOStackAlign(Align: MI->getOperand(i: 0).getImm());
1751 break;
1752 case X86::SEH_SetFrame:
1753 assert(MI->getOperand(1).getImm() == 0 &&
1754 ".cv_fpo_setframe takes no offset");
1755 XTS->emitFPOSetFrame(Reg: MI->getOperand(i: 0).getImm());
1756 break;
1757 case X86::SEH_EndPrologue:
1758 XTS->emitFPOEndPrologue();
1759 break;
1760 case X86::SEH_SaveReg:
1761 case X86::SEH_SaveXMM:
1762 case X86::SEH_PushFrame:
1763 case X86::SEH_Push2Regs:
1764 llvm_unreachable("SEH_ directive incompatible with FPO");
1765 break;
1766 default:
1767 llvm_unreachable("expected SEH_ instruction");
1768 }
1769 return;
1770 }
1771
1772 // Otherwise, use the .seh_ directives for all other Windows platforms.
1773 switch (MI->getOpcode()) {
1774 case X86::SEH_PushReg:
1775 OutStreamer->emitWinCFIPushReg(Register: MI->getOperand(i: 0).getImm());
1776 break;
1777
1778 case X86::SEH_Push2Regs:
1779 OutStreamer->emitWinCFIPush2Regs(Reg1: MI->getOperand(i: 0).getImm(),
1780 Reg2: MI->getOperand(i: 1).getImm());
1781 break;
1782
1783 case X86::SEH_SaveReg:
1784 OutStreamer->emitWinCFISaveReg(Register: MI->getOperand(i: 0).getImm(),
1785 Offset: MI->getOperand(i: 1).getImm());
1786 break;
1787
1788 case X86::SEH_SaveXMM:
1789 OutStreamer->emitWinCFISaveXMM(Register: MI->getOperand(i: 0).getImm(),
1790 Offset: MI->getOperand(i: 1).getImm());
1791 break;
1792
1793 case X86::SEH_StackAlloc:
1794 OutStreamer->emitWinCFIAllocStack(Size: MI->getOperand(i: 0).getImm());
1795 break;
1796
1797 case X86::SEH_SetFrame:
1798 OutStreamer->emitWinCFISetFrame(Register: MI->getOperand(i: 0).getImm(),
1799 Offset: MI->getOperand(i: 1).getImm());
1800 break;
1801
1802 case X86::SEH_PushFrame:
1803 OutStreamer->emitWinCFIPushFrame(Code: MI->getOperand(i: 0).getImm());
1804 break;
1805
1806 case X86::SEH_EndPrologue:
1807 OutStreamer->emitWinCFIEndProlog();
1808 break;
1809
1810 case X86::SEH_BeginEpilogue:
1811 OutStreamer->emitWinCFIBeginEpilogue();
1812 break;
1813
1814 case X86::SEH_EndEpilogue:
1815 OutStreamer->emitWinCFIEndEpilogue();
1816 break;
1817
1818 case X86::SEH_UnwindV2Start:
1819 OutStreamer->emitWinCFIUnwindV2Start();
1820 break;
1821
1822 case X86::SEH_UnwindVersion:
1823 OutStreamer->emitWinCFIUnwindVersion(Version: MI->getOperand(i: 0).getImm());
1824 break;
1825
1826 default:
1827 llvm_unreachable("expected SEH_ instruction");
1828 }
1829}
1830
1831static void addConstantComments(const MachineInstr *MI,
1832 MCStreamer &OutStreamer) {
1833 switch (MI->getOpcode()) {
1834 // Lower PSHUFB and VPERMILP normally but add a comment if we can find
1835 // a constant shuffle mask. We won't be able to do this at the MC layer
1836 // because the mask isn't an immediate.
1837 case X86::PSHUFBrm:
1838 case X86::VPSHUFBrm:
1839 case X86::VPSHUFBYrm:
1840 case X86::VPSHUFBZ128rm:
1841 case X86::VPSHUFBZ128rmk:
1842 case X86::VPSHUFBZ128rmkz:
1843 case X86::VPSHUFBZ256rm:
1844 case X86::VPSHUFBZ256rmk:
1845 case X86::VPSHUFBZ256rmkz:
1846 case X86::VPSHUFBZrm:
1847 case X86::VPSHUFBZrmk:
1848 case X86::VPSHUFBZrmkz: {
1849 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1850 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx + 1)) {
1851 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1852 SmallVector<int, 64> Mask;
1853 DecodePSHUFBMask(C, Width, ShuffleMask&: Mask);
1854 if (!Mask.empty())
1855 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: SrcIdx, SrcOp2Idx: SrcIdx, Mask));
1856 }
1857 break;
1858 }
1859
1860 case X86::VPERMILPSrm:
1861 case X86::VPERMILPSYrm:
1862 case X86::VPERMILPSZ128rm:
1863 case X86::VPERMILPSZ128rmk:
1864 case X86::VPERMILPSZ128rmkz:
1865 case X86::VPERMILPSZ256rm:
1866 case X86::VPERMILPSZ256rmk:
1867 case X86::VPERMILPSZ256rmkz:
1868 case X86::VPERMILPSZrm:
1869 case X86::VPERMILPSZrmk:
1870 case X86::VPERMILPSZrmkz: {
1871 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1872 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx + 1)) {
1873 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1874 SmallVector<int, 16> Mask;
1875 DecodeVPERMILPMask(C, ElSize: 32, Width, ShuffleMask&: Mask);
1876 if (!Mask.empty())
1877 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: SrcIdx, SrcOp2Idx: SrcIdx, Mask));
1878 }
1879 break;
1880 }
1881 case X86::VPERMILPDrm:
1882 case X86::VPERMILPDYrm:
1883 case X86::VPERMILPDZ128rm:
1884 case X86::VPERMILPDZ128rmk:
1885 case X86::VPERMILPDZ128rmkz:
1886 case X86::VPERMILPDZ256rm:
1887 case X86::VPERMILPDZ256rmk:
1888 case X86::VPERMILPDZ256rmkz:
1889 case X86::VPERMILPDZrm:
1890 case X86::VPERMILPDZrmk:
1891 case X86::VPERMILPDZrmkz: {
1892 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1893 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx + 1)) {
1894 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1895 SmallVector<int, 16> Mask;
1896 DecodeVPERMILPMask(C, ElSize: 64, Width, ShuffleMask&: Mask);
1897 if (!Mask.empty())
1898 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: SrcIdx, SrcOp2Idx: SrcIdx, Mask));
1899 }
1900 break;
1901 }
1902
1903 case X86::VPERMIL2PDrm:
1904 case X86::VPERMIL2PSrm:
1905 case X86::VPERMIL2PDYrm:
1906 case X86::VPERMIL2PSYrm: {
1907 assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands + 1) &&
1908 "Unexpected number of operands!");
1909
1910 const MachineOperand &CtrlOp = MI->getOperand(i: MI->getNumOperands() - 1);
1911 if (!CtrlOp.isImm())
1912 break;
1913
1914 unsigned ElSize;
1915 switch (MI->getOpcode()) {
1916 default: llvm_unreachable("Invalid opcode");
1917 case X86::VPERMIL2PSrm: case X86::VPERMIL2PSYrm: ElSize = 32; break;
1918 case X86::VPERMIL2PDrm: case X86::VPERMIL2PDYrm: ElSize = 64; break;
1919 }
1920
1921 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: 3)) {
1922 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1923 SmallVector<int, 16> Mask;
1924 DecodeVPERMIL2PMask(C, M2Z: (unsigned)CtrlOp.getImm(), ElSize, Width, ShuffleMask&: Mask);
1925 if (!Mask.empty())
1926 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: 1, SrcOp2Idx: 2, Mask));
1927 }
1928 break;
1929 }
1930
1931 case X86::VPPERMrrm: {
1932 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: 3)) {
1933 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1934 SmallVector<int, 16> Mask;
1935 DecodeVPPERMMask(C, Width, ShuffleMask&: Mask);
1936 if (!Mask.empty())
1937 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: 1, SrcOp2Idx: 2, Mask));
1938 }
1939 break;
1940 }
1941
1942 case X86::MMX_MOVQ64rm: {
1943 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: 1)) {
1944 std::string Comment;
1945 raw_string_ostream CS(Comment);
1946 const MachineOperand &DstOp = MI->getOperand(i: 0);
1947 CS << X86ATTInstPrinter::getRegisterName(Reg: DstOp.getReg()) << " = ";
1948 if (auto *CF = dyn_cast<ConstantFP>(Val: C)) {
1949 CS << "0x" << toString(I: CF->getValueAPF().bitcastToAPInt(), Radix: 16, Signed: false);
1950 OutStreamer.AddComment(T: CS.str());
1951 }
1952 }
1953 break;
1954 }
1955
1956 case X86::GF2P8AFFINEQBrmi:
1957 case X86::VGF2P8AFFINEQBrmi:
1958 case X86::VGF2P8AFFINEQBYrmi:
1959 case X86::VGF2P8AFFINEQBZrmi:
1960 case X86::VGF2P8AFFINEQBZ128rmi:
1961 case X86::VGF2P8AFFINEQBZ256rmi: {
1962 // TODO: Add predicate handling with test coverage.
1963 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1964 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1965 addConstantComment(MI, OutStreamer, OpNo: SrcIdx + 1, BitWidth: Width);
1966 break;
1967 }
1968
1969 case X86::VGF2P8AFFINEQBZ128rmbi:
1970 case X86::VGF2P8AFFINEQBZ256rmbi:
1971 case X86::VGF2P8AFFINEQBZrmbi: {
1972 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1973 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1974 addConstantComment(MI, OutStreamer, OpNo: SrcIdx + 1, BitWidth: 64, Repeats: Width / 64);
1975 break;
1976 }
1977
1978#define INSTR_CASE(Prefix, Instr, Suffix, Postfix) \
1979 case X86::Prefix##Instr##Suffix##rm##Postfix:
1980
1981#define CASE_AVX512_ARITH_RM(Instr) \
1982 INSTR_CASE(V, Instr, Z128, ) \
1983 INSTR_CASE(V, Instr, Z128, k) \
1984 INSTR_CASE(V, Instr, Z128, kz) \
1985 INSTR_CASE(V, Instr, Z256, ) \
1986 INSTR_CASE(V, Instr, Z256, k) \
1987 INSTR_CASE(V, Instr, Z256, kz) \
1988 INSTR_CASE(V, Instr, Z, ) \
1989 INSTR_CASE(V, Instr, Z, k) \
1990 INSTR_CASE(V, Instr, Z, kz)
1991
1992#define CASE_ARITH_RM(Instr) \
1993 INSTR_CASE(, Instr, , ) /* SSE */ \
1994 INSTR_CASE(V, Instr, , ) /* AVX-128 */ \
1995 INSTR_CASE(V, Instr, Y, ) /* AVX-256 */ \
1996 INSTR_CASE(V, Instr, Z128, ) \
1997 INSTR_CASE(V, Instr, Z128, k) \
1998 INSTR_CASE(V, Instr, Z128, kz) \
1999 INSTR_CASE(V, Instr, Z256, ) \
2000 INSTR_CASE(V, Instr, Z256, k) \
2001 INSTR_CASE(V, Instr, Z256, kz) \
2002 INSTR_CASE(V, Instr, Z, ) \
2003 INSTR_CASE(V, Instr, Z, k) \
2004 INSTR_CASE(V, Instr, Z, kz)
2005
2006 // TODO: Add additional instructions when useful.
2007 CASE_ARITH_RM(PADDB)
2008 CASE_ARITH_RM(PADDW)
2009 CASE_ARITH_RM(PADDD)
2010 CASE_ARITH_RM(PADDQ)
2011 CASE_ARITH_RM(PMADDUBSW)
2012 CASE_ARITH_RM(PMADDWD)
2013 CASE_ARITH_RM(PMULDQ)
2014 CASE_ARITH_RM(PMULUDQ)
2015 CASE_ARITH_RM(PMULLD)
2016 CASE_AVX512_ARITH_RM(PMULLQ)
2017 CASE_ARITH_RM(PMULLW)
2018 CASE_ARITH_RM(PMULHW)
2019 CASE_ARITH_RM(PMULHUW)
2020 CASE_ARITH_RM(PMULHRSW) {
2021 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
2022 unsigned VectorWidth =
2023 X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
2024 addConstantComment(MI, OutStreamer, OpNo: SrcIdx + 1, BitWidth: VectorWidth);
2025 break;
2026 }
2027
2028#define MASK_AVX512_CASE(Instr) \
2029 case Instr: \
2030 case Instr##k: \
2031 case Instr##kz:
2032
2033 case X86::MOVSDrm:
2034 case X86::VMOVSDrm:
2035 MASK_AVX512_CASE(X86::VMOVSDZrm)
2036 case X86::MOVSDrm_alt:
2037 case X86::VMOVSDrm_alt:
2038 case X86::VMOVSDZrm_alt:
2039 case X86::MOVQI2PQIrm:
2040 case X86::VMOVQI2PQIrm:
2041 case X86::VMOVQI2PQIZrm:
2042 printZeroUpperMove(MI, OutStreamer, SclWidth: 64, VecWidth: 128, ShuffleComment: "mem[0],zero");
2043 break;
2044
2045 MASK_AVX512_CASE(X86::VMOVSHZrm)
2046 case X86::VMOVSHZrm_alt:
2047 printZeroUpperMove(MI, OutStreamer, SclWidth: 16, VecWidth: 128,
2048 ShuffleComment: "mem[0],zero,zero,zero,zero,zero,zero,zero");
2049 break;
2050
2051 case X86::MOVSSrm:
2052 case X86::VMOVSSrm:
2053 MASK_AVX512_CASE(X86::VMOVSSZrm)
2054 case X86::MOVSSrm_alt:
2055 case X86::VMOVSSrm_alt:
2056 case X86::VMOVSSZrm_alt:
2057 case X86::MOVDI2PDIrm:
2058 case X86::VMOVDI2PDIrm:
2059 case X86::VMOVDI2PDIZrm:
2060 printZeroUpperMove(MI, OutStreamer, SclWidth: 32, VecWidth: 128, ShuffleComment: "mem[0],zero,zero,zero");
2061 break;
2062
2063#define MOV_CASE(Prefix, Suffix) \
2064 case X86::Prefix##MOVAPD##Suffix##rm: \
2065 case X86::Prefix##MOVAPS##Suffix##rm: \
2066 case X86::Prefix##MOVUPD##Suffix##rm: \
2067 case X86::Prefix##MOVUPS##Suffix##rm: \
2068 case X86::Prefix##MOVDQA##Suffix##rm: \
2069 case X86::Prefix##MOVDQU##Suffix##rm:
2070
2071#define MOV_AVX512_CASE(Suffix, Postfix) \
2072 case X86::VMOVDQA64##Suffix##rm##Postfix: \
2073 case X86::VMOVDQA32##Suffix##rm##Postfix: \
2074 case X86::VMOVDQU64##Suffix##rm##Postfix: \
2075 case X86::VMOVDQU32##Suffix##rm##Postfix: \
2076 case X86::VMOVDQU16##Suffix##rm##Postfix: \
2077 case X86::VMOVDQU8##Suffix##rm##Postfix: \
2078 case X86::VMOVAPS##Suffix##rm##Postfix: \
2079 case X86::VMOVAPD##Suffix##rm##Postfix: \
2080 case X86::VMOVUPS##Suffix##rm##Postfix: \
2081 case X86::VMOVUPD##Suffix##rm##Postfix:
2082
2083#define CASE_128_MOV_RM() \
2084 MOV_CASE(, ) /* SSE */ \
2085 MOV_CASE(V, ) /* AVX-128 */ \
2086 MOV_AVX512_CASE(Z128, ) \
2087 MOV_AVX512_CASE(Z128, k) \
2088 MOV_AVX512_CASE(Z128, kz)
2089
2090#define CASE_256_MOV_RM() \
2091 MOV_CASE(V, Y) /* AVX-256 */ \
2092 MOV_AVX512_CASE(Z256, ) \
2093 MOV_AVX512_CASE(Z256, k) \
2094 MOV_AVX512_CASE(Z256, kz) \
2095
2096#define CASE_512_MOV_RM() \
2097 MOV_AVX512_CASE(Z, ) \
2098 MOV_AVX512_CASE(Z, k) \
2099 MOV_AVX512_CASE(Z, kz) \
2100
2101 // For loads from a constant pool to a vector register, print the constant
2102 // loaded.
2103 CASE_128_MOV_RM()
2104 printBroadcast(MI, OutStreamer, Repeats: 1, BitWidth: 128);
2105 break;
2106 CASE_256_MOV_RM()
2107 printBroadcast(MI, OutStreamer, Repeats: 1, BitWidth: 256);
2108 break;
2109 CASE_512_MOV_RM()
2110 printBroadcast(MI, OutStreamer, Repeats: 1, BitWidth: 512);
2111 break;
2112 case X86::VBROADCASTF128rm:
2113 case X86::VBROADCASTI128rm:
2114 MASK_AVX512_CASE(X86::VBROADCASTF32X4Z256rm)
2115 MASK_AVX512_CASE(X86::VBROADCASTF64X2Z256rm)
2116 MASK_AVX512_CASE(X86::VBROADCASTI32X4Z256rm)
2117 MASK_AVX512_CASE(X86::VBROADCASTI64X2Z256rm)
2118 printBroadcast(MI, OutStreamer, Repeats: 2, BitWidth: 128);
2119 break;
2120 MASK_AVX512_CASE(X86::VBROADCASTF32X4Zrm)
2121 MASK_AVX512_CASE(X86::VBROADCASTF64X2Zrm)
2122 MASK_AVX512_CASE(X86::VBROADCASTI32X4Zrm)
2123 MASK_AVX512_CASE(X86::VBROADCASTI64X2Zrm)
2124 printBroadcast(MI, OutStreamer, Repeats: 4, BitWidth: 128);
2125 break;
2126 MASK_AVX512_CASE(X86::VBROADCASTF32X8Zrm)
2127 MASK_AVX512_CASE(X86::VBROADCASTF64X4Zrm)
2128 MASK_AVX512_CASE(X86::VBROADCASTI32X8Zrm)
2129 MASK_AVX512_CASE(X86::VBROADCASTI64X4Zrm)
2130 printBroadcast(MI, OutStreamer, Repeats: 2, BitWidth: 256);
2131 break;
2132
2133 // For broadcast loads from a constant pool to a vector register, repeatedly
2134 // print the constant loaded.
2135 case X86::MOVDDUPrm:
2136 case X86::VMOVDDUPrm:
2137 MASK_AVX512_CASE(X86::VMOVDDUPZ128rm)
2138 case X86::VPBROADCASTQrm:
2139 MASK_AVX512_CASE(X86::VPBROADCASTQZ128rm)
2140 printBroadcast(MI, OutStreamer, Repeats: 2, BitWidth: 64);
2141 break;
2142 case X86::VBROADCASTSDYrm:
2143 MASK_AVX512_CASE(X86::VBROADCASTSDZ256rm)
2144 case X86::VPBROADCASTQYrm:
2145 MASK_AVX512_CASE(X86::VPBROADCASTQZ256rm)
2146 printBroadcast(MI, OutStreamer, Repeats: 4, BitWidth: 64);
2147 break;
2148 MASK_AVX512_CASE(X86::VBROADCASTSDZrm)
2149 MASK_AVX512_CASE(X86::VPBROADCASTQZrm)
2150 printBroadcast(MI, OutStreamer, Repeats: 8, BitWidth: 64);
2151 break;
2152 case X86::VBROADCASTSSrm:
2153 MASK_AVX512_CASE(X86::VBROADCASTSSZ128rm)
2154 case X86::VPBROADCASTDrm:
2155 MASK_AVX512_CASE(X86::VPBROADCASTDZ128rm)
2156 printBroadcast(MI, OutStreamer, Repeats: 4, BitWidth: 32);
2157 break;
2158 case X86::VBROADCASTSSYrm:
2159 MASK_AVX512_CASE(X86::VBROADCASTSSZ256rm)
2160 case X86::VPBROADCASTDYrm:
2161 MASK_AVX512_CASE(X86::VPBROADCASTDZ256rm)
2162 printBroadcast(MI, OutStreamer, Repeats: 8, BitWidth: 32);
2163 break;
2164 MASK_AVX512_CASE(X86::VBROADCASTSSZrm)
2165 MASK_AVX512_CASE(X86::VPBROADCASTDZrm)
2166 printBroadcast(MI, OutStreamer, Repeats: 16, BitWidth: 32);
2167 break;
2168 case X86::VPBROADCASTWrm:
2169 MASK_AVX512_CASE(X86::VPBROADCASTWZ128rm)
2170 printBroadcast(MI, OutStreamer, Repeats: 8, BitWidth: 16);
2171 break;
2172 case X86::VPBROADCASTWYrm:
2173 MASK_AVX512_CASE(X86::VPBROADCASTWZ256rm)
2174 printBroadcast(MI, OutStreamer, Repeats: 16, BitWidth: 16);
2175 break;
2176 MASK_AVX512_CASE(X86::VPBROADCASTWZrm)
2177 printBroadcast(MI, OutStreamer, Repeats: 32, BitWidth: 16);
2178 break;
2179 case X86::VPBROADCASTBrm:
2180 MASK_AVX512_CASE(X86::VPBROADCASTBZ128rm)
2181 printBroadcast(MI, OutStreamer, Repeats: 16, BitWidth: 8);
2182 break;
2183 case X86::VPBROADCASTBYrm:
2184 MASK_AVX512_CASE(X86::VPBROADCASTBZ256rm)
2185 printBroadcast(MI, OutStreamer, Repeats: 32, BitWidth: 8);
2186 break;
2187 MASK_AVX512_CASE(X86::VPBROADCASTBZrm)
2188 printBroadcast(MI, OutStreamer, Repeats: 64, BitWidth: 8);
2189 break;
2190
2191#define MOVX_CASE(Prefix, Ext, Type, Suffix, Postfix) \
2192 case X86::Prefix##PMOV##Ext##Type##Suffix##rm##Postfix:
2193
2194#define CASE_MOVX_RM(Ext, Type) \
2195 MOVX_CASE(, Ext, Type, , ) \
2196 MOVX_CASE(V, Ext, Type, , ) \
2197 MOVX_CASE(V, Ext, Type, Y, ) \
2198 MOVX_CASE(V, Ext, Type, Z128, ) \
2199 MOVX_CASE(V, Ext, Type, Z128, k ) \
2200 MOVX_CASE(V, Ext, Type, Z128, kz ) \
2201 MOVX_CASE(V, Ext, Type, Z256, ) \
2202 MOVX_CASE(V, Ext, Type, Z256, k ) \
2203 MOVX_CASE(V, Ext, Type, Z256, kz ) \
2204 MOVX_CASE(V, Ext, Type, Z, ) \
2205 MOVX_CASE(V, Ext, Type, Z, k ) \
2206 MOVX_CASE(V, Ext, Type, Z, kz )
2207
2208 CASE_MOVX_RM(SX, BD)
2209 printSignExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 32);
2210 break;
2211 CASE_MOVX_RM(SX, BQ)
2212 printSignExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 64);
2213 break;
2214 CASE_MOVX_RM(SX, BW)
2215 printSignExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 16);
2216 break;
2217 CASE_MOVX_RM(SX, DQ)
2218 printSignExtend(MI, OutStreamer, SrcEltBits: 32, DstEltBits: 64);
2219 break;
2220 CASE_MOVX_RM(SX, WD)
2221 printSignExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 32);
2222 break;
2223 CASE_MOVX_RM(SX, WQ)
2224 printSignExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 64);
2225 break;
2226
2227 CASE_MOVX_RM(ZX, BD)
2228 printZeroExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 32);
2229 break;
2230 CASE_MOVX_RM(ZX, BQ)
2231 printZeroExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 64);
2232 break;
2233 CASE_MOVX_RM(ZX, BW)
2234 printZeroExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 16);
2235 break;
2236 CASE_MOVX_RM(ZX, DQ)
2237 printZeroExtend(MI, OutStreamer, SrcEltBits: 32, DstEltBits: 64);
2238 break;
2239 CASE_MOVX_RM(ZX, WD)
2240 printZeroExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 32);
2241 break;
2242 CASE_MOVX_RM(ZX, WQ)
2243 printZeroExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 64);
2244 break;
2245 }
2246}
2247
2248// Does the given operand refer to a DLLIMPORT function?
2249bool isImportedFunction(const MachineOperand &MO) {
2250 return MO.isGlobal() && (MO.getTargetFlags() == X86II::MO_DLLIMPORT);
2251}
2252
2253// Is the given instruction a call to a CFGuard function?
2254bool isCallToCFGuardFunction(const MachineInstr *MI) {
2255 assert(MI->getOpcode() == X86::TAILJMPm64_REX ||
2256 MI->getOpcode() == X86::CALL64m);
2257 const MachineOperand &MO = MI->getOperand(i: 3);
2258 return MO.isGlobal() && (MO.getTargetFlags() == X86II::MO_NO_FLAG) &&
2259 isCFGuardFunction(GV: MO.getGlobal());
2260}
2261
2262// Does the containing block for the given instruction contain any jump table
2263// info (indicating that the block is a dispatch for a jump table)?
2264bool hasJumpTableInfoInBlock(const llvm::MachineInstr *MI) {
2265 const MachineBasicBlock &MBB = *MI->getParent();
2266 for (auto I = MBB.instr_rbegin(), E = MBB.instr_rend(); I != E; ++I)
2267 if (I->isJumpTableDebugInfo())
2268 return true;
2269
2270 return false;
2271}
2272
2273void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
2274 // FIXME: Enable feature predicate checks once all the test pass.
2275 // X86_MC::verifyInstructionPredicates(MI->getOpcode(),
2276 // Subtarget->getFeatureBits());
2277
2278 X86MCInstLower MCInstLowering(*MF, *this);
2279 const X86RegisterInfo *RI =
2280 MF->getSubtarget<X86Subtarget>().getRegisterInfo();
2281
2282 if (MI->getOpcode() == X86::OR64rm) {
2283 for (auto &Opd : MI->operands()) {
2284 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
2285 "swift_async_extendedFramePointerFlags") {
2286 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
2287 }
2288 }
2289 }
2290
2291 // Add comments for values loaded from constant pool.
2292 if (OutStreamer->isVerboseAsm())
2293 addConstantComments(MI, OutStreamer&: *OutStreamer);
2294
2295 // Add a comment about EVEX compression
2296 if (TM.Options.MCOptions.ShowMCEncoding) {
2297 if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_LEGACY)
2298 OutStreamer->AddComment(T: "EVEX TO LEGACY Compression ", EOL: false);
2299 else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX)
2300 OutStreamer->AddComment(T: "EVEX TO VEX Compression ", EOL: false);
2301 else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_EVEX)
2302 OutStreamer->AddComment(T: "EVEX TO EVEX Compression ", EOL: false);
2303 }
2304
2305 // We use this to suppress NOP padding for Windows EH.
2306 bool IsTailJump = false;
2307
2308 switch (MI->getOpcode()) {
2309 case TargetOpcode::DBG_VALUE:
2310 llvm_unreachable("Should be handled target independently");
2311
2312 case X86::EH_RETURN:
2313 case X86::EH_RETURN64: {
2314 // Lower these as normal, but add some comments.
2315 Register Reg = MI->getOperand(i: 0).getReg();
2316 OutStreamer->AddComment(T: StringRef("eh_return, addr: %") +
2317 X86ATTInstPrinter::getRegisterName(Reg));
2318 break;
2319 }
2320 case X86::CLEANUPRET: {
2321 // Lower these as normal, but add some comments.
2322 OutStreamer->AddComment(T: "CLEANUPRET");
2323 break;
2324 }
2325
2326 case X86::CATCHRET: {
2327 // Lower these as normal, but add some comments.
2328 OutStreamer->AddComment(T: "CATCHRET");
2329 break;
2330 }
2331
2332 case X86::ENDBR32:
2333 case X86::ENDBR64: {
2334 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
2335 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2336 // non-empty. If MI is the initial ENDBR, place the
2337 // __patchable_function_entries label after ENDBR.
2338 if (CurrentPatchableFunctionEntrySym &&
2339 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
2340 MI == &MF->front().front()) {
2341 MCInst Inst;
2342 MCInstLowering.Lower(MI, OutMI&: Inst);
2343 EmitAndCountInstruction(Inst);
2344 CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch");
2345 OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym);
2346 return;
2347 }
2348 break;
2349 }
2350
2351 case X86::TAILJMPd64:
2352 if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(Reg: X86::R11))
2353 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CS_PREFIX));
2354
2355 if (EnableImportCallOptimization && isImportedFunction(MO: MI->getOperand(i: 0))) {
2356 emitLabelAndRecordForImportCallOptimization(
2357 Kind: IMAGE_RETPOLINE_AMD64_IMPORT_BR);
2358 }
2359
2360 // Lower this as normal, but add a comment.
2361 OutStreamer->AddComment(T: "TAILCALL");
2362 IsTailJump = true;
2363 break;
2364
2365 case X86::TAILJMPr:
2366 case X86::TAILJMPm:
2367 case X86::TAILJMPd:
2368 case X86::TAILJMPd_CC:
2369 case X86::TAILJMPr64:
2370 case X86::TAILJMPm64:
2371 case X86::TAILJMPd64_CC:
2372 if (EnableImportCallOptimization)
2373 report_fatal_error(reason: "Unexpected TAILJMP instruction was emitted when "
2374 "import call optimization was enabled");
2375
2376 // Lower these as normal, but add some comments.
2377 OutStreamer->AddComment(T: "TAILCALL");
2378 IsTailJump = true;
2379 break;
2380
2381 case X86::TAILJMPm64_REX:
2382 if (EnableImportCallOptimization && isCallToCFGuardFunction(MI)) {
2383 emitLabelAndRecordForImportCallOptimization(
2384 Kind: IMAGE_RETPOLINE_AMD64_CFG_BR_REX);
2385 }
2386
2387 OutStreamer->AddComment(T: "TAILCALL");
2388 IsTailJump = true;
2389 break;
2390
2391 case X86::TAILJMPr64_REX: {
2392 if (EnableImportCallOptimization) {
2393 assert(MI->getOperand(0).getReg() == X86::RAX &&
2394 "Indirect tail calls with impcall enabled must go through RAX (as "
2395 "enforced by TCRETURNImpCallri64)");
2396 emitLabelAndRecordForImportCallOptimization(
2397 Kind: IMAGE_RETPOLINE_AMD64_INDIR_BR);
2398 }
2399
2400 OutStreamer->AddComment(T: "TAILCALL");
2401 IsTailJump = true;
2402 break;
2403 }
2404
2405 case X86::JMP64r:
2406 if (EnableImportCallOptimization && hasJumpTableInfoInBlock(MI)) {
2407 uint16_t EncodedReg =
2408 this->getSubtarget().getRegisterInfo()->getEncodingValue(
2409 Reg: MI->getOperand(i: 0).getReg().asMCReg());
2410 emitLabelAndRecordForImportCallOptimization(
2411 Kind: (ImportCallKind)(IMAGE_RETPOLINE_AMD64_SWITCHTABLE_FIRST +
2412 EncodedReg));
2413 }
2414 break;
2415
2416 case X86::JMP16r:
2417 case X86::JMP16m:
2418 case X86::JMP32r:
2419 case X86::JMP32m:
2420 case X86::JMP64m:
2421 if (EnableImportCallOptimization && hasJumpTableInfoInBlock(MI))
2422 report_fatal_error(
2423 reason: "Unexpected JMP instruction was emitted for a jump-table when import "
2424 "call optimization was enabled");
2425 break;
2426
2427 case X86::TLS_addr32:
2428 case X86::TLS_addr64:
2429 case X86::TLS_addrX32:
2430 case X86::TLS_base_addr32:
2431 case X86::TLS_base_addr64:
2432 case X86::TLS_base_addrX32:
2433 case X86::TLS_desc32:
2434 case X86::TLS_desc64:
2435 return LowerTlsAddr(MCInstLowering, MI: *MI);
2436
2437 case X86::MOVPC32r: {
2438 // This is a pseudo op for a two instruction sequence with a label, which
2439 // looks like:
2440 // call "L1$pb"
2441 // "L1$pb":
2442 // popl %esi
2443
2444 // Emit the call.
2445 MCSymbol *PICBase = MF->getPICBaseSymbol();
2446 // FIXME: We would like an efficient form for this, so we don't have to do a
2447 // lot of extra uniquing.
2448 EmitAndCountInstruction(
2449 Inst&: MCInstBuilder(X86::CALLpcrel32)
2450 .addExpr(Val: MCSymbolRefExpr::create(Symbol: PICBase, Ctx&: OutContext)));
2451
2452 const X86FrameLowering *FrameLowering =
2453 MF->getSubtarget<X86Subtarget>().getFrameLowering();
2454 bool hasFP = FrameLowering->hasFP(MF: *MF);
2455
2456 // TODO: This is needed only if we require precise CFA.
2457 bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() &&
2458 !OutStreamer->getDwarfFrameInfos().back().End;
2459
2460 int stackGrowth = -RI->getSlotSize();
2461
2462 if (HasActiveDwarfFrame && !hasFP) {
2463 OutStreamer->emitCFIAdjustCfaOffset(Adjustment: -stackGrowth);
2464 MF->getInfo<X86MachineFunctionInfo>()->setHasCFIAdjustCfa(true);
2465 }
2466
2467 // Emit the label.
2468 OutStreamer->emitLabel(Symbol: PICBase);
2469
2470 // popl $reg
2471 EmitAndCountInstruction(
2472 Inst&: MCInstBuilder(X86::POP32r).addReg(Reg: MI->getOperand(i: 0).getReg()));
2473
2474 if (HasActiveDwarfFrame && !hasFP) {
2475 OutStreamer->emitCFIAdjustCfaOffset(Adjustment: stackGrowth);
2476 }
2477 return;
2478 }
2479
2480 case X86::ADD32ri: {
2481 // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri.
2482 if (MI->getOperand(i: 2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS)
2483 break;
2484
2485 // Okay, we have something like:
2486 // EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL)
2487
2488 // For this, we want to print something like:
2489 // MYGLOBAL + (. - PICBASE)
2490 // However, we can't generate a ".", so just emit a new label here and refer
2491 // to it.
2492 MCSymbol *DotSym = OutContext.createTempSymbol();
2493 OutStreamer->emitLabel(Symbol: DotSym);
2494
2495 // Now that we have emitted the label, lower the complex operand expression.
2496 MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MO: MI->getOperand(i: 2));
2497
2498 const MCExpr *DotExpr = MCSymbolRefExpr::create(Symbol: DotSym, Ctx&: OutContext);
2499 const MCExpr *PICBase =
2500 MCSymbolRefExpr::create(Symbol: MF->getPICBaseSymbol(), Ctx&: OutContext);
2501 DotExpr = MCBinaryExpr::createSub(LHS: DotExpr, RHS: PICBase, Ctx&: OutContext);
2502
2503 DotExpr = MCBinaryExpr::createAdd(
2504 LHS: MCSymbolRefExpr::create(Symbol: OpSym, Ctx&: OutContext), RHS: DotExpr, Ctx&: OutContext);
2505
2506 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::ADD32ri)
2507 .addReg(Reg: MI->getOperand(i: 0).getReg())
2508 .addReg(Reg: MI->getOperand(i: 1).getReg())
2509 .addExpr(Val: DotExpr));
2510 return;
2511 }
2512 case TargetOpcode::STATEPOINT:
2513 return LowerSTATEPOINT(MI: *MI, MCIL&: MCInstLowering);
2514
2515 case TargetOpcode::FAULTING_OP:
2516 return LowerFAULTING_OP(FaultingMI: *MI, MCIL&: MCInstLowering);
2517
2518 case TargetOpcode::FENTRY_CALL:
2519 return LowerFENTRY_CALL(MI: *MI, MCIL&: MCInstLowering);
2520
2521 case TargetOpcode::PATCHABLE_OP:
2522 return LowerPATCHABLE_OP(MI: *MI, MCIL&: MCInstLowering);
2523
2524 case TargetOpcode::STACKMAP:
2525 return LowerSTACKMAP(MI: *MI);
2526
2527 case TargetOpcode::PATCHPOINT:
2528 return LowerPATCHPOINT(MI: *MI, MCIL&: MCInstLowering);
2529
2530 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2531 return LowerPATCHABLE_FUNCTION_ENTER(MI: *MI, MCIL&: MCInstLowering);
2532
2533 case TargetOpcode::PATCHABLE_RET:
2534 return LowerPATCHABLE_RET(MI: *MI, MCIL&: MCInstLowering);
2535
2536 case TargetOpcode::PATCHABLE_TAIL_CALL:
2537 return LowerPATCHABLE_TAIL_CALL(MI: *MI, MCIL&: MCInstLowering);
2538
2539 case TargetOpcode::PATCHABLE_EVENT_CALL:
2540 return LowerPATCHABLE_EVENT_CALL(MI: *MI, MCIL&: MCInstLowering);
2541
2542 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
2543 return LowerPATCHABLE_TYPED_EVENT_CALL(MI: *MI, MCIL&: MCInstLowering);
2544
2545 case X86::MORESTACK_RET:
2546 EmitAndCountInstruction(Inst&: MCInstBuilder(getRetOpcode(Subtarget: *Subtarget)));
2547 return;
2548
2549 case X86::KCFI_CHECK:
2550 return LowerKCFI_CHECK(MI: *MI);
2551
2552 case X86::ASAN_CHECK_MEMACCESS:
2553 return LowerASAN_CHECK_MEMACCESS(MI: *MI);
2554
2555 case X86::MORESTACK_RET_RESTORE_R10:
2556 // Return, then restore R10.
2557 EmitAndCountInstruction(Inst&: MCInstBuilder(getRetOpcode(Subtarget: *Subtarget)));
2558 EmitAndCountInstruction(
2559 Inst&: MCInstBuilder(X86::MOV64rr).addReg(Reg: X86::R10).addReg(Reg: X86::RAX));
2560 return;
2561
2562 case X86::SEH_PushReg:
2563 case X86::SEH_Push2Regs:
2564 case X86::SEH_SaveReg:
2565 case X86::SEH_SaveXMM:
2566 case X86::SEH_StackAlloc:
2567 case X86::SEH_StackAlign:
2568 case X86::SEH_SetFrame:
2569 case X86::SEH_PushFrame:
2570 case X86::SEH_EndPrologue:
2571 case X86::SEH_EndEpilogue:
2572 case X86::SEH_UnwindV2Start:
2573 case X86::SEH_UnwindVersion:
2574 EmitSEHInstruction(MI);
2575 return;
2576
2577 case X86::SEH_SplitChainedAtEndOfBlock:
2578 assert(!SplitChainedAtEndOfBlock &&
2579 "Duplicate SEH_SplitChainedAtEndOfBlock in a current block");
2580 SplitChainedAtEndOfBlock = true;
2581 return;
2582
2583 case X86::SEH_SplitChained:
2584 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2585 OutStreamer->emitWinCFISplitChained();
2586 return;
2587
2588 case X86::SEH_BeginEpilogue: {
2589 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2590 EmitSEHInstruction(MI);
2591 return;
2592 }
2593 case X86::UBSAN_UD1:
2594 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::UD1Lm)
2595 .addReg(Reg: X86::EAX)
2596 .addReg(Reg: X86::EAX)
2597 .addImm(Val: 1)
2598 .addReg(Reg: X86::NoRegister)
2599 .addImm(Val: MI->getOperand(i: 0).getImm())
2600 .addReg(Reg: X86::NoRegister));
2601 return;
2602 case X86::CALL64pcrel32:
2603 if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(Reg: X86::R11))
2604 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CS_PREFIX));
2605
2606 if (EnableImportCallOptimization && isImportedFunction(MO: MI->getOperand(i: 0))) {
2607 emitLabelAndRecordForImportCallOptimization(
2608 Kind: IMAGE_RETPOLINE_AMD64_IMPORT_CALL);
2609
2610 MCInst TmpInst;
2611 MCInstLowering.Lower(MI, OutMI&: TmpInst);
2612
2613 // For Import Call Optimization to work, we need a the call instruction
2614 // with a rex prefix, and a 5-byte nop after the call instruction.
2615 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::REX64_PREFIX));
2616 emitCallInstruction(MCI: TmpInst);
2617 emitNop(OS&: *OutStreamer, NumBytes: 5, Subtarget);
2618 maybeEmitNopAfterCallForWindowsEH(MI);
2619 return;
2620 }
2621
2622 break;
2623
2624 case X86::CALL64r:
2625 if (EnableImportCallOptimization) {
2626 assert(MI->getOperand(0).getReg() == X86::RAX &&
2627 "Indirect calls with impcall enabled must go through RAX (as "
2628 "enforced by CALL64r_ImpCall)");
2629
2630 emitLabelAndRecordForImportCallOptimization(
2631 Kind: IMAGE_RETPOLINE_AMD64_INDIR_CALL);
2632 MCInst TmpInst;
2633 MCInstLowering.Lower(MI, OutMI&: TmpInst);
2634 emitCallInstruction(MCI: TmpInst);
2635
2636 // For Import Call Optimization to work, we need a 3-byte nop after the
2637 // call instruction.
2638 emitNop(OS&: *OutStreamer, NumBytes: 3, Subtarget);
2639 maybeEmitNopAfterCallForWindowsEH(MI);
2640 return;
2641 }
2642 break;
2643
2644 case X86::CALL64m:
2645 if (EnableImportCallOptimization && isCallToCFGuardFunction(MI)) {
2646 emitLabelAndRecordForImportCallOptimization(
2647 Kind: IMAGE_RETPOLINE_AMD64_CFG_CALL);
2648 }
2649 break;
2650
2651 case X86::JCC_1:
2652 // Two instruction prefixes (2EH for branch not-taken and 3EH for branch
2653 // taken) are used as branch hints. Here we add branch taken prefix for
2654 // jump instruction with higher probability than threshold.
2655 if (getSubtarget().hasBranchHint() && EnableBranchHint) {
2656 const MachineBranchProbabilityInfo *MBPI =
2657 &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
2658 MachineBasicBlock *DestBB = MI->getOperand(i: 0).getMBB();
2659 BranchProbability EdgeProb =
2660 MBPI->getEdgeProbability(Src: MI->getParent(), Dst: DestBB);
2661 BranchProbability Threshold(BranchHintProbabilityThreshold, 100);
2662 if (EdgeProb > Threshold)
2663 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DS_PREFIX));
2664 }
2665 break;
2666
2667 case X86::JCC_SELF:
2668 MCSymbol *Sym = OutContext.createTempSymbol();
2669 OutStreamer->emitLabel(Symbol: Sym);
2670 EmitAndCountInstruction(
2671 Inst&: MCInstBuilder(X86::JCC_1)
2672 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Sym, Ctx&: OutContext))
2673 .addImm(Val: MI->getOperand(i: 0).getImm()));
2674 return;
2675 }
2676
2677 MCInst TmpInst;
2678 MCInstLowering.Lower(MI, OutMI&: TmpInst);
2679
2680 if (MI->isCall()) {
2681 emitCallInstruction(MCI: TmpInst);
2682 // Since tail calls transfer control without leaving a stack frame, there is
2683 // never a need for NOP padding tail calls.
2684 if (!IsTailJump)
2685 maybeEmitNopAfterCallForWindowsEH(MI);
2686 return;
2687 }
2688
2689 EmitAndCountInstruction(Inst&: TmpInst);
2690}
2691
2692void X86AsmPrinter::emitInlineAsmEnd(const MCSubtargetInfo &StartInfo,
2693 const MCSubtargetInfo *EndInfo,
2694 const MachineInstr *MI) {
2695 if (MI) {
2696 // If unwinding inline asm ends on a call, wineh may require insertion of
2697 // a nop.
2698 unsigned ExtraInfo = MI->getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
2699 if (ExtraInfo & InlineAsm::Extra_MayUnwind)
2700 maybeEmitNopAfterCallForWindowsEH(MI);
2701 }
2702}
2703
2704void X86AsmPrinter::emitCallInstruction(const llvm::MCInst &MCI) {
2705 // Stackmap shadows cannot include branch targets, so we can count the bytes
2706 // in a call towards the shadow, but must ensure that the no thread returns
2707 // in to the stackmap shadow. The only way to achieve this is if the call
2708 // is at the end of the shadow.
2709
2710 // Count then size of the call towards the shadow
2711 SMShadowTracker.count(Inst: MCI, STI: getSubtargetInfo(), CodeEmitter: CodeEmitter.get());
2712 // Then flush the shadow so that we fill with nops before the call, not
2713 // after it.
2714 SMShadowTracker.emitShadowPadding(OutStreamer&: *OutStreamer, STI: getSubtargetInfo());
2715 // Then emit the call
2716 OutStreamer->emitInstruction(Inst: MCI, STI: getSubtargetInfo());
2717}
2718
2719// Determines whether a NOP is required after a CALL, so that Windows EH
2720// IP2State tables have the correct information.
2721//
2722// On most Windows platforms (AMD64, ARM64, ARM32, IA64, but *not* x86-32),
2723// exception handling works by looking up instruction pointers in lookup
2724// tables. These lookup tables are stored in .xdata sections in executables.
2725// One element of the lookup tables are the "IP2State" tables (Instruction
2726// Pointer to State).
2727//
2728// If a function has any instructions that require cleanup during exception
2729// unwinding, then it will have an IP2State table. Each entry in the IP2State
2730// table describes a range of bytes in the function's instruction stream, and
2731// associates an "EH state number" with that range of instructions. A value of
2732// -1 means "the null state", which does not require any code to execute.
2733// A value other than -1 is an index into the State table.
2734//
2735// The entries in the IP2State table contain byte offsets within the instruction
2736// stream of the function. The Windows ABI requires that these offsets are
2737// aligned to instruction boundaries; they are not permitted to point to a byte
2738// that is not the first byte of an instruction.
2739//
2740// Unfortunately, CALL instructions present a problem during unwinding. CALL
2741// instructions push the address of the instruction after the CALL instruction,
2742// so that execution can resume after the CALL. If the CALL is the last
2743// instruction within an IP2State region, then the return address (on the stack)
2744// points to the *next* IP2State region. This means that the unwinder will
2745// use the wrong cleanup funclet during unwinding.
2746//
2747// To fix this problem, the Windows AMD64 ABI requires that CALL instructions
2748// are never placed at the end of an IP2State region. Stated equivalently, the
2749// end of a CALL instruction cannot be aligned to an IP2State boundary. If a
2750// CALL instruction would occur at the end of an IP2State region, then the
2751// compiler must insert a NOP instruction after the CALL. The NOP instruction
2752// is placed in the same EH region as the CALL instruction, so that the return
2753// address points to the NOP and the unwinder will locate the correct region.
2754//
2755// NOP padding is only necessary on Windows AMD64 targets. On ARM64 and ARM32,
2756// instructions have a fixed size so the unwinder knows how to "back up" by
2757// one instruction.
2758//
2759// Interaction with Import Call Optimization (ICO):
2760//
2761// Import Call Optimization (ICO) is a compiler + OS feature on Windows which
2762// improves the performance and security of DLL imports. ICO relies on using a
2763// specific CALL idiom that can be replaced by the OS DLL loader. This removes
2764// a load and indirect CALL and replaces it with a single direct CALL.
2765//
2766// To achieve this, ICO also inserts NOPs after the CALL instruction. If the
2767// end of the CALL is aligned with an EH state transition, we *also* insert
2768// a single-byte NOP. **Both forms of NOPs must be preserved.** They cannot
2769// be combined into a single larger NOP; nor can the second NOP be removed.
2770//
2771// This is necessary because, if ICO is active and the call site is modified
2772// by the loader, the loader will end up overwriting the NOPs that were inserted
2773// for ICO. That means that those NOPs cannot be used for the correct
2774// termination of the exception handling region (the IP2State transition),
2775// so we still need an additional NOP instruction. The NOPs cannot be combined
2776// into a longer NOP (which is ordinarily desirable) because then ICO would
2777// split one instruction, producing a malformed instruction after the ICO call.
2778void X86AsmPrinter::maybeEmitNopAfterCallForWindowsEH(const MachineInstr *MI) {
2779 // We only need to insert NOPs after CALLs when targeting Windows on AMD64.
2780 // (Don't let the name fool you: Itanium refers to table-based exception
2781 // handling, not the Itanium architecture.)
2782 if (MAI.getExceptionHandlingType() != ExceptionHandling::WinEH ||
2783 MAI.getWinEHEncodingType() != WinEH::EncodingType::Itanium) {
2784 return;
2785 }
2786
2787 bool HasEHPersonality = MF->getWinEHFuncInfo() != nullptr;
2788
2789 // Set up MBB iterator, initially positioned on the same MBB as MI.
2790 MachineFunction::const_iterator MFI(MI->getParent());
2791 MachineFunction::const_iterator MFE(MF->end());
2792
2793 // Set up instruction iterator, positioned immediately *after* MI.
2794 MachineBasicBlock::const_iterator MBBI(MI);
2795 MachineBasicBlock::const_iterator MBBE = MI->getParent()->end();
2796 ++MBBI; // Step over MI
2797
2798 // This loop iterates MBBs
2799 for (;;) {
2800 // This loop iterates instructions
2801 for (; MBBI != MBBE; ++MBBI) {
2802 // Check the instruction that follows this CALL.
2803 const MachineInstr &NextMI = *MBBI;
2804
2805 // If there is an EH_LABEL after this CALL, then there is an EH state
2806 // transition after this CALL. This is exactly the situation which
2807 // requires NOP padding.
2808 if (NextMI.isEHLabel()) {
2809 if (HasEHPersonality) {
2810 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::NOOP));
2811 return;
2812 }
2813 // We actually want to continue, in case there is an SEH_BeginEpilogue
2814 // instruction after the EH_LABEL. In some situations, IR is produced
2815 // that contains EH_LABEL pseudo-instructions, even when we are not
2816 // generating IP2State tables. We still need to insert a NOP before
2817 // SEH_BeginEpilogue in that case.
2818 continue;
2819 }
2820
2821 // Somewhat similarly, if the CALL is the last instruction before the
2822 // SEH prologue, then we also need a NOP. This is necessary because the
2823 // Windows stack unwinder will not invoke a function's exception handler
2824 // if the instruction pointer is in the function prologue or epilogue.
2825 //
2826 // We always emit a NOP before SEH_BeginEpilogue, even if there is no
2827 // personality function (unwind info) for this frame. This is the same
2828 // behavior as MSVC.
2829 if (NextMI.getOpcode() == X86::SEH_BeginEpilogue) {
2830 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::NOOP));
2831 return;
2832 }
2833
2834 if (!NextMI.isPseudo() && !NextMI.isMetaInstruction()) {
2835 // We found a real instruction. During the CALL, the return IP will
2836 // point to this instruction. Since this instruction has the same EH
2837 // state as the call itself (because there is no intervening EH_LABEL),
2838 // the IP2State table will be accurate; there is no need to insert a
2839 // NOP.
2840 return;
2841 }
2842
2843 // The next instruction is a pseudo-op. Ignore it and keep searching.
2844 // Because these instructions do not generate any machine code, they
2845 // cannot prevent the IP2State table from pointing at the wrong
2846 // instruction during a CALL.
2847 }
2848
2849 // We've reached the end of this MBB. Find the next MBB in program order.
2850 // MBB order should be finalized by this point, so falling across MBBs is
2851 // expected.
2852 ++MFI;
2853 if (MFI == MFE) {
2854 // No more blocks; we've reached the end of the function. This should
2855 // only happen with no-return functions, but double-check to be sure.
2856 if (HasEHPersonality) {
2857 // If the CALL has no successors, then it is a noreturn function.
2858 // Insert an INT3 instead of a NOP. This accomplishes the same purpose,
2859 // but is more clear to read. Also, analysis tools will understand
2860 // that they should not continue disassembling after the CALL (unless
2861 // there are other branches to that label).
2862 if (MI->getParent()->succ_empty())
2863 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::INT3));
2864 else
2865 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::NOOP));
2866 }
2867 return;
2868 }
2869
2870 // Set up iterator to scan the next basic block.
2871 const MachineBasicBlock *NextMBB = &*MFI;
2872 MBBI = NextMBB->instr_begin();
2873 MBBE = NextMBB->instr_end();
2874 }
2875}
2876
2877void X86AsmPrinter::emitLabelAndRecordForImportCallOptimization(
2878 ImportCallKind Kind) {
2879 assert(EnableImportCallOptimization);
2880
2881 MCSymbol *CallSiteSymbol = MMI->getContext().createNamedTempSymbol(Name: "impcall");
2882 OutStreamer->emitLabel(Symbol: CallSiteSymbol);
2883
2884 SectionToImportedFunctionCalls[OutStreamer->getCurrentSectionOnly()]
2885 .push_back(x: {.CalleeSymbol: CallSiteSymbol, .Kind: Kind});
2886}
2887