1//===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains code to lower X86 MachineInstrs to their corresponding
10// MCInst records.
11//
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/X86ATTInstPrinter.h"
15#include "MCTargetDesc/X86BaseInfo.h"
16#include "MCTargetDesc/X86EncodingOptimization.h"
17#include "MCTargetDesc/X86InstComments.h"
18#include "MCTargetDesc/X86MCAsmInfo.h"
19#include "MCTargetDesc/X86ShuffleDecode.h"
20#include "MCTargetDesc/X86TargetStreamer.h"
21#include "X86AsmPrinter.h"
22#include "X86MachineFunctionInfo.h"
23#include "X86RegisterInfo.h"
24#include "X86ShuffleDecodeConstantPool.h"
25#include "X86Subtarget.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SmallString.h"
28#include "llvm/ADT/StringExtras.h"
29#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
30#include "llvm/CodeGen/MachineConstantPool.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineModuleInfoImpls.h"
33#include "llvm/CodeGen/MachineOperand.h"
34#include "llvm/CodeGen/StackMaps.h"
35#include "llvm/CodeGen/WinEHFuncInfo.h"
36#include "llvm/IR/DataLayout.h"
37#include "llvm/IR/GlobalValue.h"
38#include "llvm/IR/Mangler.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCCodeEmitter.h"
41#include "llvm/MC/MCContext.h"
42#include "llvm/MC/MCExpr.h"
43#include "llvm/MC/MCFixup.h"
44#include "llvm/MC/MCInst.h"
45#include "llvm/MC/MCInstBuilder.h"
46#include "llvm/MC/MCSection.h"
47#include "llvm/MC/MCStreamer.h"
48#include "llvm/MC/MCSymbol.h"
49#include "llvm/MC/TargetRegistry.h"
50#include "llvm/Target/TargetLoweringObjectFile.h"
51#include "llvm/Target/TargetMachine.h"
52#include "llvm/Transforms/CFGuard.h"
53#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
54#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
55#include <string>
56
57using namespace llvm;
58
59namespace {
60
61/// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.
62class X86MCInstLower {
63 MCContext &Ctx;
64 const MachineFunction &MF;
65 const TargetMachine &TM;
66 const MCAsmInfo &MAI;
67 X86AsmPrinter &AsmPrinter;
68
69public:
70 X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter);
71
72 MCOperand LowerMachineOperand(const MachineInstr *MI,
73 const MachineOperand &MO) const;
74 void Lower(const MachineInstr *MI, MCInst &OutMI) const;
75
76 MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const;
77 MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const;
78
79private:
80 MachineModuleInfoMachO &getMachOMMI() const;
81};
82
83} // end anonymous namespace
84
85/// A RAII helper which defines a region of instructions which can't have
86/// padding added between them for correctness.
87struct NoAutoPaddingScope {
88 MCStreamer &OS;
89 const bool OldAllowAutoPadding;
90 NoAutoPaddingScope(MCStreamer &OS)
91 : OS(OS), OldAllowAutoPadding(OS.getAllowAutoPadding()) {
92 changeAndComment(b: false);
93 }
94 ~NoAutoPaddingScope() { changeAndComment(b: OldAllowAutoPadding); }
95 void changeAndComment(bool b) {
96 if (b == OS.getAllowAutoPadding())
97 return;
98 OS.setAllowAutoPadding(b);
99 if (b)
100 OS.emitRawComment(T: "autopadding");
101 else
102 OS.emitRawComment(T: "noautopadding");
103 }
104};
105
106// Emit a minimal sequence of nops spanning NumBytes bytes.
107static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
108 const X86Subtarget *Subtarget);
109
110void X86AsmPrinter::StackMapShadowTracker::count(const MCInst &Inst,
111 const MCSubtargetInfo &STI,
112 MCCodeEmitter *CodeEmitter) {
113 if (InShadow) {
114 SmallString<256> Code;
115 SmallVector<MCFixup, 4> Fixups;
116 CodeEmitter->encodeInstruction(Inst, CB&: Code, Fixups, STI);
117 CurrentShadowSize += Code.size();
118 if (CurrentShadowSize >= RequiredShadowSize)
119 InShadow = false; // The shadow is big enough. Stop counting.
120 }
121}
122
123void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding(
124 MCStreamer &OutStreamer, const MCSubtargetInfo &STI) {
125 if (InShadow && CurrentShadowSize < RequiredShadowSize) {
126 InShadow = false;
127 emitX86Nops(OS&: OutStreamer, NumBytes: RequiredShadowSize - CurrentShadowSize,
128 Subtarget: &MF->getSubtarget<X86Subtarget>());
129 }
130}
131
132void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {
133 OutStreamer->emitInstruction(Inst, STI: getSubtargetInfo());
134 SMShadowTracker.count(Inst, STI: getSubtargetInfo(), CodeEmitter: CodeEmitter.get());
135}
136
137X86MCInstLower::X86MCInstLower(const MachineFunction &mf,
138 X86AsmPrinter &asmprinter)
139 : Ctx(asmprinter.OutContext), MF(mf), TM(mf.getTarget()),
140 MAI(TM.getMCAsmInfo()), AsmPrinter(asmprinter) {}
141
142MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {
143 return AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoMachO>();
144}
145
146/// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol
147/// operand to an MCSymbol.
148MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const {
149 const Triple &TT = TM.getTargetTriple();
150 if (MO.isGlobal() && TT.isOSBinFormatELF())
151 return AsmPrinter.getSymbolPreferLocal(GV: *MO.getGlobal());
152
153 const DataLayout &DL = MF.getDataLayout();
154 assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) &&
155 "Isn't a symbol reference");
156
157 MCSymbol *Sym = nullptr;
158 SmallString<128> Name;
159 StringRef Suffix;
160
161 switch (MO.getTargetFlags()) {
162 case X86II::MO_DLLIMPORT:
163 // Handle dllimport linkage.
164 Name += "__imp_";
165 break;
166 case X86II::MO_COFFSTUB:
167 Name += ".refptr.";
168 break;
169 case X86II::MO_DARWIN_NONLAZY:
170 case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
171 Suffix = "$non_lazy_ptr";
172 break;
173 }
174
175 if (!Suffix.empty())
176 Name += DL.getInternalSymbolPrefix();
177
178 if (MO.isGlobal()) {
179 const GlobalValue *GV = MO.getGlobal();
180 AsmPrinter.getNameWithPrefix(Name, GV);
181 } else if (MO.isSymbol()) {
182 Mangler::getNameWithPrefix(OutName&: Name, GVName: MO.getSymbolName(), DL);
183 } else if (MO.isMBB()) {
184 assert(Suffix.empty());
185 Sym = MO.getMBB()->getSymbol();
186 }
187
188 Name += Suffix;
189 if (!Sym)
190 Sym = Ctx.getOrCreateSymbol(Name);
191
192 // If the target flags on the operand changes the name of the symbol, do that
193 // before we return the symbol.
194 switch (MO.getTargetFlags()) {
195 default:
196 break;
197 case X86II::MO_COFFSTUB: {
198 MachineModuleInfoCOFF &MMICOFF =
199 AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoCOFF>();
200 MachineModuleInfoImpl::StubValueTy &StubSym = MMICOFF.getGVStubEntry(Sym);
201 if (!StubSym.getPointer()) {
202 assert(MO.isGlobal() && "Extern symbol not handled yet");
203 StubSym = MachineModuleInfoImpl::StubValueTy(
204 AsmPrinter.getSymbol(GV: MO.getGlobal()), true);
205 }
206 break;
207 }
208 case X86II::MO_DARWIN_NONLAZY:
209 case X86II::MO_DARWIN_NONLAZY_PIC_BASE: {
210 MachineModuleInfoImpl::StubValueTy &StubSym =
211 getMachOMMI().getGVStubEntry(Sym);
212 if (!StubSym.getPointer()) {
213 assert(MO.isGlobal() && "Extern symbol not handled yet");
214 StubSym = MachineModuleInfoImpl::StubValueTy(
215 AsmPrinter.getSymbol(GV: MO.getGlobal()),
216 !MO.getGlobal()->hasInternalLinkage());
217 }
218 break;
219 }
220 }
221
222 return Sym;
223}
224
225MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
226 MCSymbol *Sym) const {
227 // FIXME: We would like an efficient form for this, so we don't have to do a
228 // lot of extra uniquing.
229 const MCExpr *Expr = nullptr;
230 uint16_t Specifier = X86::S_None;
231
232 switch (MO.getTargetFlags()) {
233 default:
234 llvm_unreachable("Unknown target flag on GV operand");
235 case X86II::MO_NO_FLAG: // No flag.
236 // These affect the name of the symbol, not any suffix.
237 case X86II::MO_DARWIN_NONLAZY:
238 case X86II::MO_DLLIMPORT:
239 case X86II::MO_COFFSTUB:
240 break;
241
242 case X86II::MO_TLVP:
243 Specifier = X86::S_TLVP;
244 break;
245 case X86II::MO_TLVP_PIC_BASE:
246 Expr = MCSymbolRefExpr::create(Symbol: Sym, specifier: X86::S_TLVP, Ctx);
247 // Subtract the pic base.
248 Expr = MCBinaryExpr::createSub(
249 LHS: Expr, RHS: MCSymbolRefExpr::create(Symbol: MF.getPICBaseSymbol(), Ctx), Ctx);
250 break;
251 case X86II::MO_SECREL:
252 Specifier = uint16_t(X86::S_COFF_SECREL);
253 break;
254 case X86II::MO_TLSGD:
255 Specifier = X86::S_TLSGD;
256 break;
257 case X86II::MO_TLSLD:
258 Specifier = X86::S_TLSLD;
259 break;
260 case X86II::MO_TLSLDM:
261 Specifier = X86::S_TLSLDM;
262 break;
263 case X86II::MO_GOTTPOFF:
264 Specifier = X86::S_GOTTPOFF;
265 break;
266 case X86II::MO_INDNTPOFF:
267 Specifier = X86::S_INDNTPOFF;
268 break;
269 case X86II::MO_TPOFF:
270 Specifier = X86::S_TPOFF;
271 break;
272 case X86II::MO_DTPOFF:
273 Specifier = X86::S_DTPOFF;
274 break;
275 case X86II::MO_NTPOFF:
276 Specifier = X86::S_NTPOFF;
277 break;
278 case X86II::MO_GOTNTPOFF:
279 Specifier = X86::S_GOTNTPOFF;
280 break;
281 case X86II::MO_GOTPCREL:
282 Specifier = X86::S_GOTPCREL;
283 break;
284 case X86II::MO_GOTPCREL_NORELAX:
285 Specifier = X86::S_GOTPCREL_NORELAX;
286 break;
287 case X86II::MO_GOT:
288 Specifier = X86::S_GOT;
289 break;
290 case X86II::MO_GOTOFF:
291 Specifier = X86::S_GOTOFF;
292 break;
293 case X86II::MO_PLT:
294 Specifier = X86::S_PLT;
295 break;
296 case X86II::MO_ABS8:
297 Specifier = X86::S_ABS8;
298 break;
299 case X86II::MO_PIC_BASE_OFFSET:
300 case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
301 Expr = MCSymbolRefExpr::create(Symbol: Sym, Ctx);
302 // Subtract the pic base.
303 Expr = MCBinaryExpr::createSub(
304 LHS: Expr, RHS: MCSymbolRefExpr::create(Symbol: MF.getPICBaseSymbol(), Ctx), Ctx);
305 if (MO.isJTI()) {
306 assert(MAI.doesSetDirectiveSuppressReloc());
307 // If .set directive is supported, use it to reduce the number of
308 // relocations the assembler will generate for differences between
309 // local labels. This is only safe when the symbols are in the same
310 // section so we are restricting it to jumptable references.
311 MCSymbol *Label = Ctx.createTempSymbol();
312 AsmPrinter.OutStreamer->emitAssignment(Symbol: Label, Value: Expr);
313 Expr = MCSymbolRefExpr::create(Symbol: Label, Ctx);
314 }
315 break;
316 }
317
318 if (!Expr)
319 Expr = MCSymbolRefExpr::create(Symbol: Sym, specifier: Specifier, Ctx);
320
321 if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
322 Expr = MCBinaryExpr::createAdd(
323 LHS: Expr, RHS: MCConstantExpr::create(Value: MO.getOffset(), Ctx), Ctx);
324 return MCOperand::createExpr(Val: Expr);
325}
326
327static unsigned getRetOpcode(const X86Subtarget &Subtarget) {
328 return Subtarget.is64Bit() ? X86::RET64 : X86::RET32;
329}
330
331MCOperand X86MCInstLower::LowerMachineOperand(const MachineInstr *MI,
332 const MachineOperand &MO) const {
333 switch (MO.getType()) {
334 default:
335 MI->print(OS&: errs());
336 llvm_unreachable("unknown operand type");
337 case MachineOperand::MO_Register:
338 // Ignore all implicit register operands.
339 if (MO.isImplicit())
340 return MCOperand();
341 return MCOperand::createReg(Reg: MO.getReg());
342 case MachineOperand::MO_Immediate:
343 return MCOperand::createImm(Val: MO.getImm());
344 case MachineOperand::MO_MachineBasicBlock:
345 case MachineOperand::MO_GlobalAddress:
346 case MachineOperand::MO_ExternalSymbol:
347 return LowerSymbolOperand(MO, Sym: GetSymbolFromOperand(MO));
348 case MachineOperand::MO_MCSymbol:
349 return LowerSymbolOperand(MO, Sym: MO.getMCSymbol());
350 case MachineOperand::MO_JumpTableIndex:
351 return LowerSymbolOperand(MO, Sym: AsmPrinter.GetJTISymbol(JTID: MO.getIndex()));
352 case MachineOperand::MO_ConstantPoolIndex:
353 return LowerSymbolOperand(MO, Sym: AsmPrinter.GetCPISymbol(CPID: MO.getIndex()));
354 case MachineOperand::MO_BlockAddress:
355 return LowerSymbolOperand(
356 MO, Sym: AsmPrinter.GetBlockAddressSymbol(BA: MO.getBlockAddress()));
357 case MachineOperand::MO_RegisterMask:
358 // Ignore call clobbers.
359 return MCOperand();
360 }
361}
362
363// Replace TAILJMP opcodes with their equivalent opcodes that have encoding
364// information.
365static unsigned convertTailJumpOpcode(unsigned Opcode, bool IsLarge = false) {
366 switch (Opcode) {
367 case X86::TAILJMPr:
368 Opcode = X86::JMP32r;
369 break;
370 case X86::TAILJMPm:
371 Opcode = X86::JMP32m;
372 break;
373 case X86::TAILJMPr64:
374 Opcode = X86::JMP64r;
375 break;
376 case X86::TAILJMPm64:
377 Opcode = X86::JMP64m;
378 break;
379 case X86::TAILJMPr64_REX:
380 Opcode = X86::JMP64r_REX;
381 break;
382 case X86::TAILJMPm64_REX:
383 Opcode = X86::JMP64m_REX;
384 break;
385 case X86::TAILJMPd:
386 case X86::TAILJMPd64:
387 Opcode = IsLarge ? X86::JMPABS64i : X86::JMP_1;
388 break;
389 case X86::TAILJMPd_CC:
390 case X86::TAILJMPd64_CC:
391 Opcode = X86::JCC_1;
392 break;
393 }
394
395 return Opcode;
396}
397
398void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
399 OutMI.setOpcode(MI->getOpcode());
400
401 for (const MachineOperand &MO : MI->operands())
402 if (auto Op = LowerMachineOperand(MI, MO); Op.isValid())
403 OutMI.addOperand(Op);
404
405 bool In64BitMode = AsmPrinter.getSubtarget().is64Bit();
406 if (X86::optimizeInstFromVEX3ToVEX2(MI&: OutMI, Desc: MI->getDesc()) ||
407 X86::optimizeShiftRotateWithImmediateOne(MI&: OutMI) ||
408 X86::optimizeVPCMPWithImmediateOneOrSix(MI&: OutMI) ||
409 X86::optimizeMOVSX(MI&: OutMI) || X86::optimizeINCDEC(MI&: OutMI, In64BitMode) ||
410 X86::optimizeMOV(MI&: OutMI, In64BitMode) ||
411 X86::optimizeToFixedRegisterOrShortImmediateForm(MI&: OutMI))
412 return;
413
414 // Handle a few special cases to eliminate operand modifiers.
415 switch (OutMI.getOpcode()) {
416 case X86::LEA64_32r:
417 case X86::LEA64r:
418 case X86::LEA16r:
419 case X86::LEA32r:
420 // LEA should have a segment register, but it must be empty.
421 assert(OutMI.getNumOperands() == 1 + X86::AddrNumOperands &&
422 "Unexpected # of LEA operands");
423 assert(OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 &&
424 "LEA has segment specified!");
425 break;
426 case X86::MULX32Hrr:
427 case X86::MULX32Hrm:
428 case X86::MULX64Hrr:
429 case X86::MULX64Hrm: {
430 // Turn into regular MULX by duplicating the destination.
431 unsigned NewOpc;
432 switch (OutMI.getOpcode()) {
433 default: llvm_unreachable("Invalid opcode");
434 case X86::MULX32Hrr: NewOpc = X86::MULX32rr; break;
435 case X86::MULX32Hrm: NewOpc = X86::MULX32rm; break;
436 case X86::MULX64Hrr: NewOpc = X86::MULX64rr; break;
437 case X86::MULX64Hrm: NewOpc = X86::MULX64rm; break;
438 }
439 OutMI.setOpcode(NewOpc);
440 // Duplicate the destination.
441 MCRegister DestReg = OutMI.getOperand(i: 0).getReg();
442 OutMI.insert(I: OutMI.begin(), Op: MCOperand::createReg(Reg: DestReg));
443 break;
444 }
445 // CALL64r, CALL64pcrel32 - These instructions used to have
446 // register inputs modeled as normal uses instead of implicit uses. As such,
447 // they we used to truncate off all but the first operand (the callee). This
448 // issue seems to have been fixed at some point. This assert verifies that.
449 case X86::CALL64r:
450 case X86::CALL64pcrel32:
451 assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
452 break;
453 case X86::EH_RETURN:
454 case X86::EH_RETURN64: {
455 OutMI = MCInst();
456 OutMI.setOpcode(getRetOpcode(Subtarget: AsmPrinter.getSubtarget()));
457 break;
458 }
459 case X86::CLEANUPRET: {
460 // Replace CLEANUPRET with the appropriate RET.
461 OutMI = MCInst();
462 OutMI.setOpcode(getRetOpcode(Subtarget: AsmPrinter.getSubtarget()));
463 break;
464 }
465 case X86::CATCHRET: {
466 // Replace CATCHRET with the appropriate RET.
467 const X86Subtarget &Subtarget = AsmPrinter.getSubtarget();
468 unsigned ReturnReg = In64BitMode ? X86::RAX : X86::EAX;
469 OutMI = MCInst();
470 OutMI.setOpcode(getRetOpcode(Subtarget));
471 OutMI.addOperand(Op: MCOperand::createReg(Reg: ReturnReg));
472 break;
473 }
474 // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump
475 // instruction.
476 case X86::TAILJMPr:
477 case X86::TAILJMPr64:
478 case X86::TAILJMPr64_REX:
479 case X86::TAILJMPd:
480 assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
481 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode()));
482 break;
483 case X86::TAILJMPd64: {
484 assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
485 bool IsLarge = TM.getCodeModel() == CodeModel::Large;
486 assert((!IsLarge || AsmPrinter.getSubtarget().hasJMPABS()) &&
487 "Unexpected TAILJMPd64 in large code model without JMPABS");
488 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode(), IsLarge));
489 break;
490 }
491 case X86::TAILJMPd_CC:
492 case X86::TAILJMPd64_CC:
493 assert(OutMI.getNumOperands() == 2 && "Unexpected number of operands!");
494 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode()));
495 break;
496 case X86::TAILJMPm:
497 case X86::TAILJMPm64:
498 case X86::TAILJMPm64_REX:
499 assert(OutMI.getNumOperands() == X86::AddrNumOperands &&
500 "Unexpected number of operands!");
501 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode()));
502 break;
503 case X86::MASKMOVDQU:
504 case X86::VMASKMOVDQU:
505 if (In64BitMode)
506 OutMI.setFlags(X86::IP_HAS_AD_SIZE);
507 break;
508 case X86::BSF16rm:
509 case X86::BSF16rr:
510 case X86::BSF32rm:
511 case X86::BSF32rr:
512 case X86::BSF64rm:
513 case X86::BSF64rr: {
514 // Add an REP prefix to BSF instructions so that new processors can
515 // recognize as TZCNT, which has better performance than BSF.
516 // BSF and TZCNT have different interpretations on ZF bit. So make sure
517 // it won't be used later.
518 const MachineOperand *FlagDef =
519 MI->findRegisterDefOperand(Reg: X86::EFLAGS, /*TRI=*/nullptr);
520 if (!MF.getFunction().hasOptSize() && FlagDef && FlagDef->isDead())
521 OutMI.setFlags(X86::IP_HAS_REPEAT);
522 break;
523 }
524 default:
525 break;
526 }
527}
528
529void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
530 const MachineInstr &MI) {
531 NoAutoPaddingScope NoPadScope(*OutStreamer);
532 bool Is64Bits = getSubtarget().is64Bit();
533 bool Is64BitsLP64 = getSubtarget().isTarget64BitLP64();
534 MCContext &Ctx = OutStreamer->getContext();
535
536 X86::Specifier Specifier;
537 switch (MI.getOpcode()) {
538 case X86::TLS_addr32:
539 case X86::TLS_addr64:
540 case X86::TLS_addrX32:
541 Specifier = X86::S_TLSGD;
542 break;
543 case X86::TLS_base_addr32:
544 Specifier = X86::S_TLSLDM;
545 break;
546 case X86::TLS_base_addr64:
547 case X86::TLS_base_addrX32:
548 Specifier = X86::S_TLSLD;
549 break;
550 case X86::TLS_desc32:
551 case X86::TLS_desc64:
552 Specifier = X86::S_TLSDESC;
553 break;
554 default:
555 llvm_unreachable("unexpected opcode");
556 }
557
558 const MCSymbolRefExpr *Sym = MCSymbolRefExpr::create(
559 Symbol: MCInstLowering.GetSymbolFromOperand(MO: MI.getOperand(i: 3)), specifier: Specifier, Ctx);
560
561 // Before binutils 2.41, ld has a bogus TLS relaxation error when the GD/LD
562 // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is
563 // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by
564 // only using GOT when GOTPCRELX is enabled.
565 // TODO Delete the workaround when rustc no longer relies on the hack
566 bool UseGot = MMI->getModule()->getRtLibUseGOT() &&
567 Ctx.getTargetOptions().X86RelaxRelocations;
568
569 if (Specifier == X86::S_TLSDESC) {
570 const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(
571 Symbol: MCInstLowering.GetSymbolFromOperand(MO: MI.getOperand(i: 3)), specifier: X86::S_TLSCALL,
572 Ctx);
573 EmitAndCountInstruction(
574 Inst&: MCInstBuilder(Is64BitsLP64 ? X86::LEA64r : X86::LEA32r)
575 .addReg(Reg: Is64BitsLP64 ? X86::RAX : X86::EAX)
576 .addReg(Reg: Is64Bits ? X86::RIP : X86::EBX)
577 .addImm(Val: 1)
578 .addReg(Reg: 0)
579 .addExpr(Val: Sym)
580 .addReg(Reg: 0));
581 EmitAndCountInstruction(
582 Inst&: MCInstBuilder(Is64Bits ? X86::CALL64m : X86::CALL32m)
583 .addReg(Reg: Is64BitsLP64 ? X86::RAX : X86::EAX)
584 .addImm(Val: 1)
585 .addReg(Reg: 0)
586 .addExpr(Val: Expr)
587 .addReg(Reg: 0));
588 } else if (Is64Bits) {
589 bool NeedsPadding = Specifier == X86::S_TLSGD;
590 if (NeedsPadding && Is64BitsLP64)
591 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DATA16_PREFIX));
592 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::LEA64r)
593 .addReg(Reg: X86::RDI)
594 .addReg(Reg: X86::RIP)
595 .addImm(Val: 1)
596 .addReg(Reg: 0)
597 .addExpr(Val: Sym)
598 .addReg(Reg: 0));
599 const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol(Name: "__tls_get_addr");
600 if (NeedsPadding) {
601 if (!UseGot)
602 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DATA16_PREFIX));
603 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DATA16_PREFIX));
604 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::REX64_PREFIX));
605 }
606 if (UseGot) {
607 const MCExpr *Expr =
608 MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_GOTPCREL, Ctx);
609 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64m)
610 .addReg(Reg: X86::RIP)
611 .addImm(Val: 1)
612 .addReg(Reg: 0)
613 .addExpr(Val: Expr)
614 .addReg(Reg: 0));
615 } else {
616 EmitAndCountInstruction(
617 Inst&: MCInstBuilder(X86::CALL64pcrel32)
618 .addExpr(Val: MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_PLT, Ctx)));
619 }
620 } else {
621 if (Specifier == X86::S_TLSGD && !UseGot) {
622 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::LEA32r)
623 .addReg(Reg: X86::EAX)
624 .addReg(Reg: 0)
625 .addImm(Val: 1)
626 .addReg(Reg: X86::EBX)
627 .addExpr(Val: Sym)
628 .addReg(Reg: 0));
629 } else {
630 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::LEA32r)
631 .addReg(Reg: X86::EAX)
632 .addReg(Reg: X86::EBX)
633 .addImm(Val: 1)
634 .addReg(Reg: 0)
635 .addExpr(Val: Sym)
636 .addReg(Reg: 0));
637 }
638
639 const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol(Name: "___tls_get_addr");
640 if (UseGot) {
641 const MCExpr *Expr = MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_GOT, Ctx);
642 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL32m)
643 .addReg(Reg: X86::EBX)
644 .addImm(Val: 1)
645 .addReg(Reg: 0)
646 .addExpr(Val: Expr)
647 .addReg(Reg: 0));
648 } else {
649 EmitAndCountInstruction(
650 Inst&: MCInstBuilder(X86::CALLpcrel32)
651 .addExpr(Val: MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_PLT, Ctx)));
652 }
653 }
654}
655
656/// Emit the largest nop instruction smaller than or equal to \p NumBytes
657/// bytes. Return the size of nop emitted.
658static unsigned emitNop(MCStreamer &OS, unsigned NumBytes,
659 const X86Subtarget *Subtarget) {
660 // Determine the longest nop which can be efficiently decoded for the given
661 // target cpu. 15-bytes is the longest single NOP instruction, but some
662 // platforms can't decode the longest forms efficiently.
663 unsigned MaxNopLength = 1;
664 if (Subtarget->is64Bit()) {
665 // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the
666 // IndexReg/BaseReg below need to be updated.
667 if (Subtarget->hasFeature(Feature: X86::TuningFast7ByteNOP))
668 MaxNopLength = 7;
669 else if (Subtarget->hasFeature(Feature: X86::TuningFast15ByteNOP))
670 MaxNopLength = 15;
671 else if (Subtarget->hasFeature(Feature: X86::TuningFast11ByteNOP))
672 MaxNopLength = 11;
673 else
674 MaxNopLength = 10;
675 } if (Subtarget->is32Bit())
676 MaxNopLength = 2;
677
678 // Cap a single nop emission at the profitable value for the target
679 NumBytes = std::min(a: NumBytes, b: MaxNopLength);
680
681 unsigned NopSize;
682 unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg;
683 IndexReg = Displacement = SegmentReg = 0;
684 BaseReg = X86::RAX;
685 ScaleVal = 1;
686 switch (NumBytes) {
687 case 0:
688 llvm_unreachable("Zero nops?");
689 break;
690 case 1:
691 NopSize = 1;
692 Opc = X86::NOOP;
693 break;
694 case 2:
695 NopSize = 2;
696 Opc = X86::XCHG16ar;
697 break;
698 case 3:
699 NopSize = 3;
700 Opc = X86::NOOPL;
701 break;
702 case 4:
703 NopSize = 4;
704 Opc = X86::NOOPL;
705 Displacement = 8;
706 break;
707 case 5:
708 NopSize = 5;
709 Opc = X86::NOOPL;
710 Displacement = 8;
711 IndexReg = X86::RAX;
712 break;
713 case 6:
714 NopSize = 6;
715 Opc = X86::NOOPW;
716 Displacement = 8;
717 IndexReg = X86::RAX;
718 break;
719 case 7:
720 NopSize = 7;
721 Opc = X86::NOOPL;
722 Displacement = 512;
723 break;
724 case 8:
725 NopSize = 8;
726 Opc = X86::NOOPL;
727 Displacement = 512;
728 IndexReg = X86::RAX;
729 break;
730 case 9:
731 NopSize = 9;
732 Opc = X86::NOOPW;
733 Displacement = 512;
734 IndexReg = X86::RAX;
735 break;
736 default:
737 NopSize = 10;
738 Opc = X86::NOOPW;
739 Displacement = 512;
740 IndexReg = X86::RAX;
741 SegmentReg = X86::CS;
742 break;
743 }
744
745 unsigned NumPrefixes = std::min(a: NumBytes - NopSize, b: 5U);
746 NopSize += NumPrefixes;
747 for (unsigned i = 0; i != NumPrefixes; ++i)
748 OS.emitBytes(Data: "\x66");
749
750 switch (Opc) {
751 default: llvm_unreachable("Unexpected opcode");
752 case X86::NOOP:
753 OS.emitInstruction(Inst: MCInstBuilder(Opc), STI: *Subtarget);
754 break;
755 case X86::XCHG16ar:
756 OS.emitInstruction(Inst: MCInstBuilder(Opc).addReg(Reg: X86::AX).addReg(Reg: X86::AX),
757 STI: *Subtarget);
758 break;
759 case X86::NOOPL:
760 case X86::NOOPW:
761 OS.emitInstruction(Inst: MCInstBuilder(Opc)
762 .addReg(Reg: BaseReg)
763 .addImm(Val: ScaleVal)
764 .addReg(Reg: IndexReg)
765 .addImm(Val: Displacement)
766 .addReg(Reg: SegmentReg),
767 STI: *Subtarget);
768 break;
769 }
770 assert(NopSize <= NumBytes && "We overemitted?");
771 return NopSize;
772}
773
774/// Emit the optimal amount of multi-byte nops on X86.
775static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
776 const X86Subtarget *Subtarget) {
777 unsigned NopsToEmit = NumBytes;
778 (void)NopsToEmit;
779 while (NumBytes) {
780 NumBytes -= emitNop(OS, NumBytes, Subtarget);
781 assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!");
782 }
783}
784
785void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
786 X86MCInstLower &MCIL) {
787 assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
788
789 NoAutoPaddingScope NoPadScope(*OutStreamer);
790
791 StatepointOpers SOpers(&MI);
792 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
793 emitX86Nops(OS&: *OutStreamer, NumBytes: PatchBytes, Subtarget);
794 } else {
795 // Lower call target and choose correct opcode
796 const MachineOperand &CallTarget = SOpers.getCallTarget();
797 MCOperand CallTargetMCOp;
798 unsigned CallOpcode;
799 switch (CallTarget.getType()) {
800 case MachineOperand::MO_GlobalAddress:
801 case MachineOperand::MO_ExternalSymbol:
802 CallTargetMCOp = MCIL.LowerSymbolOperand(
803 MO: CallTarget, Sym: MCIL.GetSymbolFromOperand(MO: CallTarget));
804 CallOpcode = X86::CALL64pcrel32;
805 // Currently, we only support relative addressing with statepoints.
806 // Otherwise, we'll need a scratch register to hold the target
807 // address. You'll fail asserts during load & relocation if this
808 // symbol is to far away. (TODO: support non-relative addressing)
809 break;
810 case MachineOperand::MO_Immediate:
811 CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm());
812 CallOpcode = X86::CALL64pcrel32;
813 // Currently, we only support relative addressing with statepoints.
814 // Otherwise, we'll need a scratch register to hold the target
815 // immediate. You'll fail asserts during load & relocation if this
816 // address is to far away. (TODO: support non-relative addressing)
817 break;
818 case MachineOperand::MO_Register:
819 // FIXME: Add retpoline support and remove this.
820 if (Subtarget->useIndirectThunkCalls())
821 report_fatal_error(reason: "Lowering register statepoints with thunks not "
822 "yet implemented.");
823 CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg());
824 CallOpcode = X86::CALL64r;
825 break;
826 default:
827 llvm_unreachable("Unsupported operand type in statepoint call target");
828 break;
829 }
830
831 // Emit call
832 MCInst CallInst;
833 CallInst.setOpcode(CallOpcode);
834 CallInst.addOperand(Op: CallTargetMCOp);
835 OutStreamer->emitInstruction(Inst: CallInst, STI: getSubtargetInfo());
836 maybeEmitNopAfterCallForWindowsEH(MI: &MI);
837 }
838
839 // Record our statepoint node in the same section used by STACKMAP
840 // and PATCHPOINT
841 auto &Ctx = OutStreamer->getContext();
842 MCSymbol *MILabel = Ctx.createTempSymbol();
843 OutStreamer->emitLabel(Symbol: MILabel);
844 SM.recordStatepoint(L: *MILabel, MI);
845}
846
847void X86AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI,
848 X86MCInstLower &MCIL) {
849 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
850 // <opcode>, <operands>
851
852 NoAutoPaddingScope NoPadScope(*OutStreamer);
853
854 Register DefRegister = FaultingMI.getOperand(i: 0).getReg();
855 FaultMaps::FaultKind FK =
856 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm());
857 MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol();
858 unsigned Opcode = FaultingMI.getOperand(i: 3).getImm();
859 unsigned OperandsBeginIdx = 4;
860
861 auto &Ctx = OutStreamer->getContext();
862 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
863 OutStreamer->emitLabel(Symbol: FaultingLabel);
864
865 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
866 FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel);
867
868 MCInst MI;
869 MI.setOpcode(Opcode);
870
871 if (DefRegister != X86::NoRegister)
872 MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister));
873
874 for (const MachineOperand &MO :
875 llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx))
876 if (auto Op = MCIL.LowerMachineOperand(MI: &FaultingMI, MO); Op.isValid())
877 MI.addOperand(Op);
878
879 OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName());
880 OutStreamer->emitInstruction(Inst: MI, STI: getSubtargetInfo());
881}
882
883void X86AsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI,
884 X86MCInstLower &MCIL) {
885 bool Is64Bits = Subtarget->is64Bit();
886 MCContext &Ctx = OutStreamer->getContext();
887 MCSymbol *fentry = Ctx.getOrCreateSymbol(Name: "__fentry__");
888 const MCSymbolRefExpr *Op = MCSymbolRefExpr::create(Symbol: fentry, Ctx);
889
890 EmitAndCountInstruction(
891 Inst&: MCInstBuilder(Is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32)
892 .addExpr(Val: Op));
893}
894
895void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
896 assert(std::next(MI.getIterator())->isCall() &&
897 "KCFI_CHECK not followed by a call instruction");
898
899 // Adjust the offset for patchable-function-prefix. X86InstrInfo::getNop()
900 // returns a 1-byte X86::NOOP, which means the offset is the same in
901 // bytes. This assumes that patchable-function-prefix is the same for all
902 // functions.
903 const MachineFunction &MF = *MI.getMF();
904 int64_t PrefixNops = MF.getFunction().getFnAttributeAsParsedInteger(
905 Kind: "patchable-function-prefix");
906
907 // KCFI allows indirect calls to any location that's preceded by a valid
908 // type identifier. To avoid encoding the full constant into an instruction,
909 // and thus emitting potential call target gadgets at each indirect call
910 // site, load a negated constant to a register and compare that to the
911 // expected value at the call target.
912 const Register AddrReg = MI.getOperand(i: 0).getReg();
913 const uint32_t Type = MI.getOperand(i: 1).getImm();
914 // The check is immediately before the call. If the call target is in R10,
915 // we can clobber R11 for the check instead.
916 unsigned TempReg = AddrReg == X86::R10 ? X86::R11D : X86::R10D;
917 EmitAndCountInstruction(
918 Inst&: MCInstBuilder(X86::MOV32ri).addReg(Reg: TempReg).addImm(Val: -MaskKCFIType(Value: Type)));
919 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::ADD32rm)
920 .addReg(Reg: X86::NoRegister)
921 .addReg(Reg: TempReg)
922 .addReg(Reg: AddrReg)
923 .addImm(Val: 1)
924 .addReg(Reg: X86::NoRegister)
925 .addImm(Val: -(PrefixNops + 4))
926 .addReg(Reg: X86::NoRegister));
927
928 MCSymbol *Pass = OutContext.createTempSymbol();
929 EmitAndCountInstruction(
930 Inst&: MCInstBuilder(X86::JCC_1)
931 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Pass, Ctx&: OutContext))
932 .addImm(Val: X86::COND_E));
933
934 MCSymbol *Trap = OutContext.createTempSymbol();
935 OutStreamer->emitLabel(Symbol: Trap);
936 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::TRAP));
937 emitKCFITrapEntry(MF, Symbol: Trap);
938 OutStreamer->emitLabel(Symbol: Pass);
939}
940
941void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
942 // FIXME: Make this work on non-ELF.
943 if (!TM.getTargetTriple().isOSBinFormatELF()) {
944 report_fatal_error(reason: "llvm.asan.check.memaccess only supported on ELF");
945 return;
946 }
947
948 const auto &Reg = MI.getOperand(i: 0).getReg();
949 ASanAccessInfo AccessInfo(MI.getOperand(i: 1).getImm());
950
951 uint64_t ShadowBase;
952 int MappingScale;
953 bool OrShadowOffset;
954 getAddressSanitizerParams(Ctx: MF->getFunction().getContext(),
955 TargetTriple: TM.getTargetTriple(), LongSize: 64, IsKasan: AccessInfo.CompileKernel,
956 ShadowBase: &ShadowBase, MappingScale: &MappingScale, OrShadowOffset: &OrShadowOffset);
957
958 StringRef Name = AccessInfo.IsWrite ? "store" : "load";
959 StringRef Op = OrShadowOffset ? "or" : "add";
960 std::string SymName = ("__asan_check_" + Name + "_" + Op + "_" +
961 Twine(1ULL << AccessInfo.AccessSizeIndex) + "_" +
962 TM.getMCRegisterInfo().getName(RegNo: Reg.asMCReg()))
963 .str();
964 if (OrShadowOffset)
965 report_fatal_error(
966 reason: "OrShadowOffset is not supported with optimized callbacks");
967
968 EmitAndCountInstruction(
969 Inst&: MCInstBuilder(X86::CALL64pcrel32)
970 .addExpr(Val: MCSymbolRefExpr::create(
971 Symbol: OutContext.getOrCreateSymbol(Name: SymName), Ctx&: OutContext)));
972}
973
974void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI,
975 X86MCInstLower &MCIL) {
976 // PATCHABLE_OP minsize
977
978 NoAutoPaddingScope NoPadScope(*OutStreamer);
979
980 auto NextMI = std::find_if(first: std::next(x: MI.getIterator()),
981 last: MI.getParent()->end().getInstrIterator(),
982 pred: [](auto &II) { return !II.isMetaInstruction(); });
983
984 SmallString<256> Code;
985 unsigned MinSize = MI.getOperand(i: 0).getImm();
986
987 if (NextMI != MI.getParent()->end() && !NextMI->isInlineAsm()) {
988 // Lower the next MachineInstr to find its byte size.
989 // If the next instruction is inline assembly, we skip lowering it for now,
990 // and assume we should always generate NOPs.
991 MCInst MCI;
992 MCIL.Lower(MI: &*NextMI, OutMI&: MCI);
993
994 SmallVector<MCFixup, 4> Fixups;
995 CodeEmitter->encodeInstruction(Inst: MCI, CB&: Code, Fixups, STI: getSubtargetInfo());
996 }
997
998 if (Code.size() < MinSize) {
999 if (MinSize == 2 && Subtarget->is32Bit() &&
1000 Subtarget->isTargetWindowsMSVC() &&
1001 (Subtarget->getCPU().empty() || Subtarget->getCPU() == "pentium3")) {
1002 // For compatibility reasons, when targetting MSVC, it is important to
1003 // generate a 'legacy' NOP in the form of a 8B FF MOV EDI, EDI. Some tools
1004 // rely specifically on this pattern to be able to patch a function.
1005 // This is only for 32-bit targets, when using /arch:IA32 or /arch:SSE.
1006 OutStreamer->emitInstruction(
1007 Inst: MCInstBuilder(X86::MOV32rr_REV).addReg(Reg: X86::EDI).addReg(Reg: X86::EDI),
1008 STI: *Subtarget);
1009 } else {
1010 unsigned NopSize = emitNop(OS&: *OutStreamer, NumBytes: MinSize, Subtarget);
1011 assert(NopSize == MinSize && "Could not implement MinSize!");
1012 (void)NopSize;
1013 }
1014 }
1015}
1016
1017// Lower a stackmap of the form:
1018// <id>, <shadowBytes>, ...
1019void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
1020 SMShadowTracker.emitShadowPadding(OutStreamer&: *OutStreamer, STI: getSubtargetInfo());
1021
1022 auto &Ctx = OutStreamer->getContext();
1023 MCSymbol *MILabel = Ctx.createTempSymbol();
1024 OutStreamer->emitLabel(Symbol: MILabel);
1025
1026 SM.recordStackMap(L: *MILabel, MI);
1027 unsigned NumShadowBytes = MI.getOperand(i: 1).getImm();
1028 SMShadowTracker.reset(RequiredSize: NumShadowBytes);
1029}
1030
1031// Lower a patchpoint of the form:
1032// [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ...
1033void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
1034 X86MCInstLower &MCIL) {
1035 assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64");
1036
1037 SMShadowTracker.emitShadowPadding(OutStreamer&: *OutStreamer, STI: getSubtargetInfo());
1038
1039 NoAutoPaddingScope NoPadScope(*OutStreamer);
1040
1041 auto &Ctx = OutStreamer->getContext();
1042 MCSymbol *MILabel = Ctx.createTempSymbol();
1043 OutStreamer->emitLabel(Symbol: MILabel);
1044 SM.recordPatchPoint(L: *MILabel, MI);
1045
1046 PatchPointOpers opers(&MI);
1047 unsigned ScratchIdx = opers.getNextScratchIdx();
1048 unsigned EncodedBytes = 0;
1049 const MachineOperand &CalleeMO = opers.getCallTarget();
1050
1051 // Check for null target. If target is non-null (i.e. is non-zero or is
1052 // symbolic) then emit a call.
1053 if (!(CalleeMO.isImm() && !CalleeMO.getImm())) {
1054 MCOperand CalleeMCOp;
1055 switch (CalleeMO.getType()) {
1056 default:
1057 /// FIXME: Add a verifier check for bad callee types.
1058 llvm_unreachable("Unrecognized callee operand type.");
1059 case MachineOperand::MO_Immediate:
1060 if (CalleeMO.getImm())
1061 CalleeMCOp = MCOperand::createImm(Val: CalleeMO.getImm());
1062 break;
1063 case MachineOperand::MO_ExternalSymbol:
1064 case MachineOperand::MO_GlobalAddress:
1065 CalleeMCOp = MCIL.LowerSymbolOperand(MO: CalleeMO,
1066 Sym: MCIL.GetSymbolFromOperand(MO: CalleeMO));
1067 break;
1068 }
1069
1070 // Emit MOV to materialize the target address and the CALL to target.
1071 // This is encoded with 12-13 bytes, depending on which register is used.
1072 Register ScratchReg = MI.getOperand(i: ScratchIdx).getReg();
1073 if (X86II::isX86_64ExtendedReg(Reg: ScratchReg))
1074 EncodedBytes = 13;
1075 else
1076 EncodedBytes = 12;
1077
1078 EmitAndCountInstruction(
1079 Inst&: MCInstBuilder(X86::MOV64ri).addReg(Reg: ScratchReg).addOperand(Op: CalleeMCOp));
1080 // FIXME: Add retpoline support and remove this.
1081 if (Subtarget->useIndirectThunkCalls())
1082 report_fatal_error(
1083 reason: "Lowering patchpoint with thunks not yet implemented.");
1084 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64r).addReg(Reg: ScratchReg));
1085 }
1086
1087 // Emit padding.
1088 unsigned NumBytes = opers.getNumPatchBytes();
1089 assert(NumBytes >= EncodedBytes &&
1090 "Patchpoint can't request size less than the length of a call.");
1091
1092 emitX86Nops(OS&: *OutStreamer, NumBytes: NumBytes - EncodedBytes, Subtarget);
1093}
1094
1095void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
1096 X86MCInstLower &MCIL) {
1097 assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64");
1098
1099 NoAutoPaddingScope NoPadScope(*OutStreamer);
1100
1101 // We want to emit the following pattern, which follows the x86 calling
1102 // convention to prepare for the trampoline call to be patched in.
1103 //
1104 // .p2align 1, ...
1105 // .Lxray_event_sled_N:
1106 // jmp +N // jump across the instrumentation sled
1107 // ... // set up arguments in register
1108 // callq __xray_CustomEvent@plt // force dependency to symbol
1109 // ...
1110 // <jump here>
1111 //
1112 // After patching, it would look something like:
1113 //
1114 // nopw (2-byte nop)
1115 // ...
1116 // callq __xrayCustomEvent // already lowered
1117 // ...
1118 //
1119 // ---
1120 // First we emit the label and the jump.
1121 auto CurSled = OutContext.createTempSymbol(Name: "xray_event_sled_", AlwaysAddSuffix: true);
1122 OutStreamer->AddComment(T: "# XRay Custom Event Log");
1123 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1124 OutStreamer->emitLabel(Symbol: CurSled);
1125
1126 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1127 // an operand (computed as an offset from the jmp instruction).
1128 // FIXME: Find another less hacky way do force the relative jump.
1129 OutStreamer->emitBinaryData(Data: "\xeb\x0f");
1130
1131 // The default C calling convention will place two arguments into %rcx and
1132 // %rdx -- so we only work with those.
1133 const Register DestRegs[] = {X86::RDI, X86::RSI};
1134 bool UsedMask[] = {false, false};
1135 // Filled out in loop.
1136 Register SrcRegs[] = {0, 0};
1137
1138 // Then we put the operands in the %rdi and %rsi registers. We spill the
1139 // values in the register before we clobber them, and mark them as used in
1140 // UsedMask. In case the arguments are already in the correct register, we use
1141 // emit nops appropriately sized to keep the sled the same size in every
1142 // situation.
1143 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1144 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO: MI.getOperand(i: I));
1145 Op.isValid()) {
1146 assert(Op.isReg() && "Only support arguments in registers");
1147 SrcRegs[I] = getX86SubSuperRegister(Reg: Op.getReg(), Size: 64);
1148 assert(SrcRegs[I].isValid() && "Invalid operand");
1149 if (SrcRegs[I] != DestRegs[I]) {
1150 UsedMask[I] = true;
1151 EmitAndCountInstruction(
1152 Inst&: MCInstBuilder(X86::PUSH64r).addReg(Reg: DestRegs[I]));
1153 } else {
1154 emitX86Nops(OS&: *OutStreamer, NumBytes: 4, Subtarget);
1155 }
1156 }
1157
1158 // Now that the register values are stashed, mov arguments into place.
1159 // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1160 // earlier DestReg. We will have already overwritten over the register before
1161 // we can copy from it.
1162 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1163 if (SrcRegs[I] != DestRegs[I])
1164 EmitAndCountInstruction(
1165 Inst&: MCInstBuilder(X86::MOV64rr).addReg(Reg: DestRegs[I]).addReg(Reg: SrcRegs[I]));
1166
1167 // We emit a hard dependency on the __xray_CustomEvent symbol, which is the
1168 // name of the trampoline to be implemented by the XRay runtime.
1169 auto TSym = OutContext.getOrCreateSymbol(Name: "__xray_CustomEvent");
1170 MachineOperand TOp = MachineOperand::CreateMCSymbol(Sym: TSym);
1171 if (isPositionIndependent())
1172 TOp.setTargetFlags(X86II::MO_PLT);
1173
1174 // Emit the call instruction.
1175 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64pcrel32)
1176 .addOperand(Op: MCIL.LowerSymbolOperand(MO: TOp, Sym: TSym)));
1177
1178 // Restore caller-saved and used registers.
1179 for (unsigned I = sizeof UsedMask; I-- > 0;)
1180 if (UsedMask[I])
1181 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::POP64r).addReg(Reg: DestRegs[I]));
1182 else
1183 emitX86Nops(OS&: *OutStreamer, NumBytes: 1, Subtarget);
1184
1185 OutStreamer->AddComment(T: "xray custom event end.");
1186
1187 // Record the sled version. Version 0 of this sled was spelled differently, so
1188 // we let the runtime handle the different offsets we're using. Version 2
1189 // changed the absolute address to a PC-relative address.
1190 recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2);
1191}
1192
1193void X86AsmPrinter::LowerPATCHABLE_TYPED_EVENT_CALL(const MachineInstr &MI,
1194 X86MCInstLower &MCIL) {
1195 assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64");
1196
1197 NoAutoPaddingScope NoPadScope(*OutStreamer);
1198
1199 // We want to emit the following pattern, which follows the x86 calling
1200 // convention to prepare for the trampoline call to be patched in.
1201 //
1202 // .p2align 1, ...
1203 // .Lxray_event_sled_N:
1204 // jmp +N // jump across the instrumentation sled
1205 // ... // set up arguments in register
1206 // callq __xray_TypedEvent@plt // force dependency to symbol
1207 // ...
1208 // <jump here>
1209 //
1210 // After patching, it would look something like:
1211 //
1212 // nopw (2-byte nop)
1213 // ...
1214 // callq __xrayTypedEvent // already lowered
1215 // ...
1216 //
1217 // ---
1218 // First we emit the label and the jump.
1219 auto CurSled = OutContext.createTempSymbol(Name: "xray_typed_event_sled_", AlwaysAddSuffix: true);
1220 OutStreamer->AddComment(T: "# XRay Typed Event Log");
1221 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1222 OutStreamer->emitLabel(Symbol: CurSled);
1223
1224 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1225 // an operand (computed as an offset from the jmp instruction).
1226 // FIXME: Find another less hacky way do force the relative jump.
1227 OutStreamer->emitBinaryData(Data: "\xeb\x14");
1228
1229 // An x86-64 convention may place three arguments into %rcx, %rdx, and R8,
1230 // so we'll work with those. Or we may be called via SystemV, in which case
1231 // we don't have to do any translation.
1232 const Register DestRegs[] = {X86::RDI, X86::RSI, X86::RDX};
1233 bool UsedMask[] = {false, false, false};
1234
1235 // Will fill out src regs in the loop.
1236 Register SrcRegs[] = {0, 0, 0};
1237
1238 // Then we put the operands in the SystemV registers. We spill the values in
1239 // the registers before we clobber them, and mark them as used in UsedMask.
1240 // In case the arguments are already in the correct register, we emit nops
1241 // appropriately sized to keep the sled the same size in every situation.
1242 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1243 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO: MI.getOperand(i: I));
1244 Op.isValid()) {
1245 // TODO: Is register only support adequate?
1246 assert(Op.isReg() && "Only supports arguments in registers");
1247 SrcRegs[I] = getX86SubSuperRegister(Reg: Op.getReg(), Size: 64);
1248 assert(SrcRegs[I].isValid() && "Invalid operand");
1249 if (SrcRegs[I] != DestRegs[I]) {
1250 UsedMask[I] = true;
1251 EmitAndCountInstruction(
1252 Inst&: MCInstBuilder(X86::PUSH64r).addReg(Reg: DestRegs[I]));
1253 } else {
1254 emitX86Nops(OS&: *OutStreamer, NumBytes: 4, Subtarget);
1255 }
1256 }
1257
1258 // In the above loop we only stash all of the destination registers or emit
1259 // nops if the arguments are already in the right place. Doing the actually
1260 // moving is postponed until after all the registers are stashed so nothing
1261 // is clobbers. We've already added nops to account for the size of mov and
1262 // push if the register is in the right place, so we only have to worry about
1263 // emitting movs.
1264 // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1265 // earlier DestReg. We will have already overwritten over the register before
1266 // we can copy from it.
1267 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1268 if (UsedMask[I])
1269 EmitAndCountInstruction(
1270 Inst&: MCInstBuilder(X86::MOV64rr).addReg(Reg: DestRegs[I]).addReg(Reg: SrcRegs[I]));
1271
1272 // We emit a hard dependency on the __xray_TypedEvent symbol, which is the
1273 // name of the trampoline to be implemented by the XRay runtime.
1274 auto TSym = OutContext.getOrCreateSymbol(Name: "__xray_TypedEvent");
1275 MachineOperand TOp = MachineOperand::CreateMCSymbol(Sym: TSym);
1276 if (isPositionIndependent())
1277 TOp.setTargetFlags(X86II::MO_PLT);
1278
1279 // Emit the call instruction.
1280 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64pcrel32)
1281 .addOperand(Op: MCIL.LowerSymbolOperand(MO: TOp, Sym: TSym)));
1282
1283 // Restore caller-saved and used registers.
1284 for (unsigned I = sizeof UsedMask; I-- > 0;)
1285 if (UsedMask[I])
1286 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::POP64r).addReg(Reg: DestRegs[I]));
1287 else
1288 emitX86Nops(OS&: *OutStreamer, NumBytes: 1, Subtarget);
1289
1290 OutStreamer->AddComment(T: "xray typed event end.");
1291
1292 // Record the sled version.
1293 recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2);
1294}
1295
1296void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
1297 X86MCInstLower &MCIL) {
1298
1299 NoAutoPaddingScope NoPadScope(*OutStreamer);
1300
1301 const Function &F = MF->getFunction();
1302 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
1303 unsigned Num = F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry");
1304 emitX86Nops(OS&: *OutStreamer, NumBytes: Num, Subtarget);
1305 return;
1306 }
1307 // We want to emit the following pattern:
1308 //
1309 // .p2align 1, ...
1310 // .Lxray_sled_N:
1311 // jmp .tmpN
1312 // # 9 bytes worth of noops
1313 //
1314 // We need the 9 bytes because at runtime, we'd be patching over the full 11
1315 // bytes with the following pattern:
1316 //
1317 // mov %r10, <function id, 32-bit> // 6 bytes
1318 // call <relative offset, 32-bits> // 5 bytes
1319 //
1320 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
1321 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1322 OutStreamer->emitLabel(Symbol: CurSled);
1323
1324 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1325 // an operand (computed as an offset from the jmp instruction).
1326 // FIXME: Find another less hacky way do force the relative jump.
1327 OutStreamer->emitBytes(Data: "\xeb\x09");
1328 emitX86Nops(OS&: *OutStreamer, NumBytes: 9, Subtarget);
1329 recordSled(Sled: CurSled, MI, Kind: SledKind::FUNCTION_ENTER, Version: 2);
1330}
1331
1332void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
1333 X86MCInstLower &MCIL) {
1334 NoAutoPaddingScope NoPadScope(*OutStreamer);
1335
1336 // Since PATCHABLE_RET takes the opcode of the return statement as an
1337 // argument, we use that to emit the correct form of the RET that we want.
1338 // i.e. when we see this:
1339 //
1340 // PATCHABLE_RET X86::RET ...
1341 //
1342 // We should emit the RET followed by sleds.
1343 //
1344 // .p2align 1, ...
1345 // .Lxray_sled_N:
1346 // ret # or equivalent instruction
1347 // # 10 bytes worth of noops
1348 //
1349 // This just makes sure that the alignment for the next instruction is 2.
1350 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
1351 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1352 OutStreamer->emitLabel(Symbol: CurSled);
1353 unsigned OpCode = MI.getOperand(i: 0).getImm();
1354 MCInst Ret;
1355 Ret.setOpcode(OpCode);
1356 for (auto &MO : drop_begin(RangeOrContainer: MI.operands()))
1357 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO); Op.isValid())
1358 Ret.addOperand(Op);
1359 OutStreamer->emitInstruction(Inst: Ret, STI: getSubtargetInfo());
1360 emitX86Nops(OS&: *OutStreamer, NumBytes: 10, Subtarget);
1361 recordSled(Sled: CurSled, MI, Kind: SledKind::FUNCTION_EXIT, Version: 2);
1362}
1363
1364void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI,
1365 X86MCInstLower &MCIL) {
1366 MCInst TC;
1367 TC.setOpcode(convertTailJumpOpcode(Opcode: MI.getOperand(i: 0).getImm()));
1368 // Drop the tail jump opcode.
1369 auto TCOperands = drop_begin(RangeOrContainer: MI.operands());
1370 bool IsConditional = TC.getOpcode() == X86::JCC_1;
1371 MCSymbol *FallthroughLabel;
1372 if (IsConditional) {
1373 // Rewrite:
1374 // je target
1375 //
1376 // To:
1377 // jne .fallthrough
1378 // .p2align 1, ...
1379 // .Lxray_sled_N:
1380 // SLED_CODE
1381 // jmp target
1382 // .fallthrough:
1383 FallthroughLabel = OutContext.createTempSymbol();
1384 EmitToStreamer(
1385 S&: *OutStreamer,
1386 Inst: MCInstBuilder(X86::JCC_1)
1387 .addExpr(Val: MCSymbolRefExpr::create(Symbol: FallthroughLabel, Ctx&: OutContext))
1388 .addImm(Val: X86::GetOppositeBranchCondition(
1389 CC: static_cast<X86::CondCode>(MI.getOperand(i: 2).getImm()))));
1390 TC.setOpcode(X86::JMP_1);
1391 // Drop the condition code.
1392 TCOperands = drop_end(RangeOrContainer&: TCOperands);
1393 }
1394
1395 NoAutoPaddingScope NoPadScope(*OutStreamer);
1396
1397 // Like PATCHABLE_RET, we have the actual instruction in the operands to this
1398 // instruction so we lower that particular instruction and its operands.
1399 // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how
1400 // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to
1401 // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual
1402 // tail call much like how we have it in PATCHABLE_RET.
1403 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
1404 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1405 OutStreamer->emitLabel(Symbol: CurSled);
1406 auto Target = OutContext.createTempSymbol();
1407
1408 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1409 // an operand (computed as an offset from the jmp instruction).
1410 // FIXME: Find another less hacky way do force the relative jump.
1411 OutStreamer->emitBytes(Data: "\xeb\x09");
1412 emitX86Nops(OS&: *OutStreamer, NumBytes: 9, Subtarget);
1413 OutStreamer->emitLabel(Symbol: Target);
1414 recordSled(Sled: CurSled, MI, Kind: SledKind::TAIL_CALL, Version: 2);
1415
1416 // Before emitting the instruction, add a comment to indicate that this is
1417 // indeed a tail call.
1418 OutStreamer->AddComment(T: "TAILCALL");
1419 for (auto &MO : TCOperands)
1420 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO); Op.isValid())
1421 TC.addOperand(Op);
1422 OutStreamer->emitInstruction(Inst: TC, STI: getSubtargetInfo());
1423
1424 if (IsConditional)
1425 OutStreamer->emitLabel(Symbol: FallthroughLabel);
1426}
1427
1428static unsigned getSrcIdx(const MachineInstr* MI, unsigned SrcIdx) {
1429 if (X86II::isKMasked(TSFlags: MI->getDesc().TSFlags)) {
1430 // Skip mask operand.
1431 ++SrcIdx;
1432 if (X86II::isKMergeMasked(TSFlags: MI->getDesc().TSFlags)) {
1433 // Skip passthru operand.
1434 ++SrcIdx;
1435 }
1436 }
1437 return SrcIdx;
1438}
1439
1440static void printDstRegisterName(raw_ostream &CS, const MachineInstr *MI,
1441 unsigned SrcOpIdx) {
1442 const MachineOperand &DstOp = MI->getOperand(i: 0);
1443 CS << X86ATTInstPrinter::getRegisterName(Reg: DstOp.getReg());
1444
1445 // Handle AVX512 MASK/MASXZ write mask comments.
1446 // MASK: zmmX {%kY}
1447 // MASKZ: zmmX {%kY} {z}
1448 if (X86II::isKMasked(TSFlags: MI->getDesc().TSFlags)) {
1449 const MachineOperand &WriteMaskOp = MI->getOperand(i: SrcOpIdx - 1);
1450 StringRef Mask = X86ATTInstPrinter::getRegisterName(Reg: WriteMaskOp.getReg());
1451 CS << " {%" << Mask << "}";
1452 if (!X86II::isKMergeMasked(TSFlags: MI->getDesc().TSFlags)) {
1453 CS << " {z}";
1454 }
1455 }
1456}
1457
1458static void printShuffleMask(raw_ostream &CS, StringRef Src1Name,
1459 StringRef Src2Name, ArrayRef<int> Mask) {
1460 // One source operand, fix the mask to print all elements in one span.
1461 SmallVector<int, 8> ShuffleMask(Mask);
1462 if (Src1Name == Src2Name)
1463 for (int i = 0, e = ShuffleMask.size(); i != e; ++i)
1464 if (ShuffleMask[i] >= e)
1465 ShuffleMask[i] -= e;
1466
1467 for (int i = 0, e = ShuffleMask.size(); i != e; ++i) {
1468 if (i != 0)
1469 CS << ",";
1470 if (ShuffleMask[i] == SM_SentinelZero) {
1471 CS << "zero";
1472 continue;
1473 }
1474
1475 // Otherwise, it must come from src1 or src2. Print the span of elements
1476 // that comes from this src.
1477 bool isSrc1 = ShuffleMask[i] < (int)e;
1478 CS << (isSrc1 ? Src1Name : Src2Name) << '[';
1479
1480 bool IsFirst = true;
1481 while (i != e && ShuffleMask[i] != SM_SentinelZero &&
1482 (ShuffleMask[i] < (int)e) == isSrc1) {
1483 if (!IsFirst)
1484 CS << ',';
1485 else
1486 IsFirst = false;
1487 if (ShuffleMask[i] == SM_SentinelUndef)
1488 CS << "u";
1489 else
1490 CS << ShuffleMask[i] % (int)e;
1491 ++i;
1492 }
1493 CS << ']';
1494 --i; // For loop increments element #.
1495 }
1496}
1497
1498static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx,
1499 unsigned SrcOp2Idx, ArrayRef<int> Mask) {
1500 std::string Comment;
1501
1502 const MachineOperand &SrcOp1 = MI->getOperand(i: SrcOp1Idx);
1503 const MachineOperand &SrcOp2 = MI->getOperand(i: SrcOp2Idx);
1504 StringRef Src1Name = SrcOp1.isReg()
1505 ? X86ATTInstPrinter::getRegisterName(Reg: SrcOp1.getReg())
1506 : "mem";
1507 StringRef Src2Name = SrcOp2.isReg()
1508 ? X86ATTInstPrinter::getRegisterName(Reg: SrcOp2.getReg())
1509 : "mem";
1510
1511 raw_string_ostream CS(Comment);
1512 printDstRegisterName(CS, MI, SrcOpIdx: SrcOp1Idx);
1513 CS << " = ";
1514 printShuffleMask(CS, Src1Name, Src2Name, Mask);
1515
1516 return Comment;
1517}
1518
1519static void printConstant(const APInt &Val, raw_ostream &CS,
1520 bool PrintZero = false) {
1521 if (Val.getBitWidth() <= 64) {
1522 CS << (PrintZero ? 0ULL : Val.getZExtValue());
1523 } else {
1524 // print multi-word constant as (w0,w1)
1525 CS << "(";
1526 for (int i = 0, N = Val.getNumWords(); i < N; ++i) {
1527 if (i > 0)
1528 CS << ",";
1529 CS << (PrintZero ? 0ULL : Val.getRawData()[i]);
1530 }
1531 CS << ")";
1532 }
1533}
1534
1535static void printConstant(const APFloat &Flt, raw_ostream &CS,
1536 bool PrintZero = false) {
1537 SmallString<32> Str;
1538 // Force scientific notation to distinguish from integers.
1539 if (PrintZero)
1540 APFloat::getZero(Sem: Flt.getSemantics()).toString(Str, FormatPrecision: 0, FormatMaxPadding: 0);
1541 else
1542 Flt.toString(Str, FormatPrecision: 0, FormatMaxPadding: 0);
1543 CS << Str;
1544}
1545
1546static void printConstant(const Constant *COp, unsigned BitWidth,
1547 raw_ostream &CS, bool PrintZero = false) {
1548 if (isa<UndefValue>(Val: COp)) {
1549 CS << "u";
1550 } else if (auto *CI = dyn_cast<ConstantInt>(Val: COp)) {
1551 if (auto VTy = dyn_cast<FixedVectorType>(Val: CI->getType())) {
1552 for (unsigned I = 0, E = VTy->getNumElements(); I != E; ++I) {
1553 if (I != 0)
1554 CS << ',';
1555 printConstant(Val: CI->getValue(), CS, PrintZero);
1556 }
1557 } else
1558 printConstant(Val: CI->getValue(), CS, PrintZero);
1559 } else if (auto *CF = dyn_cast<ConstantFP>(Val: COp)) {
1560 if (auto VTy = dyn_cast<FixedVectorType>(Val: CF->getType())) {
1561 unsigned EltBits = VTy->getScalarSizeInBits();
1562 unsigned E = std::min(a: BitWidth / EltBits, b: VTy->getNumElements());
1563 if ((BitWidth % EltBits) == 0) {
1564 for (unsigned I = 0; I != E; ++I) {
1565 if (I != 0)
1566 CS << ",";
1567 printConstant(Flt: CF->getValueAPF(), CS, PrintZero);
1568 }
1569 } else {
1570 CS << "?";
1571 }
1572 } else
1573 printConstant(Flt: CF->getValueAPF(), CS, PrintZero);
1574 } else if (auto *CDS = dyn_cast<ConstantDataSequential>(Val: COp)) {
1575 Type *EltTy = CDS->getElementType();
1576 bool IsInteger = EltTy->isIntegerTy();
1577 bool IsFP = EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy();
1578 unsigned EltBits = EltTy->getPrimitiveSizeInBits();
1579 unsigned E = std::min(a: BitWidth / EltBits, b: (unsigned)CDS->getNumElements());
1580 if ((BitWidth % EltBits) == 0) {
1581 for (unsigned I = 0; I != E; ++I) {
1582 if (I != 0)
1583 CS << ",";
1584 if (IsInteger)
1585 printConstant(Val: CDS->getElementAsAPInt(i: I), CS, PrintZero);
1586 else if (IsFP)
1587 printConstant(Flt: CDS->getElementAsAPFloat(i: I), CS, PrintZero);
1588 else
1589 CS << "?";
1590 }
1591 } else {
1592 CS << "?";
1593 }
1594 } else if (auto *CV = dyn_cast<ConstantVector>(Val: COp)) {
1595 unsigned EltBits = CV->getType()->getScalarSizeInBits();
1596 unsigned E = std::min(a: BitWidth / EltBits, b: CV->getNumOperands());
1597 if ((BitWidth % EltBits) == 0) {
1598 for (unsigned I = 0; I != E; ++I) {
1599 if (I != 0)
1600 CS << ",";
1601 printConstant(COp: CV->getOperand(i_nocapture: I), BitWidth: EltBits, CS, PrintZero);
1602 }
1603 } else {
1604 CS << "?";
1605 }
1606 } else {
1607 CS << "?";
1608 }
1609}
1610
1611static void printZeroUpperMove(const MachineInstr *MI, MCStreamer &OutStreamer,
1612 int SclWidth, int VecWidth,
1613 const char *ShuffleComment) {
1614 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1615
1616 std::string Comment;
1617 raw_string_ostream CS(Comment);
1618 printDstRegisterName(CS, MI, SrcOpIdx: SrcIdx);
1619 CS << " = ";
1620
1621 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx)) {
1622 CS << "[";
1623 printConstant(COp: C, BitWidth: SclWidth, CS);
1624 for (int I = 1, E = VecWidth / SclWidth; I < E; ++I) {
1625 CS << ",";
1626 printConstant(COp: C, BitWidth: SclWidth, CS, PrintZero: true);
1627 }
1628 CS << "]";
1629 OutStreamer.AddComment(T: CS.str());
1630 return; // early-out
1631 }
1632
1633 // We didn't find a constant load, fallback to a shuffle mask decode.
1634 CS << ShuffleComment;
1635 OutStreamer.AddComment(T: CS.str());
1636}
1637
1638static void printBroadcast(const MachineInstr *MI, MCStreamer &OutStreamer,
1639 int Repeats, int BitWidth) {
1640 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1641 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx)) {
1642 std::string Comment;
1643 raw_string_ostream CS(Comment);
1644 printDstRegisterName(CS, MI, SrcOpIdx: SrcIdx);
1645 CS << " = [";
1646 for (int l = 0; l != Repeats; ++l) {
1647 if (l != 0)
1648 CS << ",";
1649 printConstant(COp: C, BitWidth, CS);
1650 }
1651 CS << "]";
1652 OutStreamer.AddComment(T: CS.str());
1653 }
1654}
1655
1656static void addConstantComment(const MachineInstr *MI, MCStreamer &OutStreamer,
1657 unsigned OpNo, int BitWidth, int Repeats = 1) {
1658 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo)) {
1659 std::string Comment;
1660 raw_string_ostream CS(Comment);
1661 CS << "[";
1662 for (int I = 0; I != Repeats; ++I) {
1663 if (I != 0)
1664 CS << ",";
1665 printConstant(COp: C, BitWidth, CS);
1666 }
1667 CS << "]";
1668 OutStreamer.AddComment(T: CS.str());
1669 }
1670}
1671
1672static bool printExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1673 int SrcEltBits, int DstEltBits, bool IsSext) {
1674 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1675 auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx);
1676 if (C && C->getType()->getScalarSizeInBits() == unsigned(SrcEltBits)) {
1677 if (auto *CDS = dyn_cast<ConstantDataSequential>(Val: C)) {
1678 int NumElts = CDS->getNumElements();
1679 std::string Comment;
1680 raw_string_ostream CS(Comment);
1681 printDstRegisterName(CS, MI, SrcOpIdx: SrcIdx);
1682 CS << " = [";
1683 for (int i = 0; i != NumElts; ++i) {
1684 if (i != 0)
1685 CS << ",";
1686 if (CDS->getElementType()->isIntegerTy()) {
1687 APInt Elt = CDS->getElementAsAPInt(i);
1688 Elt = IsSext ? Elt.sext(width: DstEltBits) : Elt.zext(width: DstEltBits);
1689 printConstant(Val: Elt, CS);
1690 } else
1691 CS << "?";
1692 }
1693 CS << "]";
1694 OutStreamer.AddComment(T: CS.str());
1695 return true;
1696 }
1697 }
1698
1699 return false;
1700}
1701static void printSignExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1702 int SrcEltBits, int DstEltBits) {
1703 printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, IsSext: true);
1704}
1705static void printZeroExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1706 int SrcEltBits, int DstEltBits) {
1707 if (printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, IsSext: false))
1708 return;
1709
1710 // We didn't find a constant load, fallback to a shuffle mask decode.
1711 std::string Comment;
1712 raw_string_ostream CS(Comment);
1713 printDstRegisterName(CS, MI, SrcOpIdx: getSrcIdx(MI, SrcIdx: 1));
1714 CS << " = ";
1715
1716 SmallVector<int> Mask;
1717 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1718 assert((Width % DstEltBits) == 0 && (DstEltBits % SrcEltBits) == 0 &&
1719 "Illegal extension ratio");
1720 DecodeZeroExtendMask(SrcScalarBits: SrcEltBits, DstScalarBits: DstEltBits, NumDstElts: Width / DstEltBits, IsAnyExtend: false, ShuffleMask&: Mask);
1721 printShuffleMask(CS, Src1Name: "mem", Src2Name: "", Mask);
1722
1723 OutStreamer.AddComment(T: CS.str());
1724}
1725
1726void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) {
1727 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
1728 assert(getSubtarget().isOSWindowsOrUEFI() &&
1729 "SEH_ instruction Windows and UEFI only");
1730
1731 // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86.
1732 if (EmitFPOData) {
1733 X86TargetStreamer *XTS =
1734 static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer());
1735 switch (MI->getOpcode()) {
1736 case X86::SEH_PushReg:
1737 XTS->emitFPOPushReg(Reg: MI->getOperand(i: 0).getImm());
1738 break;
1739 case X86::SEH_StackAlloc:
1740 XTS->emitFPOStackAlloc(StackAlloc: MI->getOperand(i: 0).getImm());
1741 break;
1742 case X86::SEH_StackAlign:
1743 XTS->emitFPOStackAlign(Align: MI->getOperand(i: 0).getImm());
1744 break;
1745 case X86::SEH_SetFrame:
1746 assert(MI->getOperand(1).getImm() == 0 &&
1747 ".cv_fpo_setframe takes no offset");
1748 XTS->emitFPOSetFrame(Reg: MI->getOperand(i: 0).getImm());
1749 break;
1750 case X86::SEH_EndPrologue:
1751 XTS->emitFPOEndPrologue();
1752 break;
1753 case X86::SEH_SaveReg:
1754 case X86::SEH_SaveXMM:
1755 case X86::SEH_PushFrame:
1756 case X86::SEH_Push2Regs:
1757 llvm_unreachable("SEH_ directive incompatible with FPO");
1758 break;
1759 default:
1760 llvm_unreachable("expected SEH_ instruction");
1761 }
1762 return;
1763 }
1764
1765 // Otherwise, use the .seh_ directives for all other Windows platforms.
1766 switch (MI->getOpcode()) {
1767 case X86::SEH_PushReg:
1768 OutStreamer->emitWinCFIPushReg(Register: MI->getOperand(i: 0).getImm());
1769 break;
1770
1771 case X86::SEH_Push2Regs:
1772 OutStreamer->emitWinCFIPush2Regs(Reg1: MI->getOperand(i: 0).getImm(),
1773 Reg2: MI->getOperand(i: 1).getImm());
1774 break;
1775
1776 case X86::SEH_SaveReg:
1777 OutStreamer->emitWinCFISaveReg(Register: MI->getOperand(i: 0).getImm(),
1778 Offset: MI->getOperand(i: 1).getImm());
1779 break;
1780
1781 case X86::SEH_SaveXMM:
1782 OutStreamer->emitWinCFISaveXMM(Register: MI->getOperand(i: 0).getImm(),
1783 Offset: MI->getOperand(i: 1).getImm());
1784 break;
1785
1786 case X86::SEH_StackAlloc:
1787 OutStreamer->emitWinCFIAllocStack(Size: MI->getOperand(i: 0).getImm());
1788 break;
1789
1790 case X86::SEH_SetFrame:
1791 OutStreamer->emitWinCFISetFrame(Register: MI->getOperand(i: 0).getImm(),
1792 Offset: MI->getOperand(i: 1).getImm());
1793 break;
1794
1795 case X86::SEH_PushFrame:
1796 OutStreamer->emitWinCFIPushFrame(Code: MI->getOperand(i: 0).getImm());
1797 break;
1798
1799 case X86::SEH_EndPrologue:
1800 OutStreamer->emitWinCFIEndProlog();
1801 break;
1802
1803 case X86::SEH_BeginEpilogue:
1804 OutStreamer->emitWinCFIBeginEpilogue();
1805 break;
1806
1807 case X86::SEH_EndEpilogue:
1808 OutStreamer->emitWinCFIEndEpilogue();
1809 break;
1810
1811 case X86::SEH_UnwindV2Start:
1812 OutStreamer->emitWinCFIUnwindV2Start();
1813 break;
1814
1815 case X86::SEH_UnwindVersion:
1816 OutStreamer->emitWinCFIUnwindVersion(Version: MI->getOperand(i: 0).getImm());
1817 break;
1818
1819 default:
1820 llvm_unreachable("expected SEH_ instruction");
1821 }
1822}
1823
1824static void addConstantComments(const MachineInstr *MI,
1825 MCStreamer &OutStreamer) {
1826 switch (MI->getOpcode()) {
1827 // Lower PSHUFB and VPERMILP normally but add a comment if we can find
1828 // a constant shuffle mask. We won't be able to do this at the MC layer
1829 // because the mask isn't an immediate.
1830 case X86::PSHUFBrm:
1831 case X86::VPSHUFBrm:
1832 case X86::VPSHUFBYrm:
1833 case X86::VPSHUFBZ128rm:
1834 case X86::VPSHUFBZ128rmk:
1835 case X86::VPSHUFBZ128rmkz:
1836 case X86::VPSHUFBZ256rm:
1837 case X86::VPSHUFBZ256rmk:
1838 case X86::VPSHUFBZ256rmkz:
1839 case X86::VPSHUFBZrm:
1840 case X86::VPSHUFBZrmk:
1841 case X86::VPSHUFBZrmkz: {
1842 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1843 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx + 1)) {
1844 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1845 SmallVector<int, 64> Mask;
1846 DecodePSHUFBMask(C, Width, ShuffleMask&: Mask);
1847 if (!Mask.empty())
1848 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: SrcIdx, SrcOp2Idx: SrcIdx, Mask));
1849 }
1850 break;
1851 }
1852
1853 case X86::VPERMILPSrm:
1854 case X86::VPERMILPSYrm:
1855 case X86::VPERMILPSZ128rm:
1856 case X86::VPERMILPSZ128rmk:
1857 case X86::VPERMILPSZ128rmkz:
1858 case X86::VPERMILPSZ256rm:
1859 case X86::VPERMILPSZ256rmk:
1860 case X86::VPERMILPSZ256rmkz:
1861 case X86::VPERMILPSZrm:
1862 case X86::VPERMILPSZrmk:
1863 case X86::VPERMILPSZrmkz: {
1864 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1865 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx + 1)) {
1866 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1867 SmallVector<int, 16> Mask;
1868 DecodeVPERMILPMask(C, ElSize: 32, Width, ShuffleMask&: Mask);
1869 if (!Mask.empty())
1870 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: SrcIdx, SrcOp2Idx: SrcIdx, Mask));
1871 }
1872 break;
1873 }
1874 case X86::VPERMILPDrm:
1875 case X86::VPERMILPDYrm:
1876 case X86::VPERMILPDZ128rm:
1877 case X86::VPERMILPDZ128rmk:
1878 case X86::VPERMILPDZ128rmkz:
1879 case X86::VPERMILPDZ256rm:
1880 case X86::VPERMILPDZ256rmk:
1881 case X86::VPERMILPDZ256rmkz:
1882 case X86::VPERMILPDZrm:
1883 case X86::VPERMILPDZrmk:
1884 case X86::VPERMILPDZrmkz: {
1885 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1886 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx + 1)) {
1887 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1888 SmallVector<int, 16> Mask;
1889 DecodeVPERMILPMask(C, ElSize: 64, Width, ShuffleMask&: Mask);
1890 if (!Mask.empty())
1891 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: SrcIdx, SrcOp2Idx: SrcIdx, Mask));
1892 }
1893 break;
1894 }
1895
1896 case X86::VPERMIL2PDrm:
1897 case X86::VPERMIL2PSrm:
1898 case X86::VPERMIL2PDYrm:
1899 case X86::VPERMIL2PSYrm: {
1900 assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands + 1) &&
1901 "Unexpected number of operands!");
1902
1903 const MachineOperand &CtrlOp = MI->getOperand(i: MI->getNumOperands() - 1);
1904 if (!CtrlOp.isImm())
1905 break;
1906
1907 unsigned ElSize;
1908 switch (MI->getOpcode()) {
1909 default: llvm_unreachable("Invalid opcode");
1910 case X86::VPERMIL2PSrm: case X86::VPERMIL2PSYrm: ElSize = 32; break;
1911 case X86::VPERMIL2PDrm: case X86::VPERMIL2PDYrm: ElSize = 64; break;
1912 }
1913
1914 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: 3)) {
1915 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1916 SmallVector<int, 16> Mask;
1917 DecodeVPERMIL2PMask(C, M2Z: (unsigned)CtrlOp.getImm(), ElSize, Width, ShuffleMask&: Mask);
1918 if (!Mask.empty())
1919 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: 1, SrcOp2Idx: 2, Mask));
1920 }
1921 break;
1922 }
1923
1924 case X86::VPPERMrrm: {
1925 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: 3)) {
1926 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1927 SmallVector<int, 16> Mask;
1928 DecodeVPPERMMask(C, Width, ShuffleMask&: Mask);
1929 if (!Mask.empty())
1930 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: 1, SrcOp2Idx: 2, Mask));
1931 }
1932 break;
1933 }
1934
1935 case X86::MMX_MOVQ64rm: {
1936 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: 1)) {
1937 std::string Comment;
1938 raw_string_ostream CS(Comment);
1939 const MachineOperand &DstOp = MI->getOperand(i: 0);
1940 CS << X86ATTInstPrinter::getRegisterName(Reg: DstOp.getReg()) << " = ";
1941 if (auto *CF = dyn_cast<ConstantFP>(Val: C)) {
1942 CS << "0x" << toString(I: CF->getValueAPF().bitcastToAPInt(), Radix: 16, Signed: false);
1943 OutStreamer.AddComment(T: CS.str());
1944 }
1945 }
1946 break;
1947 }
1948
1949 case X86::GF2P8AFFINEQBrmi:
1950 case X86::VGF2P8AFFINEQBrmi:
1951 case X86::VGF2P8AFFINEQBYrmi:
1952 case X86::VGF2P8AFFINEQBZrmi:
1953 case X86::VGF2P8AFFINEQBZ128rmi:
1954 case X86::VGF2P8AFFINEQBZ256rmi: {
1955 // TODO: Add predicate handling with test coverage.
1956 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1957 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1958 addConstantComment(MI, OutStreamer, OpNo: SrcIdx + 1, BitWidth: Width);
1959 break;
1960 }
1961
1962 case X86::VGF2P8AFFINEQBZ128rmbi:
1963 case X86::VGF2P8AFFINEQBZ256rmbi:
1964 case X86::VGF2P8AFFINEQBZrmbi: {
1965 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1966 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1967 addConstantComment(MI, OutStreamer, OpNo: SrcIdx + 1, BitWidth: 64, Repeats: Width / 64);
1968 break;
1969 }
1970
1971#define INSTR_CASE(Prefix, Instr, Suffix, Postfix) \
1972 case X86::Prefix##Instr##Suffix##rm##Postfix:
1973
1974#define CASE_AVX512_ARITH_RM(Instr) \
1975 INSTR_CASE(V, Instr, Z128, ) \
1976 INSTR_CASE(V, Instr, Z128, k) \
1977 INSTR_CASE(V, Instr, Z128, kz) \
1978 INSTR_CASE(V, Instr, Z256, ) \
1979 INSTR_CASE(V, Instr, Z256, k) \
1980 INSTR_CASE(V, Instr, Z256, kz) \
1981 INSTR_CASE(V, Instr, Z, ) \
1982 INSTR_CASE(V, Instr, Z, k) \
1983 INSTR_CASE(V, Instr, Z, kz)
1984
1985#define CASE_ARITH_RM(Instr) \
1986 INSTR_CASE(, Instr, , ) /* SSE */ \
1987 INSTR_CASE(V, Instr, , ) /* AVX-128 */ \
1988 INSTR_CASE(V, Instr, Y, ) /* AVX-256 */ \
1989 INSTR_CASE(V, Instr, Z128, ) \
1990 INSTR_CASE(V, Instr, Z128, k) \
1991 INSTR_CASE(V, Instr, Z128, kz) \
1992 INSTR_CASE(V, Instr, Z256, ) \
1993 INSTR_CASE(V, Instr, Z256, k) \
1994 INSTR_CASE(V, Instr, Z256, kz) \
1995 INSTR_CASE(V, Instr, Z, ) \
1996 INSTR_CASE(V, Instr, Z, k) \
1997 INSTR_CASE(V, Instr, Z, kz)
1998
1999 // TODO: Add additional instructions when useful.
2000 CASE_ARITH_RM(PADDB)
2001 CASE_ARITH_RM(PADDW)
2002 CASE_ARITH_RM(PADDD)
2003 CASE_ARITH_RM(PADDQ)
2004 CASE_ARITH_RM(PMADDUBSW)
2005 CASE_ARITH_RM(PMADDWD)
2006 CASE_ARITH_RM(PMULDQ)
2007 CASE_ARITH_RM(PMULUDQ)
2008 CASE_ARITH_RM(PMULLD)
2009 CASE_AVX512_ARITH_RM(PMULLQ)
2010 CASE_ARITH_RM(PMULLW)
2011 CASE_ARITH_RM(PMULHW)
2012 CASE_ARITH_RM(PMULHUW)
2013 CASE_ARITH_RM(PMULHRSW) {
2014 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
2015 unsigned VectorWidth =
2016 X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
2017 addConstantComment(MI, OutStreamer, OpNo: SrcIdx + 1, BitWidth: VectorWidth);
2018 break;
2019 }
2020
2021#define MASK_AVX512_CASE(Instr) \
2022 case Instr: \
2023 case Instr##k: \
2024 case Instr##kz:
2025
2026 case X86::MOVSDrm:
2027 case X86::VMOVSDrm:
2028 MASK_AVX512_CASE(X86::VMOVSDZrm)
2029 case X86::MOVSDrm_alt:
2030 case X86::VMOVSDrm_alt:
2031 case X86::VMOVSDZrm_alt:
2032 case X86::MOVQI2PQIrm:
2033 case X86::VMOVQI2PQIrm:
2034 case X86::VMOVQI2PQIZrm:
2035 printZeroUpperMove(MI, OutStreamer, SclWidth: 64, VecWidth: 128, ShuffleComment: "mem[0],zero");
2036 break;
2037
2038 MASK_AVX512_CASE(X86::VMOVSHZrm)
2039 case X86::VMOVSHZrm_alt:
2040 printZeroUpperMove(MI, OutStreamer, SclWidth: 16, VecWidth: 128,
2041 ShuffleComment: "mem[0],zero,zero,zero,zero,zero,zero,zero");
2042 break;
2043
2044 case X86::MOVSSrm:
2045 case X86::VMOVSSrm:
2046 MASK_AVX512_CASE(X86::VMOVSSZrm)
2047 case X86::MOVSSrm_alt:
2048 case X86::VMOVSSrm_alt:
2049 case X86::VMOVSSZrm_alt:
2050 case X86::MOVDI2PDIrm:
2051 case X86::VMOVDI2PDIrm:
2052 case X86::VMOVDI2PDIZrm:
2053 printZeroUpperMove(MI, OutStreamer, SclWidth: 32, VecWidth: 128, ShuffleComment: "mem[0],zero,zero,zero");
2054 break;
2055
2056#define MOV_CASE(Prefix, Suffix) \
2057 case X86::Prefix##MOVAPD##Suffix##rm: \
2058 case X86::Prefix##MOVAPS##Suffix##rm: \
2059 case X86::Prefix##MOVUPD##Suffix##rm: \
2060 case X86::Prefix##MOVUPS##Suffix##rm: \
2061 case X86::Prefix##MOVDQA##Suffix##rm: \
2062 case X86::Prefix##MOVDQU##Suffix##rm:
2063
2064#define MOV_AVX512_CASE(Suffix, Postfix) \
2065 case X86::VMOVDQA64##Suffix##rm##Postfix: \
2066 case X86::VMOVDQA32##Suffix##rm##Postfix: \
2067 case X86::VMOVDQU64##Suffix##rm##Postfix: \
2068 case X86::VMOVDQU32##Suffix##rm##Postfix: \
2069 case X86::VMOVDQU16##Suffix##rm##Postfix: \
2070 case X86::VMOVDQU8##Suffix##rm##Postfix: \
2071 case X86::VMOVAPS##Suffix##rm##Postfix: \
2072 case X86::VMOVAPD##Suffix##rm##Postfix: \
2073 case X86::VMOVUPS##Suffix##rm##Postfix: \
2074 case X86::VMOVUPD##Suffix##rm##Postfix:
2075
2076#define CASE_128_MOV_RM() \
2077 MOV_CASE(, ) /* SSE */ \
2078 MOV_CASE(V, ) /* AVX-128 */ \
2079 MOV_AVX512_CASE(Z128, ) \
2080 MOV_AVX512_CASE(Z128, k) \
2081 MOV_AVX512_CASE(Z128, kz)
2082
2083#define CASE_256_MOV_RM() \
2084 MOV_CASE(V, Y) /* AVX-256 */ \
2085 MOV_AVX512_CASE(Z256, ) \
2086 MOV_AVX512_CASE(Z256, k) \
2087 MOV_AVX512_CASE(Z256, kz) \
2088
2089#define CASE_512_MOV_RM() \
2090 MOV_AVX512_CASE(Z, ) \
2091 MOV_AVX512_CASE(Z, k) \
2092 MOV_AVX512_CASE(Z, kz) \
2093
2094 // For loads from a constant pool to a vector register, print the constant
2095 // loaded.
2096 CASE_128_MOV_RM()
2097 printBroadcast(MI, OutStreamer, Repeats: 1, BitWidth: 128);
2098 break;
2099 CASE_256_MOV_RM()
2100 printBroadcast(MI, OutStreamer, Repeats: 1, BitWidth: 256);
2101 break;
2102 CASE_512_MOV_RM()
2103 printBroadcast(MI, OutStreamer, Repeats: 1, BitWidth: 512);
2104 break;
2105 case X86::VBROADCASTF128rm:
2106 case X86::VBROADCASTI128rm:
2107 MASK_AVX512_CASE(X86::VBROADCASTF32X4Z256rm)
2108 MASK_AVX512_CASE(X86::VBROADCASTF64X2Z256rm)
2109 MASK_AVX512_CASE(X86::VBROADCASTI32X4Z256rm)
2110 MASK_AVX512_CASE(X86::VBROADCASTI64X2Z256rm)
2111 printBroadcast(MI, OutStreamer, Repeats: 2, BitWidth: 128);
2112 break;
2113 MASK_AVX512_CASE(X86::VBROADCASTF32X4Zrm)
2114 MASK_AVX512_CASE(X86::VBROADCASTF64X2Zrm)
2115 MASK_AVX512_CASE(X86::VBROADCASTI32X4Zrm)
2116 MASK_AVX512_CASE(X86::VBROADCASTI64X2Zrm)
2117 printBroadcast(MI, OutStreamer, Repeats: 4, BitWidth: 128);
2118 break;
2119 MASK_AVX512_CASE(X86::VBROADCASTF32X8Zrm)
2120 MASK_AVX512_CASE(X86::VBROADCASTF64X4Zrm)
2121 MASK_AVX512_CASE(X86::VBROADCASTI32X8Zrm)
2122 MASK_AVX512_CASE(X86::VBROADCASTI64X4Zrm)
2123 printBroadcast(MI, OutStreamer, Repeats: 2, BitWidth: 256);
2124 break;
2125
2126 // For broadcast loads from a constant pool to a vector register, repeatedly
2127 // print the constant loaded.
2128 case X86::MOVDDUPrm:
2129 case X86::VMOVDDUPrm:
2130 MASK_AVX512_CASE(X86::VMOVDDUPZ128rm)
2131 case X86::VPBROADCASTQrm:
2132 MASK_AVX512_CASE(X86::VPBROADCASTQZ128rm)
2133 printBroadcast(MI, OutStreamer, Repeats: 2, BitWidth: 64);
2134 break;
2135 case X86::VBROADCASTSDYrm:
2136 MASK_AVX512_CASE(X86::VBROADCASTSDZ256rm)
2137 case X86::VPBROADCASTQYrm:
2138 MASK_AVX512_CASE(X86::VPBROADCASTQZ256rm)
2139 printBroadcast(MI, OutStreamer, Repeats: 4, BitWidth: 64);
2140 break;
2141 MASK_AVX512_CASE(X86::VBROADCASTSDZrm)
2142 MASK_AVX512_CASE(X86::VPBROADCASTQZrm)
2143 printBroadcast(MI, OutStreamer, Repeats: 8, BitWidth: 64);
2144 break;
2145 case X86::VBROADCASTSSrm:
2146 MASK_AVX512_CASE(X86::VBROADCASTSSZ128rm)
2147 case X86::VPBROADCASTDrm:
2148 MASK_AVX512_CASE(X86::VPBROADCASTDZ128rm)
2149 printBroadcast(MI, OutStreamer, Repeats: 4, BitWidth: 32);
2150 break;
2151 case X86::VBROADCASTSSYrm:
2152 MASK_AVX512_CASE(X86::VBROADCASTSSZ256rm)
2153 case X86::VPBROADCASTDYrm:
2154 MASK_AVX512_CASE(X86::VPBROADCASTDZ256rm)
2155 printBroadcast(MI, OutStreamer, Repeats: 8, BitWidth: 32);
2156 break;
2157 MASK_AVX512_CASE(X86::VBROADCASTSSZrm)
2158 MASK_AVX512_CASE(X86::VPBROADCASTDZrm)
2159 printBroadcast(MI, OutStreamer, Repeats: 16, BitWidth: 32);
2160 break;
2161 case X86::VPBROADCASTWrm:
2162 MASK_AVX512_CASE(X86::VPBROADCASTWZ128rm)
2163 printBroadcast(MI, OutStreamer, Repeats: 8, BitWidth: 16);
2164 break;
2165 case X86::VPBROADCASTWYrm:
2166 MASK_AVX512_CASE(X86::VPBROADCASTWZ256rm)
2167 printBroadcast(MI, OutStreamer, Repeats: 16, BitWidth: 16);
2168 break;
2169 MASK_AVX512_CASE(X86::VPBROADCASTWZrm)
2170 printBroadcast(MI, OutStreamer, Repeats: 32, BitWidth: 16);
2171 break;
2172 case X86::VPBROADCASTBrm:
2173 MASK_AVX512_CASE(X86::VPBROADCASTBZ128rm)
2174 printBroadcast(MI, OutStreamer, Repeats: 16, BitWidth: 8);
2175 break;
2176 case X86::VPBROADCASTBYrm:
2177 MASK_AVX512_CASE(X86::VPBROADCASTBZ256rm)
2178 printBroadcast(MI, OutStreamer, Repeats: 32, BitWidth: 8);
2179 break;
2180 MASK_AVX512_CASE(X86::VPBROADCASTBZrm)
2181 printBroadcast(MI, OutStreamer, Repeats: 64, BitWidth: 8);
2182 break;
2183
2184#define MOVX_CASE(Prefix, Ext, Type, Suffix, Postfix) \
2185 case X86::Prefix##PMOV##Ext##Type##Suffix##rm##Postfix:
2186
2187#define CASE_MOVX_RM(Ext, Type) \
2188 MOVX_CASE(, Ext, Type, , ) \
2189 MOVX_CASE(V, Ext, Type, , ) \
2190 MOVX_CASE(V, Ext, Type, Y, ) \
2191 MOVX_CASE(V, Ext, Type, Z128, ) \
2192 MOVX_CASE(V, Ext, Type, Z128, k ) \
2193 MOVX_CASE(V, Ext, Type, Z128, kz ) \
2194 MOVX_CASE(V, Ext, Type, Z256, ) \
2195 MOVX_CASE(V, Ext, Type, Z256, k ) \
2196 MOVX_CASE(V, Ext, Type, Z256, kz ) \
2197 MOVX_CASE(V, Ext, Type, Z, ) \
2198 MOVX_CASE(V, Ext, Type, Z, k ) \
2199 MOVX_CASE(V, Ext, Type, Z, kz )
2200
2201 CASE_MOVX_RM(SX, BD)
2202 printSignExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 32);
2203 break;
2204 CASE_MOVX_RM(SX, BQ)
2205 printSignExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 64);
2206 break;
2207 CASE_MOVX_RM(SX, BW)
2208 printSignExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 16);
2209 break;
2210 CASE_MOVX_RM(SX, DQ)
2211 printSignExtend(MI, OutStreamer, SrcEltBits: 32, DstEltBits: 64);
2212 break;
2213 CASE_MOVX_RM(SX, WD)
2214 printSignExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 32);
2215 break;
2216 CASE_MOVX_RM(SX, WQ)
2217 printSignExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 64);
2218 break;
2219
2220 CASE_MOVX_RM(ZX, BD)
2221 printZeroExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 32);
2222 break;
2223 CASE_MOVX_RM(ZX, BQ)
2224 printZeroExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 64);
2225 break;
2226 CASE_MOVX_RM(ZX, BW)
2227 printZeroExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 16);
2228 break;
2229 CASE_MOVX_RM(ZX, DQ)
2230 printZeroExtend(MI, OutStreamer, SrcEltBits: 32, DstEltBits: 64);
2231 break;
2232 CASE_MOVX_RM(ZX, WD)
2233 printZeroExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 32);
2234 break;
2235 CASE_MOVX_RM(ZX, WQ)
2236 printZeroExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 64);
2237 break;
2238 }
2239}
2240
2241// Does the given operand refer to a DLLIMPORT function?
2242bool isImportedFunction(const MachineOperand &MO) {
2243 return MO.isGlobal() && (MO.getTargetFlags() == X86II::MO_DLLIMPORT);
2244}
2245
2246// Is the given instruction a call to a CFGuard function?
2247bool isCallToCFGuardFunction(const MachineInstr *MI) {
2248 assert(MI->getOpcode() == X86::TAILJMPm64_REX ||
2249 MI->getOpcode() == X86::CALL64m);
2250 const MachineOperand &MO = MI->getOperand(i: 3);
2251 return MO.isGlobal() && (MO.getTargetFlags() == X86II::MO_NO_FLAG) &&
2252 isCFGuardFunction(GV: MO.getGlobal());
2253}
2254
2255// Does the containing block for the given instruction contain any jump table
2256// info (indicating that the block is a dispatch for a jump table)?
2257bool hasJumpTableInfoInBlock(const llvm::MachineInstr *MI) {
2258 const MachineBasicBlock &MBB = *MI->getParent();
2259 for (auto I = MBB.instr_rbegin(), E = MBB.instr_rend(); I != E; ++I)
2260 if (I->isJumpTableDebugInfo())
2261 return true;
2262
2263 return false;
2264}
2265
2266void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
2267 // FIXME: Enable feature predicate checks once all the test pass.
2268 // X86_MC::verifyInstructionPredicates(MI->getOpcode(),
2269 // Subtarget->getFeatureBits());
2270
2271 X86MCInstLower MCInstLowering(*MF, *this);
2272 const X86RegisterInfo *RI =
2273 MF->getSubtarget<X86Subtarget>().getRegisterInfo();
2274
2275 if (MI->getOpcode() == X86::OR64rm) {
2276 for (auto &Opd : MI->operands()) {
2277 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
2278 "swift_async_extendedFramePointerFlags") {
2279 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
2280 }
2281 }
2282 }
2283
2284 // Add comments for values loaded from constant pool.
2285 if (OutStreamer->isVerboseAsm())
2286 addConstantComments(MI, OutStreamer&: *OutStreamer);
2287
2288 // Add a comment about EVEX compression
2289 if (TM.Options.MCOptions.ShowMCEncoding) {
2290 if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_LEGACY)
2291 OutStreamer->AddComment(T: "EVEX TO LEGACY Compression ", EOL: false);
2292 else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX)
2293 OutStreamer->AddComment(T: "EVEX TO VEX Compression ", EOL: false);
2294 else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_EVEX)
2295 OutStreamer->AddComment(T: "EVEX TO EVEX Compression ", EOL: false);
2296 }
2297
2298 // We use this to suppress NOP padding for Windows EH.
2299 bool IsTailJump = false;
2300
2301 switch (MI->getOpcode()) {
2302 case TargetOpcode::DBG_VALUE:
2303 llvm_unreachable("Should be handled target independently");
2304
2305 case X86::EH_RETURN:
2306 case X86::EH_RETURN64: {
2307 // Lower these as normal, but add some comments.
2308 Register Reg = MI->getOperand(i: 0).getReg();
2309 OutStreamer->AddComment(T: StringRef("eh_return, addr: %") +
2310 X86ATTInstPrinter::getRegisterName(Reg));
2311 break;
2312 }
2313 case X86::CLEANUPRET: {
2314 // Lower these as normal, but add some comments.
2315 OutStreamer->AddComment(T: "CLEANUPRET");
2316 break;
2317 }
2318
2319 case X86::CATCHRET: {
2320 // Lower these as normal, but add some comments.
2321 OutStreamer->AddComment(T: "CATCHRET");
2322 break;
2323 }
2324
2325 case X86::ENDBR32:
2326 case X86::ENDBR64: {
2327 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
2328 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2329 // non-empty. If MI is the initial ENDBR, place the
2330 // __patchable_function_entries label after ENDBR.
2331 if (CurrentPatchableFunctionEntrySym &&
2332 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
2333 MI == &MF->front().front()) {
2334 MCInst Inst;
2335 MCInstLowering.Lower(MI, OutMI&: Inst);
2336 EmitAndCountInstruction(Inst);
2337 CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch");
2338 OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym);
2339 return;
2340 }
2341 break;
2342 }
2343
2344 case X86::TAILJMPd64:
2345 if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(Reg: X86::R11))
2346 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CS_PREFIX));
2347
2348 if (EnableImportCallOptimization && isImportedFunction(MO: MI->getOperand(i: 0))) {
2349 emitLabelAndRecordForImportCallOptimization(
2350 Kind: IMAGE_RETPOLINE_AMD64_IMPORT_BR);
2351 }
2352
2353 // Lower this as normal, but add a comment.
2354 OutStreamer->AddComment(T: "TAILCALL");
2355 IsTailJump = true;
2356 break;
2357
2358 case X86::TAILJMPr:
2359 case X86::TAILJMPm:
2360 case X86::TAILJMPd:
2361 case X86::TAILJMPd_CC:
2362 case X86::TAILJMPr64:
2363 case X86::TAILJMPm64:
2364 case X86::TAILJMPd64_CC:
2365 if (EnableImportCallOptimization)
2366 report_fatal_error(reason: "Unexpected TAILJMP instruction was emitted when "
2367 "import call optimization was enabled");
2368
2369 // Lower these as normal, but add some comments.
2370 OutStreamer->AddComment(T: "TAILCALL");
2371 IsTailJump = true;
2372 break;
2373
2374 case X86::TAILJMPm64_REX:
2375 if (EnableImportCallOptimization && isCallToCFGuardFunction(MI)) {
2376 emitLabelAndRecordForImportCallOptimization(
2377 Kind: IMAGE_RETPOLINE_AMD64_CFG_BR_REX);
2378 }
2379
2380 OutStreamer->AddComment(T: "TAILCALL");
2381 IsTailJump = true;
2382 break;
2383
2384 case X86::TAILJMPr64_REX: {
2385 if (EnableImportCallOptimization) {
2386 assert(MI->getOperand(0).getReg() == X86::RAX &&
2387 "Indirect tail calls with impcall enabled must go through RAX (as "
2388 "enforced by TCRETURNImpCallri64)");
2389 emitLabelAndRecordForImportCallOptimization(
2390 Kind: IMAGE_RETPOLINE_AMD64_INDIR_BR);
2391 }
2392
2393 OutStreamer->AddComment(T: "TAILCALL");
2394 IsTailJump = true;
2395 break;
2396 }
2397
2398 case X86::JMP64r:
2399 if (EnableImportCallOptimization && hasJumpTableInfoInBlock(MI)) {
2400 uint16_t EncodedReg =
2401 this->getSubtarget().getRegisterInfo()->getEncodingValue(
2402 Reg: MI->getOperand(i: 0).getReg().asMCReg());
2403 emitLabelAndRecordForImportCallOptimization(
2404 Kind: (ImportCallKind)(IMAGE_RETPOLINE_AMD64_SWITCHTABLE_FIRST +
2405 EncodedReg));
2406 }
2407 break;
2408
2409 case X86::JMP16r:
2410 case X86::JMP16m:
2411 case X86::JMP32r:
2412 case X86::JMP32m:
2413 case X86::JMP64m:
2414 if (EnableImportCallOptimization && hasJumpTableInfoInBlock(MI))
2415 report_fatal_error(
2416 reason: "Unexpected JMP instruction was emitted for a jump-table when import "
2417 "call optimization was enabled");
2418 break;
2419
2420 case X86::TLS_addr32:
2421 case X86::TLS_addr64:
2422 case X86::TLS_addrX32:
2423 case X86::TLS_base_addr32:
2424 case X86::TLS_base_addr64:
2425 case X86::TLS_base_addrX32:
2426 case X86::TLS_desc32:
2427 case X86::TLS_desc64:
2428 return LowerTlsAddr(MCInstLowering, MI: *MI);
2429
2430 case X86::MOVPC32r: {
2431 // This is a pseudo op for a two instruction sequence with a label, which
2432 // looks like:
2433 // call "L1$pb"
2434 // "L1$pb":
2435 // popl %esi
2436
2437 // Emit the call.
2438 MCSymbol *PICBase = MF->getPICBaseSymbol();
2439 // FIXME: We would like an efficient form for this, so we don't have to do a
2440 // lot of extra uniquing.
2441 EmitAndCountInstruction(
2442 Inst&: MCInstBuilder(X86::CALLpcrel32)
2443 .addExpr(Val: MCSymbolRefExpr::create(Symbol: PICBase, Ctx&: OutContext)));
2444
2445 const X86FrameLowering *FrameLowering =
2446 MF->getSubtarget<X86Subtarget>().getFrameLowering();
2447 bool hasFP = FrameLowering->hasFP(MF: *MF);
2448
2449 // TODO: This is needed only if we require precise CFA.
2450 bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() &&
2451 !OutStreamer->getDwarfFrameInfos().back().End;
2452
2453 int stackGrowth = -RI->getSlotSize();
2454
2455 if (HasActiveDwarfFrame && !hasFP) {
2456 OutStreamer->emitCFIAdjustCfaOffset(Adjustment: -stackGrowth);
2457 MF->getInfo<X86MachineFunctionInfo>()->setHasCFIAdjustCfa(true);
2458 }
2459
2460 // Emit the label.
2461 OutStreamer->emitLabel(Symbol: PICBase);
2462
2463 // popl $reg
2464 EmitAndCountInstruction(
2465 Inst&: MCInstBuilder(X86::POP32r).addReg(Reg: MI->getOperand(i: 0).getReg()));
2466
2467 if (HasActiveDwarfFrame && !hasFP) {
2468 OutStreamer->emitCFIAdjustCfaOffset(Adjustment: stackGrowth);
2469 }
2470 return;
2471 }
2472
2473 case X86::ADD32ri: {
2474 // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri.
2475 if (MI->getOperand(i: 2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS)
2476 break;
2477
2478 // Okay, we have something like:
2479 // EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL)
2480
2481 // For this, we want to print something like:
2482 // MYGLOBAL + (. - PICBASE)
2483 // However, we can't generate a ".", so just emit a new label here and refer
2484 // to it.
2485 MCSymbol *DotSym = OutContext.createTempSymbol();
2486 OutStreamer->emitLabel(Symbol: DotSym);
2487
2488 // Now that we have emitted the label, lower the complex operand expression.
2489 MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MO: MI->getOperand(i: 2));
2490
2491 const MCExpr *DotExpr = MCSymbolRefExpr::create(Symbol: DotSym, Ctx&: OutContext);
2492 const MCExpr *PICBase =
2493 MCSymbolRefExpr::create(Symbol: MF->getPICBaseSymbol(), Ctx&: OutContext);
2494 DotExpr = MCBinaryExpr::createSub(LHS: DotExpr, RHS: PICBase, Ctx&: OutContext);
2495
2496 DotExpr = MCBinaryExpr::createAdd(
2497 LHS: MCSymbolRefExpr::create(Symbol: OpSym, Ctx&: OutContext), RHS: DotExpr, Ctx&: OutContext);
2498
2499 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::ADD32ri)
2500 .addReg(Reg: MI->getOperand(i: 0).getReg())
2501 .addReg(Reg: MI->getOperand(i: 1).getReg())
2502 .addExpr(Val: DotExpr));
2503 return;
2504 }
2505 case TargetOpcode::STATEPOINT:
2506 return LowerSTATEPOINT(MI: *MI, MCIL&: MCInstLowering);
2507
2508 case TargetOpcode::FAULTING_OP:
2509 return LowerFAULTING_OP(FaultingMI: *MI, MCIL&: MCInstLowering);
2510
2511 case TargetOpcode::FENTRY_CALL:
2512 return LowerFENTRY_CALL(MI: *MI, MCIL&: MCInstLowering);
2513
2514 case TargetOpcode::PATCHABLE_OP:
2515 return LowerPATCHABLE_OP(MI: *MI, MCIL&: MCInstLowering);
2516
2517 case TargetOpcode::STACKMAP:
2518 return LowerSTACKMAP(MI: *MI);
2519
2520 case TargetOpcode::PATCHPOINT:
2521 return LowerPATCHPOINT(MI: *MI, MCIL&: MCInstLowering);
2522
2523 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2524 return LowerPATCHABLE_FUNCTION_ENTER(MI: *MI, MCIL&: MCInstLowering);
2525
2526 case TargetOpcode::PATCHABLE_RET:
2527 return LowerPATCHABLE_RET(MI: *MI, MCIL&: MCInstLowering);
2528
2529 case TargetOpcode::PATCHABLE_TAIL_CALL:
2530 return LowerPATCHABLE_TAIL_CALL(MI: *MI, MCIL&: MCInstLowering);
2531
2532 case TargetOpcode::PATCHABLE_EVENT_CALL:
2533 return LowerPATCHABLE_EVENT_CALL(MI: *MI, MCIL&: MCInstLowering);
2534
2535 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
2536 return LowerPATCHABLE_TYPED_EVENT_CALL(MI: *MI, MCIL&: MCInstLowering);
2537
2538 case X86::MORESTACK_RET:
2539 EmitAndCountInstruction(Inst&: MCInstBuilder(getRetOpcode(Subtarget: *Subtarget)));
2540 return;
2541
2542 case X86::KCFI_CHECK:
2543 return LowerKCFI_CHECK(MI: *MI);
2544
2545 case X86::ASAN_CHECK_MEMACCESS:
2546 return LowerASAN_CHECK_MEMACCESS(MI: *MI);
2547
2548 case X86::MORESTACK_RET_RESTORE_R10:
2549 // Return, then restore R10.
2550 EmitAndCountInstruction(Inst&: MCInstBuilder(getRetOpcode(Subtarget: *Subtarget)));
2551 EmitAndCountInstruction(
2552 Inst&: MCInstBuilder(X86::MOV64rr).addReg(Reg: X86::R10).addReg(Reg: X86::RAX));
2553 return;
2554
2555 case X86::SEH_PushReg:
2556 case X86::SEH_Push2Regs:
2557 case X86::SEH_SaveReg:
2558 case X86::SEH_SaveXMM:
2559 case X86::SEH_StackAlloc:
2560 case X86::SEH_StackAlign:
2561 case X86::SEH_SetFrame:
2562 case X86::SEH_PushFrame:
2563 case X86::SEH_EndPrologue:
2564 case X86::SEH_EndEpilogue:
2565 case X86::SEH_UnwindV2Start:
2566 case X86::SEH_UnwindVersion:
2567 EmitSEHInstruction(MI);
2568 return;
2569
2570 case X86::SEH_SplitChainedAtEndOfBlock:
2571 assert(!SplitChainedAtEndOfBlock &&
2572 "Duplicate SEH_SplitChainedAtEndOfBlock in a current block");
2573 SplitChainedAtEndOfBlock = true;
2574 return;
2575
2576 case X86::SEH_SplitChained:
2577 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2578 OutStreamer->emitWinCFISplitChained();
2579 return;
2580
2581 case X86::SEH_BeginEpilogue: {
2582 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2583 EmitSEHInstruction(MI);
2584 return;
2585 }
2586 case X86::UBSAN_UD1:
2587 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::UD1Lm)
2588 .addReg(Reg: X86::EAX)
2589 .addReg(Reg: X86::EAX)
2590 .addImm(Val: 1)
2591 .addReg(Reg: X86::NoRegister)
2592 .addImm(Val: MI->getOperand(i: 0).getImm())
2593 .addReg(Reg: X86::NoRegister));
2594 return;
2595 case X86::CALL64pcrel32:
2596 if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(Reg: X86::R11))
2597 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CS_PREFIX));
2598
2599 if (EnableImportCallOptimization && isImportedFunction(MO: MI->getOperand(i: 0))) {
2600 emitLabelAndRecordForImportCallOptimization(
2601 Kind: IMAGE_RETPOLINE_AMD64_IMPORT_CALL);
2602
2603 MCInst TmpInst;
2604 MCInstLowering.Lower(MI, OutMI&: TmpInst);
2605
2606 // For Import Call Optimization to work, we need a the call instruction
2607 // with a rex prefix, and a 5-byte nop after the call instruction.
2608 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::REX64_PREFIX));
2609 emitCallInstruction(MCI: TmpInst);
2610 emitNop(OS&: *OutStreamer, NumBytes: 5, Subtarget);
2611 maybeEmitNopAfterCallForWindowsEH(MI);
2612 return;
2613 }
2614
2615 break;
2616
2617 case X86::CALL64r:
2618 if (EnableImportCallOptimization) {
2619 assert(MI->getOperand(0).getReg() == X86::RAX &&
2620 "Indirect calls with impcall enabled must go through RAX (as "
2621 "enforced by CALL64r_ImpCall)");
2622
2623 emitLabelAndRecordForImportCallOptimization(
2624 Kind: IMAGE_RETPOLINE_AMD64_INDIR_CALL);
2625 MCInst TmpInst;
2626 MCInstLowering.Lower(MI, OutMI&: TmpInst);
2627 emitCallInstruction(MCI: TmpInst);
2628
2629 // For Import Call Optimization to work, we need a 3-byte nop after the
2630 // call instruction.
2631 emitNop(OS&: *OutStreamer, NumBytes: 3, Subtarget);
2632 maybeEmitNopAfterCallForWindowsEH(MI);
2633 return;
2634 }
2635 break;
2636
2637 case X86::CALL64m:
2638 if (EnableImportCallOptimization && isCallToCFGuardFunction(MI)) {
2639 emitLabelAndRecordForImportCallOptimization(
2640 Kind: IMAGE_RETPOLINE_AMD64_CFG_CALL);
2641 }
2642 break;
2643
2644 case X86::JCC_1:
2645 // Two instruction prefixes (2EH for branch not-taken and 3EH for branch
2646 // taken) are used as branch hints. Here we add branch taken prefix for
2647 // jump instruction with higher probability than threshold.
2648 if (getSubtarget().hasBranchHint() &&
2649 getSubtarget().getCLOpts().enable_branch_hint) {
2650 const MachineBranchProbabilityInfo *MBPI = GetMBPI(*MF);
2651 MachineBasicBlock *DestBB = MI->getOperand(i: 0).getMBB();
2652 BranchProbability EdgeProb =
2653 MBPI->getEdgeProbability(Src: MI->getParent(), Dst: DestBB);
2654 BranchProbability Threshold(
2655 getSubtarget().getCLOpts().branch_hint_probability_threshold, 100);
2656 if (EdgeProb > Threshold)
2657 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DS_PREFIX));
2658 }
2659 break;
2660
2661 case X86::JCC_SELF:
2662 MCSymbol *Sym = OutContext.createTempSymbol();
2663 OutStreamer->emitLabel(Symbol: Sym);
2664 EmitAndCountInstruction(
2665 Inst&: MCInstBuilder(X86::JCC_1)
2666 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Sym, Ctx&: OutContext))
2667 .addImm(Val: MI->getOperand(i: 0).getImm()));
2668 return;
2669 }
2670
2671 MCInst TmpInst;
2672 MCInstLowering.Lower(MI, OutMI&: TmpInst);
2673
2674 if (MI->isCall()) {
2675 emitCallInstruction(MCI: TmpInst);
2676 // Since tail calls transfer control without leaving a stack frame, there is
2677 // never a need for NOP padding tail calls.
2678 if (!IsTailJump)
2679 maybeEmitNopAfterCallForWindowsEH(MI);
2680 return;
2681 }
2682
2683 EmitAndCountInstruction(Inst&: TmpInst);
2684}
2685
2686void X86AsmPrinter::emitInlineAsmEnd(const MCSubtargetInfo &StartInfo,
2687 const MCSubtargetInfo *EndInfo,
2688 const MachineInstr *MI) {
2689 if (MI) {
2690 // If unwinding inline asm ends on a call, wineh may require insertion of
2691 // a nop.
2692 unsigned ExtraInfo = MI->getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
2693 if (ExtraInfo & InlineAsm::Extra_MayUnwind)
2694 maybeEmitNopAfterCallForWindowsEH(MI);
2695 }
2696}
2697
2698void X86AsmPrinter::emitCallInstruction(const llvm::MCInst &MCI) {
2699 // Stackmap shadows cannot include branch targets, so we can count the bytes
2700 // in a call towards the shadow, but must ensure that the no thread returns
2701 // in to the stackmap shadow. The only way to achieve this is if the call
2702 // is at the end of the shadow.
2703
2704 // Count then size of the call towards the shadow
2705 SMShadowTracker.count(Inst: MCI, STI: getSubtargetInfo(), CodeEmitter: CodeEmitter.get());
2706 // Then flush the shadow so that we fill with nops before the call, not
2707 // after it.
2708 SMShadowTracker.emitShadowPadding(OutStreamer&: *OutStreamer, STI: getSubtargetInfo());
2709 // Then emit the call
2710 OutStreamer->emitInstruction(Inst: MCI, STI: getSubtargetInfo());
2711}
2712
2713// Determines whether a NOP is required after a CALL, so that Windows EH
2714// IP2State tables have the correct information.
2715//
2716// On most Windows platforms (AMD64, ARM64, ARM32, IA64, but *not* x86-32),
2717// exception handling works by looking up instruction pointers in lookup
2718// tables. These lookup tables are stored in .xdata sections in executables.
2719// One element of the lookup tables are the "IP2State" tables (Instruction
2720// Pointer to State).
2721//
2722// If a function has any instructions that require cleanup during exception
2723// unwinding, then it will have an IP2State table. Each entry in the IP2State
2724// table describes a range of bytes in the function's instruction stream, and
2725// associates an "EH state number" with that range of instructions. A value of
2726// -1 means "the null state", which does not require any code to execute.
2727// A value other than -1 is an index into the State table.
2728//
2729// The entries in the IP2State table contain byte offsets within the instruction
2730// stream of the function. The Windows ABI requires that these offsets are
2731// aligned to instruction boundaries; they are not permitted to point to a byte
2732// that is not the first byte of an instruction.
2733//
2734// Unfortunately, CALL instructions present a problem during unwinding. CALL
2735// instructions push the address of the instruction after the CALL instruction,
2736// so that execution can resume after the CALL. If the CALL is the last
2737// instruction within an IP2State region, then the return address (on the stack)
2738// points to the *next* IP2State region. This means that the unwinder will
2739// use the wrong cleanup funclet during unwinding.
2740//
2741// To fix this problem, the Windows AMD64 ABI requires that CALL instructions
2742// are never placed at the end of an IP2State region. Stated equivalently, the
2743// end of a CALL instruction cannot be aligned to an IP2State boundary. If a
2744// CALL instruction would occur at the end of an IP2State region, then the
2745// compiler must insert a NOP instruction after the CALL. The NOP instruction
2746// is placed in the same EH region as the CALL instruction, so that the return
2747// address points to the NOP and the unwinder will locate the correct region.
2748//
2749// NOP padding is only necessary on Windows AMD64 targets. On ARM64 and ARM32,
2750// instructions have a fixed size so the unwinder knows how to "back up" by
2751// one instruction.
2752//
2753// Interaction with Import Call Optimization (ICO):
2754//
2755// Import Call Optimization (ICO) is a compiler + OS feature on Windows which
2756// improves the performance and security of DLL imports. ICO relies on using a
2757// specific CALL idiom that can be replaced by the OS DLL loader. This removes
2758// a load and indirect CALL and replaces it with a single direct CALL.
2759//
2760// To achieve this, ICO also inserts NOPs after the CALL instruction. If the
2761// end of the CALL is aligned with an EH state transition, we *also* insert
2762// a single-byte NOP. **Both forms of NOPs must be preserved.** They cannot
2763// be combined into a single larger NOP; nor can the second NOP be removed.
2764//
2765// This is necessary because, if ICO is active and the call site is modified
2766// by the loader, the loader will end up overwriting the NOPs that were inserted
2767// for ICO. That means that those NOPs cannot be used for the correct
2768// termination of the exception handling region (the IP2State transition),
2769// so we still need an additional NOP instruction. The NOPs cannot be combined
2770// into a longer NOP (which is ordinarily desirable) because then ICO would
2771// split one instruction, producing a malformed instruction after the ICO call.
2772void X86AsmPrinter::maybeEmitNopAfterCallForWindowsEH(const MachineInstr *MI) {
2773 // We only need to insert NOPs after CALLs when targeting Windows on AMD64.
2774 // (Don't let the name fool you: Itanium refers to table-based exception
2775 // handling, not the Itanium architecture.)
2776 if (MAI.getExceptionHandlingType() != ExceptionHandling::WinEH ||
2777 MAI.getWinEHEncodingType() != WinEH::EncodingType::Itanium) {
2778 return;
2779 }
2780
2781 bool HasEHPersonality = MF->getWinEHFuncInfo() != nullptr;
2782
2783 // Set up MBB iterator, initially positioned on the same MBB as MI.
2784 MachineFunction::const_iterator MFI(MI->getParent());
2785 MachineFunction::const_iterator MFE(MF->end());
2786
2787 // Set up instruction iterator, positioned immediately *after* MI.
2788 MachineBasicBlock::const_iterator MBBI(MI);
2789 MachineBasicBlock::const_iterator MBBE = MI->getParent()->end();
2790 ++MBBI; // Step over MI
2791
2792 // This loop iterates MBBs
2793 for (;;) {
2794 // This loop iterates instructions
2795 for (; MBBI != MBBE; ++MBBI) {
2796 // Check the instruction that follows this CALL.
2797 const MachineInstr &NextMI = *MBBI;
2798
2799 // If there is an EH_LABEL after this CALL, then there is an EH state
2800 // transition after this CALL. This is exactly the situation which
2801 // requires NOP padding.
2802 if (NextMI.isEHLabel()) {
2803 if (HasEHPersonality) {
2804 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::NOOP));
2805 return;
2806 }
2807 // We actually want to continue, in case there is an SEH_BeginEpilogue
2808 // instruction after the EH_LABEL. In some situations, IR is produced
2809 // that contains EH_LABEL pseudo-instructions, even when we are not
2810 // generating IP2State tables. We still need to insert a NOP before
2811 // SEH_BeginEpilogue in that case.
2812 continue;
2813 }
2814
2815 // Somewhat similarly, if the CALL is the last instruction before the
2816 // SEH prologue, then we also need a NOP. This is necessary because the
2817 // Windows stack unwinder will not invoke a function's exception handler
2818 // if the instruction pointer is in the function prologue or epilogue.
2819 //
2820 // We always emit a NOP before SEH_BeginEpilogue, even if there is no
2821 // personality function (unwind info) for this frame. This is the same
2822 // behavior as MSVC.
2823 if (NextMI.getOpcode() == X86::SEH_BeginEpilogue) {
2824 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::NOOP));
2825 return;
2826 }
2827
2828 if (!NextMI.isPseudo() && !NextMI.isMetaInstruction()) {
2829 // We found a real instruction. During the CALL, the return IP will
2830 // point to this instruction. Since this instruction has the same EH
2831 // state as the call itself (because there is no intervening EH_LABEL),
2832 // the IP2State table will be accurate; there is no need to insert a
2833 // NOP.
2834 return;
2835 }
2836
2837 // The next instruction is a pseudo-op. Ignore it and keep searching.
2838 // Because these instructions do not generate any machine code, they
2839 // cannot prevent the IP2State table from pointing at the wrong
2840 // instruction during a CALL.
2841 }
2842
2843 // We've reached the end of this MBB. Find the next MBB in program order.
2844 // MBB order should be finalized by this point, so falling across MBBs is
2845 // expected.
2846 ++MFI;
2847 if (MFI == MFE) {
2848 // No more blocks; we've reached the end of the function. This should
2849 // only happen with no-return functions, but double-check to be sure.
2850 if (HasEHPersonality) {
2851 // If the CALL has no successors, then it is a noreturn function.
2852 // Insert an INT3 instead of a NOP. This accomplishes the same purpose,
2853 // but is more clear to read. Also, analysis tools will understand
2854 // that they should not continue disassembling after the CALL (unless
2855 // there are other branches to that label).
2856 if (MI->getParent()->succ_empty())
2857 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::INT3));
2858 else
2859 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::NOOP));
2860 }
2861 return;
2862 }
2863
2864 // Set up iterator to scan the next basic block.
2865 const MachineBasicBlock *NextMBB = &*MFI;
2866 MBBI = NextMBB->instr_begin();
2867 MBBE = NextMBB->instr_end();
2868 }
2869}
2870
2871void X86AsmPrinter::emitLabelAndRecordForImportCallOptimization(
2872 ImportCallKind Kind) {
2873 assert(EnableImportCallOptimization);
2874
2875 MCSymbol *CallSiteSymbol = MMI->getContext().createNamedTempSymbol(Name: "impcall");
2876 OutStreamer->emitLabel(Symbol: CallSiteSymbol);
2877
2878 SectionToImportedFunctionCalls[OutStreamer->getCurrentSectionOnly()]
2879 .push_back(x: {.CalleeSymbol: CallSiteSymbol, .Kind: Kind});
2880}
2881