1//===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains code to lower X86 MachineInstrs to their corresponding
10// MCInst records.
11//
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/X86ATTInstPrinter.h"
15#include "MCTargetDesc/X86BaseInfo.h"
16#include "MCTargetDesc/X86EncodingOptimization.h"
17#include "MCTargetDesc/X86InstComments.h"
18#include "MCTargetDesc/X86MCAsmInfo.h"
19#include "MCTargetDesc/X86ShuffleDecode.h"
20#include "MCTargetDesc/X86TargetStreamer.h"
21#include "X86AsmPrinter.h"
22#include "X86MachineFunctionInfo.h"
23#include "X86RegisterInfo.h"
24#include "X86ShuffleDecodeConstantPool.h"
25#include "X86Subtarget.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SmallString.h"
28#include "llvm/ADT/StringExtras.h"
29#include "llvm/BinaryFormat/ELF.h"
30#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
31#include "llvm/CodeGen/MachineConstantPool.h"
32#include "llvm/CodeGen/MachineFunction.h"
33#include "llvm/CodeGen/MachineModuleInfoImpls.h"
34#include "llvm/CodeGen/MachineOperand.h"
35#include "llvm/CodeGen/StackMaps.h"
36#include "llvm/CodeGen/WinEHFuncInfo.h"
37#include "llvm/IR/DataLayout.h"
38#include "llvm/IR/GlobalValue.h"
39#include "llvm/IR/Mangler.h"
40#include "llvm/MC/MCAsmInfo.h"
41#include "llvm/MC/MCCodeEmitter.h"
42#include "llvm/MC/MCContext.h"
43#include "llvm/MC/MCExpr.h"
44#include "llvm/MC/MCFixup.h"
45#include "llvm/MC/MCInst.h"
46#include "llvm/MC/MCInstBuilder.h"
47#include "llvm/MC/MCSection.h"
48#include "llvm/MC/MCSectionELF.h"
49#include "llvm/MC/MCStreamer.h"
50#include "llvm/MC/MCSymbol.h"
51#include "llvm/MC/TargetRegistry.h"
52#include "llvm/Target/TargetLoweringObjectFile.h"
53#include "llvm/Target/TargetMachine.h"
54#include "llvm/Transforms/CFGuard.h"
55#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
56#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
57#include <string>
58
59using namespace llvm;
60
61namespace {
62
63/// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.
64class X86MCInstLower {
65 MCContext &Ctx;
66 const MachineFunction &MF;
67 const TargetMachine &TM;
68 const MCAsmInfo &MAI;
69 X86AsmPrinter &AsmPrinter;
70
71public:
72 X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter);
73
74 MCOperand LowerMachineOperand(const MachineInstr *MI,
75 const MachineOperand &MO) const;
76 void Lower(const MachineInstr *MI, MCInst &OutMI) const;
77
78 MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const;
79 MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const;
80
81private:
82 MachineModuleInfoMachO &getMachOMMI() const;
83};
84
85} // end anonymous namespace
86
87/// A RAII helper which defines a region of instructions which can't have
88/// padding added between them for correctness.
89struct NoAutoPaddingScope {
90 MCStreamer &OS;
91 const bool OldAllowAutoPadding;
92 NoAutoPaddingScope(MCStreamer &OS)
93 : OS(OS), OldAllowAutoPadding(OS.getAllowAutoPadding()) {
94 changeAndComment(b: false);
95 }
96 ~NoAutoPaddingScope() { changeAndComment(b: OldAllowAutoPadding); }
97 void changeAndComment(bool b) {
98 if (b == OS.getAllowAutoPadding())
99 return;
100 OS.setAllowAutoPadding(b);
101 if (b)
102 OS.emitRawComment(T: "autopadding");
103 else
104 OS.emitRawComment(T: "noautopadding");
105 }
106};
107
108// Emit a minimal sequence of nops spanning NumBytes bytes.
109static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
110 const X86Subtarget *Subtarget);
111
112void X86AsmPrinter::StackMapShadowTracker::count(const MCInst &Inst,
113 const MCSubtargetInfo &STI,
114 MCCodeEmitter *CodeEmitter) {
115 if (InShadow) {
116 SmallString<256> Code;
117 SmallVector<MCFixup, 4> Fixups;
118 CodeEmitter->encodeInstruction(Inst, CB&: Code, Fixups, STI);
119 CurrentShadowSize += Code.size();
120 if (CurrentShadowSize >= RequiredShadowSize)
121 InShadow = false; // The shadow is big enough. Stop counting.
122 }
123}
124
125void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding(
126 MCStreamer &OutStreamer, const MCSubtargetInfo &STI) {
127 if (InShadow && CurrentShadowSize < RequiredShadowSize) {
128 InShadow = false;
129 emitX86Nops(OS&: OutStreamer, NumBytes: RequiredShadowSize - CurrentShadowSize,
130 Subtarget: &MF->getSubtarget<X86Subtarget>());
131 }
132}
133
134void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {
135 OutStreamer->emitInstruction(Inst, STI: getSubtargetInfo());
136 SMShadowTracker.count(Inst, STI: getSubtargetInfo(), CodeEmitter: CodeEmitter.get());
137}
138
139X86MCInstLower::X86MCInstLower(const MachineFunction &mf,
140 X86AsmPrinter &asmprinter)
141 : Ctx(asmprinter.OutContext), MF(mf), TM(mf.getTarget()),
142 MAI(TM.getMCAsmInfo()), AsmPrinter(asmprinter) {}
143
144MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {
145 return AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoMachO>();
146}
147
148/// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol
149/// operand to an MCSymbol.
150MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const {
151 const Triple &TT = TM.getTargetTriple();
152 if (MO.isGlobal() && TT.isOSBinFormatELF())
153 return AsmPrinter.getSymbolPreferLocal(GV: *MO.getGlobal());
154
155 const DataLayout &DL = MF.getDataLayout();
156 assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) &&
157 "Isn't a symbol reference");
158
159 MCSymbol *Sym = nullptr;
160 SmallString<128> Name;
161 StringRef Suffix;
162
163 switch (MO.getTargetFlags()) {
164 case X86II::MO_DLLIMPORT:
165 // Handle dllimport linkage.
166 Name += "__imp_";
167 break;
168 case X86II::MO_COFFSTUB:
169 Name += ".refptr.";
170 break;
171 case X86II::MO_DARWIN_NONLAZY:
172 case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
173 Suffix = "$non_lazy_ptr";
174 break;
175 }
176
177 if (!Suffix.empty())
178 Name += DL.getInternalSymbolPrefix();
179
180 if (MO.isGlobal()) {
181 const GlobalValue *GV = MO.getGlobal();
182 AsmPrinter.getNameWithPrefix(Name, GV);
183 } else if (MO.isSymbol()) {
184 Mangler::getNameWithPrefix(OutName&: Name, GVName: MO.getSymbolName(), DL);
185 } else if (MO.isMBB()) {
186 assert(Suffix.empty());
187 Sym = MO.getMBB()->getSymbol();
188 }
189
190 Name += Suffix;
191 if (!Sym)
192 Sym = Ctx.getOrCreateSymbol(Name);
193
194 // If the target flags on the operand changes the name of the symbol, do that
195 // before we return the symbol.
196 switch (MO.getTargetFlags()) {
197 default:
198 break;
199 case X86II::MO_COFFSTUB: {
200 MachineModuleInfoCOFF &MMICOFF =
201 AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoCOFF>();
202 MachineModuleInfoImpl::StubValueTy &StubSym = MMICOFF.getGVStubEntry(Sym);
203 if (!StubSym.getPointer()) {
204 assert(MO.isGlobal() && "Extern symbol not handled yet");
205 StubSym = MachineModuleInfoImpl::StubValueTy(
206 AsmPrinter.getSymbol(GV: MO.getGlobal()), true);
207 }
208 break;
209 }
210 case X86II::MO_DARWIN_NONLAZY:
211 case X86II::MO_DARWIN_NONLAZY_PIC_BASE: {
212 MachineModuleInfoImpl::StubValueTy &StubSym =
213 getMachOMMI().getGVStubEntry(Sym);
214 if (!StubSym.getPointer()) {
215 assert(MO.isGlobal() && "Extern symbol not handled yet");
216 StubSym = MachineModuleInfoImpl::StubValueTy(
217 AsmPrinter.getSymbol(GV: MO.getGlobal()),
218 !MO.getGlobal()->hasInternalLinkage());
219 }
220 break;
221 }
222 }
223
224 return Sym;
225}
226
227MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
228 MCSymbol *Sym) const {
229 // FIXME: We would like an efficient form for this, so we don't have to do a
230 // lot of extra uniquing.
231 const MCExpr *Expr = nullptr;
232 uint16_t Specifier = X86::S_None;
233
234 switch (MO.getTargetFlags()) {
235 default:
236 llvm_unreachable("Unknown target flag on GV operand");
237 case X86II::MO_NO_FLAG: // No flag.
238 // These affect the name of the symbol, not any suffix.
239 case X86II::MO_DARWIN_NONLAZY:
240 case X86II::MO_DLLIMPORT:
241 case X86II::MO_COFFSTUB:
242 break;
243
244 case X86II::MO_TLVP:
245 Specifier = X86::S_TLVP;
246 break;
247 case X86II::MO_TLVP_PIC_BASE:
248 Expr = MCSymbolRefExpr::create(Symbol: Sym, specifier: X86::S_TLVP, Ctx);
249 // Subtract the pic base.
250 Expr = MCBinaryExpr::createSub(
251 LHS: Expr, RHS: MCSymbolRefExpr::create(Symbol: MF.getPICBaseSymbol(), Ctx), Ctx);
252 break;
253 case X86II::MO_SECREL:
254 Specifier = uint16_t(X86::S_COFF_SECREL);
255 break;
256 case X86II::MO_TLSGD:
257 Specifier = X86::S_TLSGD;
258 break;
259 case X86II::MO_TLSLD:
260 Specifier = X86::S_TLSLD;
261 break;
262 case X86II::MO_TLSLDM:
263 Specifier = X86::S_TLSLDM;
264 break;
265 case X86II::MO_GOTTPOFF:
266 Specifier = X86::S_GOTTPOFF;
267 break;
268 case X86II::MO_INDNTPOFF:
269 Specifier = X86::S_INDNTPOFF;
270 break;
271 case X86II::MO_TPOFF:
272 Specifier = X86::S_TPOFF;
273 break;
274 case X86II::MO_DTPOFF:
275 Specifier = X86::S_DTPOFF;
276 break;
277 case X86II::MO_NTPOFF:
278 Specifier = X86::S_NTPOFF;
279 break;
280 case X86II::MO_GOTNTPOFF:
281 Specifier = X86::S_GOTNTPOFF;
282 break;
283 case X86II::MO_GOTPCREL:
284 Specifier = X86::S_GOTPCREL;
285 break;
286 case X86II::MO_GOTPCREL_NORELAX:
287 Specifier = X86::S_GOTPCREL_NORELAX;
288 break;
289 case X86II::MO_GOT:
290 Specifier = X86::S_GOT;
291 break;
292 case X86II::MO_GOTOFF:
293 Specifier = X86::S_GOTOFF;
294 break;
295 case X86II::MO_PLT:
296 Specifier = X86::S_PLT;
297 break;
298 case X86II::MO_ABS8:
299 Specifier = X86::S_ABS8;
300 break;
301 case X86II::MO_PIC_BASE_OFFSET:
302 case X86II::MO_DARWIN_NONLAZY_PIC_BASE:
303 Expr = MCSymbolRefExpr::create(Symbol: Sym, Ctx);
304 // Subtract the pic base.
305 Expr = MCBinaryExpr::createSub(
306 LHS: Expr, RHS: MCSymbolRefExpr::create(Symbol: MF.getPICBaseSymbol(), Ctx), Ctx);
307 if (MO.isJTI()) {
308 assert(MAI.doesSetDirectiveSuppressReloc());
309 // If .set directive is supported, use it to reduce the number of
310 // relocations the assembler will generate for differences between
311 // local labels. This is only safe when the symbols are in the same
312 // section so we are restricting it to jumptable references.
313 MCSymbol *Label = Ctx.createTempSymbol();
314 AsmPrinter.OutStreamer->emitAssignment(Symbol: Label, Value: Expr);
315 Expr = MCSymbolRefExpr::create(Symbol: Label, Ctx);
316 }
317 break;
318 }
319
320 if (!Expr)
321 Expr = MCSymbolRefExpr::create(Symbol: Sym, specifier: Specifier, Ctx);
322
323 if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
324 Expr = MCBinaryExpr::createAdd(
325 LHS: Expr, RHS: MCConstantExpr::create(Value: MO.getOffset(), Ctx), Ctx);
326 return MCOperand::createExpr(Val: Expr);
327}
328
329static unsigned getRetOpcode(const X86Subtarget &Subtarget) {
330 return Subtarget.is64Bit() ? X86::RET64 : X86::RET32;
331}
332
333MCOperand X86MCInstLower::LowerMachineOperand(const MachineInstr *MI,
334 const MachineOperand &MO) const {
335 switch (MO.getType()) {
336 default:
337 MI->print(OS&: errs());
338 llvm_unreachable("unknown operand type");
339 case MachineOperand::MO_Register:
340 // Ignore all implicit register operands.
341 if (MO.isImplicit())
342 return MCOperand();
343 return MCOperand::createReg(Reg: MO.getReg());
344 case MachineOperand::MO_Immediate:
345 return MCOperand::createImm(Val: MO.getImm());
346 case MachineOperand::MO_MachineBasicBlock:
347 case MachineOperand::MO_GlobalAddress:
348 case MachineOperand::MO_ExternalSymbol:
349 return LowerSymbolOperand(MO, Sym: GetSymbolFromOperand(MO));
350 case MachineOperand::MO_MCSymbol:
351 return LowerSymbolOperand(MO, Sym: MO.getMCSymbol());
352 case MachineOperand::MO_JumpTableIndex:
353 return LowerSymbolOperand(MO, Sym: AsmPrinter.GetJTISymbol(JTID: MO.getIndex()));
354 case MachineOperand::MO_ConstantPoolIndex:
355 return LowerSymbolOperand(MO, Sym: AsmPrinter.GetCPISymbol(CPID: MO.getIndex()));
356 case MachineOperand::MO_BlockAddress:
357 return LowerSymbolOperand(
358 MO, Sym: AsmPrinter.GetBlockAddressSymbol(BA: MO.getBlockAddress()));
359 case MachineOperand::MO_RegisterMask:
360 // Ignore call clobbers.
361 return MCOperand();
362 }
363}
364
365// Replace TAILJMP opcodes with their equivalent opcodes that have encoding
366// information.
367static unsigned convertTailJumpOpcode(unsigned Opcode, bool IsLarge = false) {
368 switch (Opcode) {
369 case X86::TAILJMPr:
370 Opcode = X86::JMP32r;
371 break;
372 case X86::TAILJMPm:
373 Opcode = X86::JMP32m;
374 break;
375 case X86::TAILJMPr64:
376 Opcode = X86::JMP64r;
377 break;
378 case X86::TAILJMPm64:
379 Opcode = X86::JMP64m;
380 break;
381 case X86::TAILJMPr64_REX:
382 Opcode = X86::JMP64r_REX;
383 break;
384 case X86::TAILJMPm64_REX:
385 Opcode = X86::JMP64m_REX;
386 break;
387 case X86::TAILJMPd:
388 case X86::TAILJMPd64:
389 Opcode = IsLarge ? X86::JMPABS64i : X86::JMP_1;
390 break;
391 case X86::TAILJMPd_CC:
392 case X86::TAILJMPd64_CC:
393 Opcode = X86::JCC_1;
394 break;
395 }
396
397 return Opcode;
398}
399
400void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
401 OutMI.setOpcode(MI->getOpcode());
402
403 for (const MachineOperand &MO : MI->operands())
404 if (auto Op = LowerMachineOperand(MI, MO); Op.isValid())
405 OutMI.addOperand(Op);
406
407 bool In64BitMode = AsmPrinter.getSubtarget().is64Bit();
408 if (X86::optimizeInstFromVEX3ToVEX2(MI&: OutMI, Desc: MI->getDesc()) ||
409 X86::optimizeShiftRotateWithImmediateOne(MI&: OutMI) ||
410 X86::optimizeVPCMPWithImmediateOneOrSix(MI&: OutMI) ||
411 X86::optimizeMOVSX(MI&: OutMI) || X86::optimizeINCDEC(MI&: OutMI, In64BitMode) ||
412 X86::optimizeMOV(MI&: OutMI, In64BitMode) ||
413 X86::optimizeToFixedRegisterOrShortImmediateForm(MI&: OutMI))
414 return;
415
416 // Handle a few special cases to eliminate operand modifiers.
417 switch (OutMI.getOpcode()) {
418 case X86::LEA64_32r:
419 case X86::LEA64r:
420 case X86::LEA16r:
421 case X86::LEA32r:
422 // LEA should have a segment register, but it must be empty.
423 assert(OutMI.getNumOperands() == 1 + X86::AddrNumOperands &&
424 "Unexpected # of LEA operands");
425 assert(OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 &&
426 "LEA has segment specified!");
427 break;
428 case X86::MULX32Hrr:
429 case X86::MULX32Hrm:
430 case X86::MULX64Hrr:
431 case X86::MULX64Hrm: {
432 // Turn into regular MULX by duplicating the destination.
433 unsigned NewOpc;
434 switch (OutMI.getOpcode()) {
435 default: llvm_unreachable("Invalid opcode");
436 case X86::MULX32Hrr: NewOpc = X86::MULX32rr; break;
437 case X86::MULX32Hrm: NewOpc = X86::MULX32rm; break;
438 case X86::MULX64Hrr: NewOpc = X86::MULX64rr; break;
439 case X86::MULX64Hrm: NewOpc = X86::MULX64rm; break;
440 }
441 OutMI.setOpcode(NewOpc);
442 // Duplicate the destination.
443 MCRegister DestReg = OutMI.getOperand(i: 0).getReg();
444 OutMI.insert(I: OutMI.begin(), Op: MCOperand::createReg(Reg: DestReg));
445 break;
446 }
447 // CALL64r, CALL64pcrel32 - These instructions used to have
448 // register inputs modeled as normal uses instead of implicit uses. As such,
449 // they we used to truncate off all but the first operand (the callee). This
450 // issue seems to have been fixed at some point. This assert verifies that.
451 case X86::CALL64r:
452 case X86::CALL64pcrel32:
453 assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
454 break;
455 case X86::EH_RETURN:
456 case X86::EH_RETURN64: {
457 OutMI = MCInst();
458 OutMI.setOpcode(getRetOpcode(Subtarget: AsmPrinter.getSubtarget()));
459 break;
460 }
461 case X86::CLEANUPRET: {
462 // Replace CLEANUPRET with the appropriate RET.
463 OutMI = MCInst();
464 OutMI.setOpcode(getRetOpcode(Subtarget: AsmPrinter.getSubtarget()));
465 break;
466 }
467 case X86::CATCHRET: {
468 // Replace CATCHRET with the appropriate RET.
469 const X86Subtarget &Subtarget = AsmPrinter.getSubtarget();
470 unsigned ReturnReg = In64BitMode ? X86::RAX : X86::EAX;
471 OutMI = MCInst();
472 OutMI.setOpcode(getRetOpcode(Subtarget));
473 OutMI.addOperand(Op: MCOperand::createReg(Reg: ReturnReg));
474 break;
475 }
476 // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump
477 // instruction.
478 case X86::TAILJMPr:
479 case X86::TAILJMPr64:
480 case X86::TAILJMPr64_REX:
481 case X86::TAILJMPd:
482 assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
483 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode()));
484 break;
485 case X86::TAILJMPd64: {
486 assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
487 bool IsLarge = TM.getCodeModel() == CodeModel::Large;
488 assert((!IsLarge || AsmPrinter.getSubtarget().hasJMPABS()) &&
489 "Unexpected TAILJMPd64 in large code model without JMPABS");
490 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode(), IsLarge));
491 break;
492 }
493 case X86::TAILJMPd_CC:
494 case X86::TAILJMPd64_CC:
495 assert(OutMI.getNumOperands() == 2 && "Unexpected number of operands!");
496 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode()));
497 break;
498 case X86::TAILJMPm:
499 case X86::TAILJMPm64:
500 case X86::TAILJMPm64_REX:
501 assert(OutMI.getNumOperands() == X86::AddrNumOperands &&
502 "Unexpected number of operands!");
503 OutMI.setOpcode(convertTailJumpOpcode(Opcode: OutMI.getOpcode()));
504 break;
505 case X86::MASKMOVDQU:
506 case X86::VMASKMOVDQU:
507 if (In64BitMode)
508 OutMI.setFlags(X86::IP_HAS_AD_SIZE);
509 break;
510 case X86::BSF16rm:
511 case X86::BSF16rr:
512 case X86::BSF32rm:
513 case X86::BSF32rr:
514 case X86::BSF64rm:
515 case X86::BSF64rr: {
516 // Add an REP prefix to BSF instructions so that new processors can
517 // recognize as TZCNT, which has better performance than BSF.
518 // BSF and TZCNT have different interpretations on ZF bit. So make sure
519 // it won't be used later.
520 const MachineOperand *FlagDef =
521 MI->findRegisterDefOperand(Reg: X86::EFLAGS, /*TRI=*/nullptr);
522 if (!MF.getFunction().hasOptSize() && FlagDef && FlagDef->isDead())
523 OutMI.setFlags(X86::IP_HAS_REPEAT);
524 break;
525 }
526 default:
527 break;
528 }
529}
530
531void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
532 const MachineInstr &MI) {
533 NoAutoPaddingScope NoPadScope(*OutStreamer);
534 bool Is64Bits = getSubtarget().is64Bit();
535 bool Is64BitsLP64 = getSubtarget().isTarget64BitLP64();
536 MCContext &Ctx = OutStreamer->getContext();
537
538 X86::Specifier Specifier;
539 switch (MI.getOpcode()) {
540 case X86::TLS_addr32:
541 case X86::TLS_addr64:
542 case X86::TLS_addrX32:
543 Specifier = X86::S_TLSGD;
544 break;
545 case X86::TLS_base_addr32:
546 Specifier = X86::S_TLSLDM;
547 break;
548 case X86::TLS_base_addr64:
549 case X86::TLS_base_addrX32:
550 Specifier = X86::S_TLSLD;
551 break;
552 case X86::TLS_desc32:
553 case X86::TLS_desc64:
554 Specifier = X86::S_TLSDESC;
555 break;
556 default:
557 llvm_unreachable("unexpected opcode");
558 }
559
560 const MCSymbolRefExpr *Sym = MCSymbolRefExpr::create(
561 Symbol: MCInstLowering.GetSymbolFromOperand(MO: MI.getOperand(i: 3)), specifier: Specifier, Ctx);
562
563 // Before binutils 2.41, ld has a bogus TLS relaxation error when the GD/LD
564 // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is
565 // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by
566 // only using GOT when GOTPCRELX is enabled.
567 // TODO Delete the workaround when rustc no longer relies on the hack
568 bool UseGot = MMI->getModule()->getRtLibUseGOT() &&
569 Ctx.getTargetOptions().X86RelaxRelocations;
570
571 if (Specifier == X86::S_TLSDESC) {
572 const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(
573 Symbol: MCInstLowering.GetSymbolFromOperand(MO: MI.getOperand(i: 3)), specifier: X86::S_TLSCALL,
574 Ctx);
575 EmitAndCountInstruction(
576 Inst&: MCInstBuilder(Is64BitsLP64 ? X86::LEA64r : X86::LEA32r)
577 .addReg(Reg: Is64BitsLP64 ? X86::RAX : X86::EAX)
578 .addReg(Reg: Is64Bits ? X86::RIP : X86::EBX)
579 .addImm(Val: 1)
580 .addReg(Reg: 0)
581 .addExpr(Val: Sym)
582 .addReg(Reg: 0));
583 EmitAndCountInstruction(
584 Inst&: MCInstBuilder(Is64Bits ? X86::CALL64m : X86::CALL32m)
585 .addReg(Reg: Is64BitsLP64 ? X86::RAX : X86::EAX)
586 .addImm(Val: 1)
587 .addReg(Reg: 0)
588 .addExpr(Val: Expr)
589 .addReg(Reg: 0));
590 } else if (Is64Bits) {
591 bool NeedsPadding = Specifier == X86::S_TLSGD;
592 if (NeedsPadding && Is64BitsLP64)
593 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DATA16_PREFIX));
594 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::LEA64r)
595 .addReg(Reg: X86::RDI)
596 .addReg(Reg: X86::RIP)
597 .addImm(Val: 1)
598 .addReg(Reg: 0)
599 .addExpr(Val: Sym)
600 .addReg(Reg: 0));
601 const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol(Name: "__tls_get_addr");
602 if (NeedsPadding) {
603 if (!UseGot)
604 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DATA16_PREFIX));
605 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DATA16_PREFIX));
606 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::REX64_PREFIX));
607 }
608 if (UseGot) {
609 const MCExpr *Expr =
610 MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_GOTPCREL, Ctx);
611 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64m)
612 .addReg(Reg: X86::RIP)
613 .addImm(Val: 1)
614 .addReg(Reg: 0)
615 .addExpr(Val: Expr)
616 .addReg(Reg: 0));
617 } else {
618 EmitAndCountInstruction(
619 Inst&: MCInstBuilder(X86::CALL64pcrel32)
620 .addExpr(Val: MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_PLT, Ctx)));
621 }
622 } else {
623 if (Specifier == X86::S_TLSGD && !UseGot) {
624 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::LEA32r)
625 .addReg(Reg: X86::EAX)
626 .addReg(Reg: 0)
627 .addImm(Val: 1)
628 .addReg(Reg: X86::EBX)
629 .addExpr(Val: Sym)
630 .addReg(Reg: 0));
631 } else {
632 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::LEA32r)
633 .addReg(Reg: X86::EAX)
634 .addReg(Reg: X86::EBX)
635 .addImm(Val: 1)
636 .addReg(Reg: 0)
637 .addExpr(Val: Sym)
638 .addReg(Reg: 0));
639 }
640
641 const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol(Name: "___tls_get_addr");
642 if (UseGot) {
643 const MCExpr *Expr = MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_GOT, Ctx);
644 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL32m)
645 .addReg(Reg: X86::EBX)
646 .addImm(Val: 1)
647 .addReg(Reg: 0)
648 .addExpr(Val: Expr)
649 .addReg(Reg: 0));
650 } else {
651 EmitAndCountInstruction(
652 Inst&: MCInstBuilder(X86::CALLpcrel32)
653 .addExpr(Val: MCSymbolRefExpr::create(Symbol: TlsGetAddr, specifier: X86::S_PLT, Ctx)));
654 }
655 }
656}
657
658/// Emit the largest nop instruction smaller than or equal to \p NumBytes
659/// bytes. Return the size of nop emitted.
660static unsigned emitNop(MCStreamer &OS, unsigned NumBytes,
661 const X86Subtarget *Subtarget) {
662 // Determine the longest nop which can be efficiently decoded for the given
663 // target cpu. 15-bytes is the longest single NOP instruction, but some
664 // platforms can't decode the longest forms efficiently.
665 unsigned MaxNopLength = 1;
666 unsigned BaseReg = X86::RAX;
667 if (Subtarget->is64Bit()) {
668 if (Subtarget->hasFeature(Feature: X86::TuningFast7ByteNOP))
669 MaxNopLength = 7;
670 else if (Subtarget->hasFeature(Feature: X86::TuningFast15ByteNOP))
671 MaxNopLength = 15;
672 else if (Subtarget->hasFeature(Feature: X86::TuningFast11ByteNOP))
673 MaxNopLength = 11;
674 else
675 MaxNopLength = 10;
676 } else if (Subtarget->is32Bit() && Subtarget->hasFeature(Feature: X86::FeatureNOPL)) {
677 BaseReg = X86::EAX;
678 MaxNopLength = 10;
679 } else if (Subtarget->is32Bit())
680 MaxNopLength = 2;
681
682 // Cap a single nop emission at the profitable value for the target
683 NumBytes = std::min(a: NumBytes, b: MaxNopLength);
684
685 unsigned NopSize;
686 unsigned Opc, ScaleVal, IndexReg, Displacement, SegmentReg;
687 IndexReg = Displacement = SegmentReg = 0;
688 ScaleVal = 1;
689 switch (NumBytes) {
690 case 0:
691 llvm_unreachable("Zero nops?");
692 break;
693 case 1:
694 NopSize = 1;
695 Opc = X86::NOOP;
696 break;
697 case 2:
698 NopSize = 2;
699 Opc = X86::XCHG16ar;
700 break;
701 case 3:
702 NopSize = 3;
703 Opc = X86::NOOPL;
704 break;
705 case 4:
706 NopSize = 4;
707 Opc = X86::NOOPL;
708 Displacement = 8;
709 break;
710 case 5:
711 NopSize = 5;
712 Opc = X86::NOOPL;
713 Displacement = 8;
714 IndexReg = BaseReg;
715 break;
716 case 6:
717 NopSize = 6;
718 Opc = X86::NOOPW;
719 Displacement = 8;
720 IndexReg = BaseReg;
721 break;
722 case 7:
723 NopSize = 7;
724 Opc = X86::NOOPL;
725 Displacement = 512;
726 break;
727 case 8:
728 NopSize = 8;
729 Opc = X86::NOOPL;
730 Displacement = 512;
731 IndexReg = BaseReg;
732 break;
733 case 9:
734 NopSize = 9;
735 Opc = X86::NOOPW;
736 Displacement = 512;
737 IndexReg = BaseReg;
738 break;
739 default:
740 NopSize = 10;
741 Opc = X86::NOOPW;
742 Displacement = 512;
743 IndexReg = BaseReg;
744 SegmentReg = X86::CS;
745 break;
746 }
747
748 unsigned NumPrefixes = std::min(a: NumBytes - NopSize, b: 5U);
749 NopSize += NumPrefixes;
750 for (unsigned i = 0; i != NumPrefixes; ++i)
751 OS.emitBytes(Data: "\x66");
752
753 switch (Opc) {
754 default: llvm_unreachable("Unexpected opcode");
755 case X86::NOOP:
756 OS.emitInstruction(Inst: MCInstBuilder(Opc), STI: *Subtarget);
757 break;
758 case X86::XCHG16ar:
759 OS.emitInstruction(Inst: MCInstBuilder(Opc).addReg(Reg: X86::AX).addReg(Reg: X86::AX),
760 STI: *Subtarget);
761 break;
762 case X86::NOOPL:
763 case X86::NOOPW:
764 OS.emitInstruction(Inst: MCInstBuilder(Opc)
765 .addReg(Reg: BaseReg)
766 .addImm(Val: ScaleVal)
767 .addReg(Reg: IndexReg)
768 .addImm(Val: Displacement)
769 .addReg(Reg: SegmentReg),
770 STI: *Subtarget);
771 break;
772 }
773 assert(NopSize <= NumBytes && "We overemitted?");
774 return NopSize;
775}
776
777/// Emit the optimal amount of multi-byte nops on X86.
778static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
779 const X86Subtarget *Subtarget) {
780 unsigned NopsToEmit = NumBytes;
781 (void)NopsToEmit;
782 while (NumBytes) {
783 NumBytes -= emitNop(OS, NumBytes, Subtarget);
784 assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!");
785 }
786}
787
788void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
789 X86MCInstLower &MCIL) {
790 assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
791
792 NoAutoPaddingScope NoPadScope(*OutStreamer);
793
794 StatepointOpers SOpers(&MI);
795 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
796 emitX86Nops(OS&: *OutStreamer, NumBytes: PatchBytes, Subtarget);
797 } else {
798 // Lower call target and choose correct opcode
799 const MachineOperand &CallTarget = SOpers.getCallTarget();
800 MCOperand CallTargetMCOp;
801 unsigned CallOpcode;
802 switch (CallTarget.getType()) {
803 case MachineOperand::MO_GlobalAddress:
804 case MachineOperand::MO_ExternalSymbol:
805 CallTargetMCOp = MCIL.LowerSymbolOperand(
806 MO: CallTarget, Sym: MCIL.GetSymbolFromOperand(MO: CallTarget));
807 CallOpcode = X86::CALL64pcrel32;
808 // Currently, we only support relative addressing with statepoints.
809 // Otherwise, we'll need a scratch register to hold the target
810 // address. You'll fail asserts during load & relocation if this
811 // symbol is to far away. (TODO: support non-relative addressing)
812 break;
813 case MachineOperand::MO_Immediate:
814 CallTargetMCOp = MCOperand::createImm(Val: CallTarget.getImm());
815 CallOpcode = X86::CALL64pcrel32;
816 // Currently, we only support relative addressing with statepoints.
817 // Otherwise, we'll need a scratch register to hold the target
818 // immediate. You'll fail asserts during load & relocation if this
819 // address is to far away. (TODO: support non-relative addressing)
820 break;
821 case MachineOperand::MO_Register:
822 // FIXME: Add retpoline support and remove this.
823 if (Subtarget->useIndirectThunkCalls())
824 report_fatal_error(reason: "Lowering register statepoints with thunks not "
825 "yet implemented.");
826 CallTargetMCOp = MCOperand::createReg(Reg: CallTarget.getReg());
827 CallOpcode = X86::CALL64r;
828 break;
829 default:
830 llvm_unreachable("Unsupported operand type in statepoint call target");
831 break;
832 }
833
834 // Emit call
835 MCInst CallInst;
836 CallInst.setOpcode(CallOpcode);
837 CallInst.addOperand(Op: CallTargetMCOp);
838 OutStreamer->emitInstruction(Inst: CallInst, STI: getSubtargetInfo());
839 maybeEmitNopAfterCallForWindowsEH(MI: &MI);
840 }
841
842 // Record our statepoint node in the same section used by STACKMAP
843 // and PATCHPOINT
844 auto &Ctx = OutStreamer->getContext();
845 MCSymbol *MILabel = Ctx.createTempSymbol();
846 OutStreamer->emitLabel(Symbol: MILabel);
847 SM.recordStatepoint(L: *MILabel, MI);
848}
849
850void X86AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI,
851 X86MCInstLower &MCIL) {
852 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
853 // <opcode>, <operands>
854
855 NoAutoPaddingScope NoPadScope(*OutStreamer);
856
857 Register DefRegister = FaultingMI.getOperand(i: 0).getReg();
858 FaultMaps::FaultKind FK =
859 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(i: 1).getImm());
860 MCSymbol *HandlerLabel = FaultingMI.getOperand(i: 2).getMBB()->getSymbol();
861 unsigned Opcode = FaultingMI.getOperand(i: 3).getImm();
862 unsigned OperandsBeginIdx = 4;
863
864 auto &Ctx = OutStreamer->getContext();
865 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
866 OutStreamer->emitLabel(Symbol: FaultingLabel);
867
868 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
869 FM.recordFaultingOp(FaultTy: FK, FaultingLabel, HandlerLabel);
870
871 MCInst MI;
872 MI.setOpcode(Opcode);
873
874 if (DefRegister != X86::NoRegister)
875 MI.addOperand(Op: MCOperand::createReg(Reg: DefRegister));
876
877 for (const MachineOperand &MO :
878 llvm::drop_begin(RangeOrContainer: FaultingMI.operands(), N: OperandsBeginIdx))
879 if (auto Op = MCIL.LowerMachineOperand(MI: &FaultingMI, MO); Op.isValid())
880 MI.addOperand(Op);
881
882 OutStreamer->AddComment(T: "on-fault: " + HandlerLabel->getName());
883 OutStreamer->emitInstruction(Inst: MI, STI: getSubtargetInfo());
884}
885
886void X86AsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI,
887 X86MCInstLower &MCIL) {
888 bool Is64Bits = Subtarget->is64Bit();
889 MCContext &Ctx = OutStreamer->getContext();
890
891 if (MF->getFunction().hasFnAttribute(Kind: "mrecord-mcount")) {
892 MCSymbol *DotSym = OutContext.createTempSymbol();
893 OutStreamer->pushSection();
894 OutStreamer->switchSection(
895 Section: Ctx.getELFSection(Section: "__mcount_loc", Type: ELF::SHT_PROGBITS, Flags: ELF::SHF_ALLOC));
896 OutStreamer->emitSymbolValue(Sym: DotSym, Size: Is64Bits ? 8 : 4);
897 OutStreamer->popSection();
898 OutStreamer->emitLabel(Symbol: DotSym);
899 }
900
901 if (MF->getFunction().hasFnAttribute(Kind: "mnop-mcount")) {
902 // Insert a NOP stream which can be rewritten into a 5-byte CALL
903 // instruction at runtime.
904 emitX86Nops(OS&: *OutStreamer, NumBytes: 5, Subtarget);
905 return;
906 }
907
908 MCSymbol *fentry = Ctx.getOrCreateSymbol(Name: "__fentry__");
909 const MCSymbolRefExpr *Op = MCSymbolRefExpr::create(Symbol: fentry, Ctx);
910
911 EmitAndCountInstruction(
912 Inst&: MCInstBuilder(Is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32)
913 .addExpr(Val: Op));
914}
915
916void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
917 assert(std::next(MI.getIterator())->isCall() &&
918 "KCFI_CHECK not followed by a call instruction");
919
920 // Adjust the offset for patchable-function-prefix. X86InstrInfo::getNop()
921 // returns a 1-byte X86::NOOP, which means the offset is the same in
922 // bytes. This assumes that patchable-function-prefix is the same for all
923 // functions.
924 const MachineFunction &MF = *MI.getMF();
925 int64_t PrefixNops = MF.getFunction().getFnAttributeAsParsedInteger(
926 Kind: "patchable-function-prefix");
927
928 // KCFI allows indirect calls to any location that's preceded by a valid
929 // type identifier. To avoid encoding the full constant into an instruction,
930 // and thus emitting potential call target gadgets at each indirect call
931 // site, load a negated constant to a register and compare that to the
932 // expected value at the call target.
933 const Register AddrReg = MI.getOperand(i: 0).getReg();
934 const uint32_t Type = MI.getOperand(i: 1).getImm();
935 // The check is immediately before the call. If the call target is in R10,
936 // we can clobber R11 for the check instead.
937 unsigned TempReg = AddrReg == X86::R10 ? X86::R11D : X86::R10D;
938 EmitAndCountInstruction(
939 Inst&: MCInstBuilder(X86::MOV32ri).addReg(Reg: TempReg).addImm(Val: -MaskKCFIType(Value: Type)));
940 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::ADD32rm)
941 .addReg(Reg: X86::NoRegister)
942 .addReg(Reg: TempReg)
943 .addReg(Reg: AddrReg)
944 .addImm(Val: 1)
945 .addReg(Reg: X86::NoRegister)
946 .addImm(Val: -(PrefixNops + 4))
947 .addReg(Reg: X86::NoRegister));
948
949 MCSymbol *Pass = OutContext.createTempSymbol();
950 EmitAndCountInstruction(
951 Inst&: MCInstBuilder(X86::JCC_1)
952 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Pass, Ctx&: OutContext))
953 .addImm(Val: X86::COND_E));
954
955 MCSymbol *Trap = OutContext.createTempSymbol();
956 OutStreamer->emitLabel(Symbol: Trap);
957 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::TRAP));
958 emitKCFITrapEntry(MF, Symbol: Trap);
959 OutStreamer->emitLabel(Symbol: Pass);
960}
961
962void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
963 // FIXME: Make this work on non-ELF.
964 if (!TM.getTargetTriple().isOSBinFormatELF()) {
965 report_fatal_error(reason: "llvm.asan.check.memaccess only supported on ELF");
966 return;
967 }
968
969 const auto &Reg = MI.getOperand(i: 0).getReg();
970 ASanAccessInfo AccessInfo(MI.getOperand(i: 1).getImm());
971
972 uint64_t ShadowBase;
973 int MappingScale;
974 bool OrShadowOffset;
975 getAddressSanitizerParams(Ctx: MF->getFunction().getContext(),
976 TargetTriple: TM.getTargetTriple(), LongSize: 64, IsKasan: AccessInfo.CompileKernel,
977 ShadowBase: &ShadowBase, MappingScale: &MappingScale, OrShadowOffset: &OrShadowOffset);
978
979 StringRef Name = AccessInfo.IsWrite ? "store" : "load";
980 StringRef Op = OrShadowOffset ? "or" : "add";
981 std::string SymName = ("__asan_check_" + Name + "_" + Op + "_" +
982 Twine(1ULL << AccessInfo.AccessSizeIndex) + "_" +
983 TM.getMCRegisterInfo().getName(RegNo: Reg.asMCReg()))
984 .str();
985 if (OrShadowOffset)
986 report_fatal_error(
987 reason: "OrShadowOffset is not supported with optimized callbacks");
988
989 EmitAndCountInstruction(
990 Inst&: MCInstBuilder(X86::CALL64pcrel32)
991 .addExpr(Val: MCSymbolRefExpr::create(
992 Symbol: OutContext.getOrCreateSymbol(Name: SymName), Ctx&: OutContext)));
993}
994
995void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI,
996 X86MCInstLower &MCIL) {
997 // PATCHABLE_OP minsize
998
999 NoAutoPaddingScope NoPadScope(*OutStreamer);
1000
1001 auto NextMI = std::find_if(first: std::next(x: MI.getIterator()),
1002 last: MI.getParent()->end().getInstrIterator(),
1003 pred: [](auto &II) { return !II.isMetaInstruction(); });
1004
1005 SmallString<256> Code;
1006 unsigned MinSize = MI.getOperand(i: 0).getImm();
1007
1008 if (NextMI != MI.getParent()->end() && !NextMI->isInlineAsm()) {
1009 // Lower the next MachineInstr to find its byte size.
1010 // If the next instruction is inline assembly, we skip lowering it for now,
1011 // and assume we should always generate NOPs.
1012 MCInst MCI;
1013 MCIL.Lower(MI: &*NextMI, OutMI&: MCI);
1014
1015 SmallVector<MCFixup, 4> Fixups;
1016 CodeEmitter->encodeInstruction(Inst: MCI, CB&: Code, Fixups, STI: getSubtargetInfo());
1017 }
1018
1019 if (Code.size() < MinSize) {
1020 if (MinSize == 2 && Subtarget->is32Bit() &&
1021 Subtarget->isTargetWindowsMSVC() &&
1022 (Subtarget->getCPU().empty() || Subtarget->getCPU() == "pentium3")) {
1023 // For compatibility reasons, when targetting MSVC, it is important to
1024 // generate a 'legacy' NOP in the form of a 8B FF MOV EDI, EDI. Some tools
1025 // rely specifically on this pattern to be able to patch a function.
1026 // This is only for 32-bit targets, when using /arch:IA32 or /arch:SSE.
1027 OutStreamer->emitInstruction(
1028 Inst: MCInstBuilder(X86::MOV32rr_REV).addReg(Reg: X86::EDI).addReg(Reg: X86::EDI),
1029 STI: *Subtarget);
1030 } else {
1031 unsigned NopSize = emitNop(OS&: *OutStreamer, NumBytes: MinSize, Subtarget);
1032 assert(NopSize == MinSize && "Could not implement MinSize!");
1033 (void)NopSize;
1034 }
1035 }
1036}
1037
1038// Lower a stackmap of the form:
1039// <id>, <shadowBytes>, ...
1040void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
1041 SMShadowTracker.emitShadowPadding(OutStreamer&: *OutStreamer, STI: getSubtargetInfo());
1042
1043 auto &Ctx = OutStreamer->getContext();
1044 MCSymbol *MILabel = Ctx.createTempSymbol();
1045 OutStreamer->emitLabel(Symbol: MILabel);
1046
1047 SM.recordStackMap(L: *MILabel, MI);
1048 unsigned NumShadowBytes = MI.getOperand(i: 1).getImm();
1049 SMShadowTracker.reset(RequiredSize: NumShadowBytes);
1050}
1051
1052// Lower a patchpoint of the form:
1053// [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ...
1054void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
1055 X86MCInstLower &MCIL) {
1056 assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64");
1057
1058 SMShadowTracker.emitShadowPadding(OutStreamer&: *OutStreamer, STI: getSubtargetInfo());
1059
1060 NoAutoPaddingScope NoPadScope(*OutStreamer);
1061
1062 auto &Ctx = OutStreamer->getContext();
1063 MCSymbol *MILabel = Ctx.createTempSymbol();
1064 OutStreamer->emitLabel(Symbol: MILabel);
1065 SM.recordPatchPoint(L: *MILabel, MI);
1066
1067 PatchPointOpers opers(&MI);
1068 unsigned ScratchIdx = opers.getNextScratchIdx();
1069 unsigned EncodedBytes = 0;
1070 const MachineOperand &CalleeMO = opers.getCallTarget();
1071
1072 // Check for null target. If target is non-null (i.e. is non-zero or is
1073 // symbolic) then emit a call.
1074 if (!(CalleeMO.isImm() && !CalleeMO.getImm())) {
1075 MCOperand CalleeMCOp;
1076 switch (CalleeMO.getType()) {
1077 default:
1078 /// FIXME: Add a verifier check for bad callee types.
1079 llvm_unreachable("Unrecognized callee operand type.");
1080 case MachineOperand::MO_Immediate:
1081 if (CalleeMO.getImm())
1082 CalleeMCOp = MCOperand::createImm(Val: CalleeMO.getImm());
1083 break;
1084 case MachineOperand::MO_ExternalSymbol:
1085 case MachineOperand::MO_GlobalAddress:
1086 CalleeMCOp = MCIL.LowerSymbolOperand(MO: CalleeMO,
1087 Sym: MCIL.GetSymbolFromOperand(MO: CalleeMO));
1088 break;
1089 }
1090
1091 // Emit MOV to materialize the target address and the CALL to target.
1092 // This is encoded with 12-13 bytes, depending on which register is used.
1093 Register ScratchReg = MI.getOperand(i: ScratchIdx).getReg();
1094 if (X86II::isX86_64ExtendedReg(Reg: ScratchReg))
1095 EncodedBytes = 13;
1096 else
1097 EncodedBytes = 12;
1098
1099 EmitAndCountInstruction(
1100 Inst&: MCInstBuilder(X86::MOV64ri).addReg(Reg: ScratchReg).addOperand(Op: CalleeMCOp));
1101 // FIXME: Add retpoline support and remove this.
1102 if (Subtarget->useIndirectThunkCalls())
1103 report_fatal_error(
1104 reason: "Lowering patchpoint with thunks not yet implemented.");
1105 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64r).addReg(Reg: ScratchReg));
1106 }
1107
1108 // Emit padding.
1109 unsigned NumBytes = opers.getNumPatchBytes();
1110 assert(NumBytes >= EncodedBytes &&
1111 "Patchpoint can't request size less than the length of a call.");
1112
1113 emitX86Nops(OS&: *OutStreamer, NumBytes: NumBytes - EncodedBytes, Subtarget);
1114}
1115
1116void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
1117 X86MCInstLower &MCIL) {
1118 assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64");
1119
1120 NoAutoPaddingScope NoPadScope(*OutStreamer);
1121
1122 // We want to emit the following pattern, which follows the x86 calling
1123 // convention to prepare for the trampoline call to be patched in.
1124 //
1125 // .p2align 1, ...
1126 // .Lxray_event_sled_N:
1127 // jmp +N // jump across the instrumentation sled
1128 // ... // set up arguments in register
1129 // callq __xray_CustomEvent@plt // force dependency to symbol
1130 // ...
1131 // <jump here>
1132 //
1133 // After patching, it would look something like:
1134 //
1135 // nopw (2-byte nop)
1136 // ...
1137 // callq __xrayCustomEvent // already lowered
1138 // ...
1139 //
1140 // ---
1141 // First we emit the label and the jump.
1142 auto CurSled = OutContext.createTempSymbol(Name: "xray_event_sled_", AlwaysAddSuffix: true);
1143 OutStreamer->AddComment(T: "# XRay Custom Event Log");
1144 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1145 OutStreamer->emitLabel(Symbol: CurSled);
1146
1147 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1148 // an operand (computed as an offset from the jmp instruction).
1149 // FIXME: Find another less hacky way do force the relative jump.
1150 OutStreamer->emitBinaryData(Data: "\xeb\x0f");
1151
1152 // The default C calling convention will place two arguments into %rcx and
1153 // %rdx -- so we only work with those.
1154 const Register DestRegs[] = {X86::RDI, X86::RSI};
1155 bool UsedMask[] = {false, false};
1156 // Filled out in loop.
1157 Register SrcRegs[] = {0, 0};
1158
1159 // Then we put the operands in the %rdi and %rsi registers. We spill the
1160 // values in the register before we clobber them, and mark them as used in
1161 // UsedMask. In case the arguments are already in the correct register, we use
1162 // emit nops appropriately sized to keep the sled the same size in every
1163 // situation.
1164 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1165 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO: MI.getOperand(i: I));
1166 Op.isValid()) {
1167 assert(Op.isReg() && "Only support arguments in registers");
1168 SrcRegs[I] = getX86SubSuperRegister(Reg: Op.getReg(), Size: 64);
1169 assert(SrcRegs[I].isValid() && "Invalid operand");
1170 if (SrcRegs[I] != DestRegs[I]) {
1171 UsedMask[I] = true;
1172 EmitAndCountInstruction(
1173 Inst&: MCInstBuilder(X86::PUSH64r).addReg(Reg: DestRegs[I]));
1174 } else {
1175 emitX86Nops(OS&: *OutStreamer, NumBytes: 4, Subtarget);
1176 }
1177 }
1178
1179 // Now that the register values are stashed, mov arguments into place.
1180 // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1181 // earlier DestReg. We will have already overwritten over the register before
1182 // we can copy from it.
1183 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1184 if (SrcRegs[I] != DestRegs[I])
1185 EmitAndCountInstruction(
1186 Inst&: MCInstBuilder(X86::MOV64rr).addReg(Reg: DestRegs[I]).addReg(Reg: SrcRegs[I]));
1187
1188 // We emit a hard dependency on the __xray_CustomEvent symbol, which is the
1189 // name of the trampoline to be implemented by the XRay runtime.
1190 auto TSym = OutContext.getOrCreateSymbol(Name: "__xray_CustomEvent");
1191 MachineOperand TOp = MachineOperand::CreateMCSymbol(Sym: TSym);
1192 if (isPositionIndependent())
1193 TOp.setTargetFlags(X86II::MO_PLT);
1194
1195 // Emit the call instruction.
1196 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64pcrel32)
1197 .addOperand(Op: MCIL.LowerSymbolOperand(MO: TOp, Sym: TSym)));
1198
1199 // Restore caller-saved and used registers.
1200 for (unsigned I = sizeof UsedMask; I-- > 0;)
1201 if (UsedMask[I])
1202 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::POP64r).addReg(Reg: DestRegs[I]));
1203 else
1204 emitX86Nops(OS&: *OutStreamer, NumBytes: 1, Subtarget);
1205
1206 OutStreamer->AddComment(T: "xray custom event end.");
1207
1208 // Record the sled version. Version 0 of this sled was spelled differently, so
1209 // we let the runtime handle the different offsets we're using. Version 2
1210 // changed the absolute address to a PC-relative address.
1211 recordSled(Sled: CurSled, MI, Kind: SledKind::CUSTOM_EVENT, Version: 2);
1212}
1213
1214void X86AsmPrinter::LowerPATCHABLE_TYPED_EVENT_CALL(const MachineInstr &MI,
1215 X86MCInstLower &MCIL) {
1216 assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64");
1217
1218 NoAutoPaddingScope NoPadScope(*OutStreamer);
1219
1220 // We want to emit the following pattern, which follows the x86 calling
1221 // convention to prepare for the trampoline call to be patched in.
1222 //
1223 // .p2align 1, ...
1224 // .Lxray_event_sled_N:
1225 // jmp +N // jump across the instrumentation sled
1226 // ... // set up arguments in register
1227 // callq __xray_TypedEvent@plt // force dependency to symbol
1228 // ...
1229 // <jump here>
1230 //
1231 // After patching, it would look something like:
1232 //
1233 // nopw (2-byte nop)
1234 // ...
1235 // callq __xrayTypedEvent // already lowered
1236 // ...
1237 //
1238 // ---
1239 // First we emit the label and the jump.
1240 auto CurSled = OutContext.createTempSymbol(Name: "xray_typed_event_sled_", AlwaysAddSuffix: true);
1241 OutStreamer->AddComment(T: "# XRay Typed Event Log");
1242 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1243 OutStreamer->emitLabel(Symbol: CurSled);
1244
1245 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1246 // an operand (computed as an offset from the jmp instruction).
1247 // FIXME: Find another less hacky way do force the relative jump.
1248 OutStreamer->emitBinaryData(Data: "\xeb\x14");
1249
1250 // An x86-64 convention may place three arguments into %rcx, %rdx, and R8,
1251 // so we'll work with those. Or we may be called via SystemV, in which case
1252 // we don't have to do any translation.
1253 const Register DestRegs[] = {X86::RDI, X86::RSI, X86::RDX};
1254 bool UsedMask[] = {false, false, false};
1255
1256 // Will fill out src regs in the loop.
1257 Register SrcRegs[] = {0, 0, 0};
1258
1259 // Then we put the operands in the SystemV registers. We spill the values in
1260 // the registers before we clobber them, and mark them as used in UsedMask.
1261 // In case the arguments are already in the correct register, we emit nops
1262 // appropriately sized to keep the sled the same size in every situation.
1263 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1264 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO: MI.getOperand(i: I));
1265 Op.isValid()) {
1266 // TODO: Is register only support adequate?
1267 assert(Op.isReg() && "Only supports arguments in registers");
1268 SrcRegs[I] = getX86SubSuperRegister(Reg: Op.getReg(), Size: 64);
1269 assert(SrcRegs[I].isValid() && "Invalid operand");
1270 if (SrcRegs[I] != DestRegs[I]) {
1271 UsedMask[I] = true;
1272 EmitAndCountInstruction(
1273 Inst&: MCInstBuilder(X86::PUSH64r).addReg(Reg: DestRegs[I]));
1274 } else {
1275 emitX86Nops(OS&: *OutStreamer, NumBytes: 4, Subtarget);
1276 }
1277 }
1278
1279 // In the above loop we only stash all of the destination registers or emit
1280 // nops if the arguments are already in the right place. Doing the actually
1281 // moving is postponed until after all the registers are stashed so nothing
1282 // is clobbers. We've already added nops to account for the size of mov and
1283 // push if the register is in the right place, so we only have to worry about
1284 // emitting movs.
1285 // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1286 // earlier DestReg. We will have already overwritten over the register before
1287 // we can copy from it.
1288 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1289 if (UsedMask[I])
1290 EmitAndCountInstruction(
1291 Inst&: MCInstBuilder(X86::MOV64rr).addReg(Reg: DestRegs[I]).addReg(Reg: SrcRegs[I]));
1292
1293 // We emit a hard dependency on the __xray_TypedEvent symbol, which is the
1294 // name of the trampoline to be implemented by the XRay runtime.
1295 auto TSym = OutContext.getOrCreateSymbol(Name: "__xray_TypedEvent");
1296 MachineOperand TOp = MachineOperand::CreateMCSymbol(Sym: TSym);
1297 if (isPositionIndependent())
1298 TOp.setTargetFlags(X86II::MO_PLT);
1299
1300 // Emit the call instruction.
1301 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CALL64pcrel32)
1302 .addOperand(Op: MCIL.LowerSymbolOperand(MO: TOp, Sym: TSym)));
1303
1304 // Restore caller-saved and used registers.
1305 for (unsigned I = sizeof UsedMask; I-- > 0;)
1306 if (UsedMask[I])
1307 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::POP64r).addReg(Reg: DestRegs[I]));
1308 else
1309 emitX86Nops(OS&: *OutStreamer, NumBytes: 1, Subtarget);
1310
1311 OutStreamer->AddComment(T: "xray typed event end.");
1312
1313 // Record the sled version.
1314 recordSled(Sled: CurSled, MI, Kind: SledKind::TYPED_EVENT, Version: 2);
1315}
1316
1317void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
1318 X86MCInstLower &MCIL) {
1319
1320 NoAutoPaddingScope NoPadScope(*OutStreamer);
1321
1322 const Function &F = MF->getFunction();
1323 if (F.hasFnAttribute(Kind: "patchable-function-entry")) {
1324 unsigned Num = F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry");
1325 emitX86Nops(OS&: *OutStreamer, NumBytes: Num, Subtarget);
1326 return;
1327 }
1328 // We want to emit the following pattern:
1329 //
1330 // .p2align 1, ...
1331 // .Lxray_sled_N:
1332 // jmp .tmpN
1333 // # 9 bytes worth of noops
1334 //
1335 // We need the 9 bytes because at runtime, we'd be patching over the full 11
1336 // bytes with the following pattern:
1337 //
1338 // mov %r10, <function id, 32-bit> // 6 bytes
1339 // call <relative offset, 32-bits> // 5 bytes
1340 //
1341 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
1342 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1343 OutStreamer->emitLabel(Symbol: CurSled);
1344
1345 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1346 // an operand (computed as an offset from the jmp instruction).
1347 // FIXME: Find another less hacky way do force the relative jump.
1348 OutStreamer->emitBytes(Data: "\xeb\x09");
1349 emitX86Nops(OS&: *OutStreamer, NumBytes: 9, Subtarget);
1350 recordSled(Sled: CurSled, MI, Kind: SledKind::FUNCTION_ENTER, Version: 2);
1351}
1352
1353void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
1354 X86MCInstLower &MCIL) {
1355 NoAutoPaddingScope NoPadScope(*OutStreamer);
1356
1357 // Since PATCHABLE_RET takes the opcode of the return statement as an
1358 // argument, we use that to emit the correct form of the RET that we want.
1359 // i.e. when we see this:
1360 //
1361 // PATCHABLE_RET X86::RET ...
1362 //
1363 // We should emit the RET followed by sleds.
1364 //
1365 // .p2align 1, ...
1366 // .Lxray_sled_N:
1367 // ret # or equivalent instruction
1368 // # 10 bytes worth of noops
1369 //
1370 // This just makes sure that the alignment for the next instruction is 2.
1371 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
1372 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1373 OutStreamer->emitLabel(Symbol: CurSled);
1374 unsigned OpCode = MI.getOperand(i: 0).getImm();
1375 MCInst Ret;
1376 Ret.setOpcode(OpCode);
1377 for (auto &MO : drop_begin(RangeOrContainer: MI.operands()))
1378 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO); Op.isValid())
1379 Ret.addOperand(Op);
1380 OutStreamer->emitInstruction(Inst: Ret, STI: getSubtargetInfo());
1381 emitX86Nops(OS&: *OutStreamer, NumBytes: 10, Subtarget);
1382 recordSled(Sled: CurSled, MI, Kind: SledKind::FUNCTION_EXIT, Version: 2);
1383}
1384
1385void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI,
1386 X86MCInstLower &MCIL) {
1387 MCInst TC;
1388 TC.setOpcode(convertTailJumpOpcode(Opcode: MI.getOperand(i: 0).getImm()));
1389 // Drop the tail jump opcode.
1390 auto TCOperands = drop_begin(RangeOrContainer: MI.operands());
1391 bool IsConditional = TC.getOpcode() == X86::JCC_1;
1392 MCSymbol *FallthroughLabel;
1393 if (IsConditional) {
1394 // Rewrite:
1395 // je target
1396 //
1397 // To:
1398 // jne .fallthrough
1399 // .p2align 1, ...
1400 // .Lxray_sled_N:
1401 // SLED_CODE
1402 // jmp target
1403 // .fallthrough:
1404 FallthroughLabel = OutContext.createTempSymbol();
1405 EmitToStreamer(
1406 S&: *OutStreamer,
1407 Inst: MCInstBuilder(X86::JCC_1)
1408 .addExpr(Val: MCSymbolRefExpr::create(Symbol: FallthroughLabel, Ctx&: OutContext))
1409 .addImm(Val: X86::GetOppositeBranchCondition(
1410 CC: static_cast<X86::CondCode>(MI.getOperand(i: 2).getImm()))));
1411 TC.setOpcode(X86::JMP_1);
1412 // Drop the condition code.
1413 TCOperands = drop_end(RangeOrContainer&: TCOperands);
1414 }
1415
1416 NoAutoPaddingScope NoPadScope(*OutStreamer);
1417
1418 // Like PATCHABLE_RET, we have the actual instruction in the operands to this
1419 // instruction so we lower that particular instruction and its operands.
1420 // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how
1421 // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to
1422 // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual
1423 // tail call much like how we have it in PATCHABLE_RET.
1424 auto CurSled = OutContext.createTempSymbol(Name: "xray_sled_", AlwaysAddSuffix: true);
1425 OutStreamer->emitCodeAlignment(Alignment: Align(2), STI: getSubtargetInfo());
1426 OutStreamer->emitLabel(Symbol: CurSled);
1427 auto Target = OutContext.createTempSymbol();
1428
1429 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1430 // an operand (computed as an offset from the jmp instruction).
1431 // FIXME: Find another less hacky way do force the relative jump.
1432 OutStreamer->emitBytes(Data: "\xeb\x09");
1433 emitX86Nops(OS&: *OutStreamer, NumBytes: 9, Subtarget);
1434 OutStreamer->emitLabel(Symbol: Target);
1435 recordSled(Sled: CurSled, MI, Kind: SledKind::TAIL_CALL, Version: 2);
1436
1437 // Before emitting the instruction, add a comment to indicate that this is
1438 // indeed a tail call.
1439 OutStreamer->AddComment(T: "TAILCALL");
1440 for (auto &MO : TCOperands)
1441 if (auto Op = MCIL.LowerMachineOperand(MI: &MI, MO); Op.isValid())
1442 TC.addOperand(Op);
1443 OutStreamer->emitInstruction(Inst: TC, STI: getSubtargetInfo());
1444
1445 if (IsConditional)
1446 OutStreamer->emitLabel(Symbol: FallthroughLabel);
1447}
1448
1449static unsigned getSrcIdx(const MachineInstr* MI, unsigned SrcIdx) {
1450 if (X86II::isKMasked(TSFlags: MI->getDesc().TSFlags)) {
1451 // Skip mask operand.
1452 ++SrcIdx;
1453 if (X86II::isKMergeMasked(TSFlags: MI->getDesc().TSFlags)) {
1454 // Skip passthru operand.
1455 ++SrcIdx;
1456 }
1457 }
1458 return SrcIdx;
1459}
1460
1461static void printDstRegisterName(raw_ostream &CS, const MachineInstr *MI,
1462 unsigned SrcOpIdx) {
1463 const MachineOperand &DstOp = MI->getOperand(i: 0);
1464 CS << X86ATTInstPrinter::getRegisterName(Reg: DstOp.getReg());
1465
1466 // Handle AVX512 MASK/MASXZ write mask comments.
1467 // MASK: zmmX {%kY}
1468 // MASKZ: zmmX {%kY} {z}
1469 if (X86II::isKMasked(TSFlags: MI->getDesc().TSFlags)) {
1470 const MachineOperand &WriteMaskOp = MI->getOperand(i: SrcOpIdx - 1);
1471 StringRef Mask = X86ATTInstPrinter::getRegisterName(Reg: WriteMaskOp.getReg());
1472 CS << " {%" << Mask << "}";
1473 if (!X86II::isKMergeMasked(TSFlags: MI->getDesc().TSFlags)) {
1474 CS << " {z}";
1475 }
1476 }
1477}
1478
1479static void printShuffleMask(raw_ostream &CS, StringRef Src1Name,
1480 StringRef Src2Name, ArrayRef<int> Mask) {
1481 // One source operand, fix the mask to print all elements in one span.
1482 SmallVector<int, 8> ShuffleMask(Mask);
1483 if (Src1Name == Src2Name)
1484 for (int i = 0, e = ShuffleMask.size(); i != e; ++i)
1485 if (ShuffleMask[i] >= e)
1486 ShuffleMask[i] -= e;
1487
1488 for (int i = 0, e = ShuffleMask.size(); i != e; ++i) {
1489 if (i != 0)
1490 CS << ",";
1491 if (ShuffleMask[i] == SM_SentinelZero) {
1492 CS << "zero";
1493 continue;
1494 }
1495
1496 // Otherwise, it must come from src1 or src2. Print the span of elements
1497 // that comes from this src.
1498 bool isSrc1 = ShuffleMask[i] < (int)e;
1499 CS << (isSrc1 ? Src1Name : Src2Name) << '[';
1500
1501 bool IsFirst = true;
1502 while (i != e && ShuffleMask[i] != SM_SentinelZero &&
1503 (ShuffleMask[i] < (int)e) == isSrc1) {
1504 if (!IsFirst)
1505 CS << ',';
1506 else
1507 IsFirst = false;
1508 if (ShuffleMask[i] == SM_SentinelUndef)
1509 CS << "u";
1510 else
1511 CS << ShuffleMask[i] % (int)e;
1512 ++i;
1513 }
1514 CS << ']';
1515 --i; // For loop increments element #.
1516 }
1517}
1518
1519static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx,
1520 unsigned SrcOp2Idx, ArrayRef<int> Mask) {
1521 std::string Comment;
1522
1523 const MachineOperand &SrcOp1 = MI->getOperand(i: SrcOp1Idx);
1524 const MachineOperand &SrcOp2 = MI->getOperand(i: SrcOp2Idx);
1525 StringRef Src1Name = SrcOp1.isReg()
1526 ? X86ATTInstPrinter::getRegisterName(Reg: SrcOp1.getReg())
1527 : "mem";
1528 StringRef Src2Name = SrcOp2.isReg()
1529 ? X86ATTInstPrinter::getRegisterName(Reg: SrcOp2.getReg())
1530 : "mem";
1531
1532 raw_string_ostream CS(Comment);
1533 printDstRegisterName(CS, MI, SrcOpIdx: SrcOp1Idx);
1534 CS << " = ";
1535 printShuffleMask(CS, Src1Name, Src2Name, Mask);
1536
1537 return Comment;
1538}
1539
1540static void printConstant(const APInt &Val, raw_ostream &CS,
1541 bool PrintZero = false) {
1542 if (Val.getBitWidth() <= 64) {
1543 CS << (PrintZero ? 0ULL : Val.getZExtValue());
1544 } else {
1545 // print multi-word constant as (w0,w1)
1546 CS << "(";
1547 for (int i = 0, N = Val.getNumWords(); i < N; ++i) {
1548 if (i > 0)
1549 CS << ",";
1550 CS << (PrintZero ? 0ULL : Val.getRawData()[i]);
1551 }
1552 CS << ")";
1553 }
1554}
1555
1556static void printConstant(const APFloat &Flt, raw_ostream &CS,
1557 bool PrintZero = false) {
1558 SmallString<32> Str;
1559 // Force scientific notation to distinguish from integers.
1560 if (PrintZero)
1561 APFloat::getZero(Sem: Flt.getSemantics()).toString(Str, FormatPrecision: 0, FormatMaxPadding: 0);
1562 else
1563 Flt.toString(Str, FormatPrecision: 0, FormatMaxPadding: 0);
1564 CS << Str;
1565}
1566
1567static void printConstant(const Constant *COp, unsigned BitWidth,
1568 raw_ostream &CS, bool PrintZero = false) {
1569 if (isa<UndefValue>(Val: COp)) {
1570 CS << "u";
1571 } else if (auto *CI = dyn_cast<ConstantInt>(Val: COp)) {
1572 if (auto VTy = dyn_cast<FixedVectorType>(Val: CI->getType())) {
1573 for (unsigned I = 0, E = VTy->getNumElements(); I != E; ++I) {
1574 if (I != 0)
1575 CS << ',';
1576 printConstant(Val: CI->getValue(), CS, PrintZero);
1577 }
1578 } else
1579 printConstant(Val: CI->getValue(), CS, PrintZero);
1580 } else if (auto *CF = dyn_cast<ConstantFP>(Val: COp)) {
1581 if (auto VTy = dyn_cast<FixedVectorType>(Val: CF->getType())) {
1582 unsigned EltBits = VTy->getScalarSizeInBits();
1583 unsigned E = std::min(a: BitWidth / EltBits, b: VTy->getNumElements());
1584 if ((BitWidth % EltBits) == 0) {
1585 for (unsigned I = 0; I != E; ++I) {
1586 if (I != 0)
1587 CS << ",";
1588 printConstant(Flt: CF->getValueAPF(), CS, PrintZero);
1589 }
1590 } else {
1591 CS << "?";
1592 }
1593 } else
1594 printConstant(Flt: CF->getValueAPF(), CS, PrintZero);
1595 } else if (auto *CDS = dyn_cast<ConstantDataSequential>(Val: COp)) {
1596 Type *EltTy = CDS->getElementType();
1597 bool IsInteger = EltTy->isIntegerTy();
1598 bool IsFP = EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy();
1599 unsigned EltBits = EltTy->getPrimitiveSizeInBits();
1600 unsigned E = std::min(a: BitWidth / EltBits, b: (unsigned)CDS->getNumElements());
1601 if ((BitWidth % EltBits) == 0) {
1602 for (unsigned I = 0; I != E; ++I) {
1603 if (I != 0)
1604 CS << ",";
1605 if (IsInteger)
1606 printConstant(Val: CDS->getElementAsAPInt(i: I), CS, PrintZero);
1607 else if (IsFP)
1608 printConstant(Flt: CDS->getElementAsAPFloat(i: I), CS, PrintZero);
1609 else
1610 CS << "?";
1611 }
1612 } else {
1613 CS << "?";
1614 }
1615 } else if (auto *CV = dyn_cast<ConstantVector>(Val: COp)) {
1616 unsigned EltBits = CV->getType()->getScalarSizeInBits();
1617 unsigned E = std::min(a: BitWidth / EltBits, b: CV->getNumOperands());
1618 if ((BitWidth % EltBits) == 0) {
1619 for (unsigned I = 0; I != E; ++I) {
1620 if (I != 0)
1621 CS << ",";
1622 printConstant(COp: CV->getOperand(i_nocapture: I), BitWidth: EltBits, CS, PrintZero);
1623 }
1624 } else {
1625 CS << "?";
1626 }
1627 } else {
1628 CS << "?";
1629 }
1630}
1631
1632static void printZeroUpperMove(const MachineInstr *MI, MCStreamer &OutStreamer,
1633 int SclWidth, int VecWidth,
1634 const char *ShuffleComment) {
1635 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1636
1637 std::string Comment;
1638 raw_string_ostream CS(Comment);
1639 printDstRegisterName(CS, MI, SrcOpIdx: SrcIdx);
1640 CS << " = ";
1641
1642 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx)) {
1643 CS << "[";
1644 printConstant(COp: C, BitWidth: SclWidth, CS);
1645 for (int I = 1, E = VecWidth / SclWidth; I < E; ++I) {
1646 CS << ",";
1647 printConstant(COp: C, BitWidth: SclWidth, CS, PrintZero: true);
1648 }
1649 CS << "]";
1650 OutStreamer.AddComment(T: CS.str());
1651 return; // early-out
1652 }
1653
1654 // We didn't find a constant load, fallback to a shuffle mask decode.
1655 CS << ShuffleComment;
1656 OutStreamer.AddComment(T: CS.str());
1657}
1658
1659static void printBroadcast(const MachineInstr *MI, MCStreamer &OutStreamer,
1660 int Repeats, int BitWidth) {
1661 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1662 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx)) {
1663 std::string Comment;
1664 raw_string_ostream CS(Comment);
1665 printDstRegisterName(CS, MI, SrcOpIdx: SrcIdx);
1666 CS << " = [";
1667 for (int l = 0; l != Repeats; ++l) {
1668 if (l != 0)
1669 CS << ",";
1670 printConstant(COp: C, BitWidth, CS);
1671 }
1672 CS << "]";
1673 OutStreamer.AddComment(T: CS.str());
1674 }
1675}
1676
1677static void addConstantComment(const MachineInstr *MI, MCStreamer &OutStreamer,
1678 unsigned OpNo, int BitWidth, int Repeats = 1) {
1679 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo)) {
1680 std::string Comment;
1681 raw_string_ostream CS(Comment);
1682 CS << "[";
1683 for (int I = 0; I != Repeats; ++I) {
1684 if (I != 0)
1685 CS << ",";
1686 printConstant(COp: C, BitWidth, CS);
1687 }
1688 CS << "]";
1689 OutStreamer.AddComment(T: CS.str());
1690 }
1691}
1692
1693static bool printExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1694 int SrcEltBits, int DstEltBits, bool IsSext) {
1695 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1696 auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx);
1697 if (C && C->getType()->getScalarSizeInBits() == unsigned(SrcEltBits)) {
1698 if (auto *CDS = dyn_cast<ConstantDataSequential>(Val: C)) {
1699 int NumElts = CDS->getNumElements();
1700 std::string Comment;
1701 raw_string_ostream CS(Comment);
1702 printDstRegisterName(CS, MI, SrcOpIdx: SrcIdx);
1703 CS << " = [";
1704 for (int i = 0; i != NumElts; ++i) {
1705 if (i != 0)
1706 CS << ",";
1707 if (CDS->getElementType()->isIntegerTy()) {
1708 APInt Elt = CDS->getElementAsAPInt(i);
1709 Elt = IsSext ? Elt.sext(width: DstEltBits) : Elt.zext(width: DstEltBits);
1710 printConstant(Val: Elt, CS);
1711 } else
1712 CS << "?";
1713 }
1714 CS << "]";
1715 OutStreamer.AddComment(T: CS.str());
1716 return true;
1717 }
1718 }
1719
1720 return false;
1721}
1722static void printSignExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1723 int SrcEltBits, int DstEltBits) {
1724 printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, IsSext: true);
1725}
1726static void printZeroExtend(const MachineInstr *MI, MCStreamer &OutStreamer,
1727 int SrcEltBits, int DstEltBits) {
1728 if (printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, IsSext: false))
1729 return;
1730
1731 // We didn't find a constant load, fallback to a shuffle mask decode.
1732 std::string Comment;
1733 raw_string_ostream CS(Comment);
1734 printDstRegisterName(CS, MI, SrcOpIdx: getSrcIdx(MI, SrcIdx: 1));
1735 CS << " = ";
1736
1737 SmallVector<int> Mask;
1738 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1739 assert((Width % DstEltBits) == 0 && (DstEltBits % SrcEltBits) == 0 &&
1740 "Illegal extension ratio");
1741 DecodeZeroExtendMask(SrcScalarBits: SrcEltBits, DstScalarBits: DstEltBits, NumDstElts: Width / DstEltBits, IsAnyExtend: false, ShuffleMask&: Mask);
1742 printShuffleMask(CS, Src1Name: "mem", Src2Name: "", Mask);
1743
1744 OutStreamer.AddComment(T: CS.str());
1745}
1746
1747void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) {
1748 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
1749 assert(getSubtarget().isOSWindowsOrUEFI() &&
1750 "SEH_ instruction Windows and UEFI only");
1751
1752 // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86.
1753 if (EmitFPOData) {
1754 X86TargetStreamer *XTS =
1755 static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer());
1756 switch (MI->getOpcode()) {
1757 case X86::SEH_PushReg:
1758 XTS->emitFPOPushReg(Reg: MI->getOperand(i: 0).getImm());
1759 break;
1760 case X86::SEH_StackAlloc:
1761 XTS->emitFPOStackAlloc(StackAlloc: MI->getOperand(i: 0).getImm());
1762 break;
1763 case X86::SEH_StackAlign:
1764 XTS->emitFPOStackAlign(Align: MI->getOperand(i: 0).getImm());
1765 break;
1766 case X86::SEH_SetFrame:
1767 assert(MI->getOperand(1).getImm() == 0 &&
1768 ".cv_fpo_setframe takes no offset");
1769 XTS->emitFPOSetFrame(Reg: MI->getOperand(i: 0).getImm());
1770 break;
1771 case X86::SEH_EndPrologue:
1772 XTS->emitFPOEndPrologue();
1773 break;
1774 case X86::SEH_SaveReg:
1775 case X86::SEH_SaveXMM:
1776 case X86::SEH_PushFrame:
1777 case X86::SEH_Push2Regs:
1778 llvm_unreachable("SEH_ directive incompatible with FPO");
1779 break;
1780 default:
1781 llvm_unreachable("expected SEH_ instruction");
1782 }
1783 return;
1784 }
1785
1786 // Otherwise, use the .seh_ directives for all other Windows platforms.
1787 switch (MI->getOpcode()) {
1788 case X86::SEH_PushReg:
1789 OutStreamer->emitWinCFIPushReg(Register: MI->getOperand(i: 0).getImm());
1790 break;
1791
1792 case X86::SEH_Push2Regs:
1793 OutStreamer->emitWinCFIPush2Regs(Reg1: MI->getOperand(i: 0).getImm(),
1794 Reg2: MI->getOperand(i: 1).getImm());
1795 break;
1796
1797 case X86::SEH_SaveReg:
1798 OutStreamer->emitWinCFISaveReg(Register: MI->getOperand(i: 0).getImm(),
1799 Offset: MI->getOperand(i: 1).getImm());
1800 break;
1801
1802 case X86::SEH_SaveXMM:
1803 OutStreamer->emitWinCFISaveXMM(Register: MI->getOperand(i: 0).getImm(),
1804 Offset: MI->getOperand(i: 1).getImm());
1805 break;
1806
1807 case X86::SEH_StackAlloc:
1808 OutStreamer->emitWinCFIAllocStack(Size: MI->getOperand(i: 0).getImm());
1809 break;
1810
1811 case X86::SEH_SetFrame:
1812 OutStreamer->emitWinCFISetFrame(Register: MI->getOperand(i: 0).getImm(),
1813 Offset: MI->getOperand(i: 1).getImm());
1814 break;
1815
1816 case X86::SEH_PushFrame:
1817 OutStreamer->emitWinCFIPushFrame(Code: MI->getOperand(i: 0).getImm());
1818 break;
1819
1820 case X86::SEH_EndPrologue:
1821 OutStreamer->emitWinCFIEndProlog();
1822 break;
1823
1824 case X86::SEH_BeginEpilogue:
1825 OutStreamer->emitWinCFIBeginEpilogue();
1826 break;
1827
1828 case X86::SEH_EndEpilogue:
1829 OutStreamer->emitWinCFIEndEpilogue();
1830 break;
1831
1832 case X86::SEH_UnwindV2Start:
1833 OutStreamer->emitWinCFIUnwindV2Start();
1834 break;
1835
1836 case X86::SEH_UnwindVersion:
1837 OutStreamer->emitWinCFIUnwindVersion(Version: MI->getOperand(i: 0).getImm());
1838 break;
1839
1840 default:
1841 llvm_unreachable("expected SEH_ instruction");
1842 }
1843}
1844
1845static void addConstantComments(const MachineInstr *MI,
1846 MCStreamer &OutStreamer) {
1847 switch (MI->getOpcode()) {
1848 // Lower PSHUFB and VPERMILP normally but add a comment if we can find
1849 // a constant shuffle mask. We won't be able to do this at the MC layer
1850 // because the mask isn't an immediate.
1851 case X86::PSHUFBrm:
1852 case X86::VPSHUFBrm:
1853 case X86::VPSHUFBYrm:
1854 case X86::VPSHUFBZ128rm:
1855 case X86::VPSHUFBZ128rmk:
1856 case X86::VPSHUFBZ128rmkz:
1857 case X86::VPSHUFBZ256rm:
1858 case X86::VPSHUFBZ256rmk:
1859 case X86::VPSHUFBZ256rmkz:
1860 case X86::VPSHUFBZrm:
1861 case X86::VPSHUFBZrmk:
1862 case X86::VPSHUFBZrmkz: {
1863 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1864 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx + 1)) {
1865 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1866 SmallVector<int, 64> Mask;
1867 DecodePSHUFBMask(C, Width, ShuffleMask&: Mask);
1868 if (!Mask.empty())
1869 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: SrcIdx, SrcOp2Idx: SrcIdx, Mask));
1870 }
1871 break;
1872 }
1873
1874 case X86::VPERMILPSrm:
1875 case X86::VPERMILPSYrm:
1876 case X86::VPERMILPSZ128rm:
1877 case X86::VPERMILPSZ128rmk:
1878 case X86::VPERMILPSZ128rmkz:
1879 case X86::VPERMILPSZ256rm:
1880 case X86::VPERMILPSZ256rmk:
1881 case X86::VPERMILPSZ256rmkz:
1882 case X86::VPERMILPSZrm:
1883 case X86::VPERMILPSZrmk:
1884 case X86::VPERMILPSZrmkz: {
1885 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1886 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx + 1)) {
1887 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1888 SmallVector<int, 16> Mask;
1889 DecodeVPERMILPMask(C, ElSize: 32, Width, ShuffleMask&: Mask);
1890 if (!Mask.empty())
1891 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: SrcIdx, SrcOp2Idx: SrcIdx, Mask));
1892 }
1893 break;
1894 }
1895 case X86::VPERMILPDrm:
1896 case X86::VPERMILPDYrm:
1897 case X86::VPERMILPDZ128rm:
1898 case X86::VPERMILPDZ128rmk:
1899 case X86::VPERMILPDZ128rmkz:
1900 case X86::VPERMILPDZ256rm:
1901 case X86::VPERMILPDZ256rmk:
1902 case X86::VPERMILPDZ256rmkz:
1903 case X86::VPERMILPDZrm:
1904 case X86::VPERMILPDZrmk:
1905 case X86::VPERMILPDZrmkz: {
1906 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1907 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: SrcIdx + 1)) {
1908 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1909 SmallVector<int, 16> Mask;
1910 DecodeVPERMILPMask(C, ElSize: 64, Width, ShuffleMask&: Mask);
1911 if (!Mask.empty())
1912 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: SrcIdx, SrcOp2Idx: SrcIdx, Mask));
1913 }
1914 break;
1915 }
1916
1917 case X86::VPERMIL2PDrm:
1918 case X86::VPERMIL2PSrm:
1919 case X86::VPERMIL2PDYrm:
1920 case X86::VPERMIL2PSYrm: {
1921 assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands + 1) &&
1922 "Unexpected number of operands!");
1923
1924 const MachineOperand &CtrlOp = MI->getOperand(i: MI->getNumOperands() - 1);
1925 if (!CtrlOp.isImm())
1926 break;
1927
1928 unsigned ElSize;
1929 switch (MI->getOpcode()) {
1930 default: llvm_unreachable("Invalid opcode");
1931 case X86::VPERMIL2PSrm: case X86::VPERMIL2PSYrm: ElSize = 32; break;
1932 case X86::VPERMIL2PDrm: case X86::VPERMIL2PDYrm: ElSize = 64; break;
1933 }
1934
1935 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: 3)) {
1936 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1937 SmallVector<int, 16> Mask;
1938 DecodeVPERMIL2PMask(C, M2Z: (unsigned)CtrlOp.getImm(), ElSize, Width, ShuffleMask&: Mask);
1939 if (!Mask.empty())
1940 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: 1, SrcOp2Idx: 2, Mask));
1941 }
1942 break;
1943 }
1944
1945 case X86::VPPERMrrm: {
1946 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: 3)) {
1947 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1948 SmallVector<int, 16> Mask;
1949 DecodeVPPERMMask(C, Width, ShuffleMask&: Mask);
1950 if (!Mask.empty())
1951 OutStreamer.AddComment(T: getShuffleComment(MI, SrcOp1Idx: 1, SrcOp2Idx: 2, Mask));
1952 }
1953 break;
1954 }
1955
1956 case X86::MMX_MOVQ64rm: {
1957 if (auto *C = X86::getConstantFromPool(MI: *MI, OpNo: 1)) {
1958 std::string Comment;
1959 raw_string_ostream CS(Comment);
1960 const MachineOperand &DstOp = MI->getOperand(i: 0);
1961 CS << X86ATTInstPrinter::getRegisterName(Reg: DstOp.getReg()) << " = ";
1962 if (auto *CF = dyn_cast<ConstantFP>(Val: C)) {
1963 CS << "0x" << toString(I: CF->getValueAPF().bitcastToAPInt(), Radix: 16, Signed: false);
1964 OutStreamer.AddComment(T: CS.str());
1965 }
1966 }
1967 break;
1968 }
1969
1970 case X86::GF2P8AFFINEQBrmi:
1971 case X86::VGF2P8AFFINEQBrmi:
1972 case X86::VGF2P8AFFINEQBYrmi:
1973 case X86::VGF2P8AFFINEQBZrmi:
1974 case X86::VGF2P8AFFINEQBZ128rmi:
1975 case X86::VGF2P8AFFINEQBZ256rmi: {
1976 // TODO: Add predicate handling with test coverage.
1977 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1978 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1979 addConstantComment(MI, OutStreamer, OpNo: SrcIdx + 1, BitWidth: Width);
1980 break;
1981 }
1982
1983 case X86::VGF2P8AFFINEQBZ128rmbi:
1984 case X86::VGF2P8AFFINEQBZ256rmbi:
1985 case X86::VGF2P8AFFINEQBZrmbi: {
1986 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
1987 unsigned Width = X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
1988 addConstantComment(MI, OutStreamer, OpNo: SrcIdx + 1, BitWidth: 64, Repeats: Width / 64);
1989 break;
1990 }
1991
1992#define INSTR_CASE(Prefix, Instr, Suffix, Postfix) \
1993 case X86::Prefix##Instr##Suffix##rm##Postfix:
1994
1995#define CASE_AVX512_ARITH_RM(Instr) \
1996 INSTR_CASE(V, Instr, Z128, ) \
1997 INSTR_CASE(V, Instr, Z128, k) \
1998 INSTR_CASE(V, Instr, Z128, kz) \
1999 INSTR_CASE(V, Instr, Z256, ) \
2000 INSTR_CASE(V, Instr, Z256, k) \
2001 INSTR_CASE(V, Instr, Z256, kz) \
2002 INSTR_CASE(V, Instr, Z, ) \
2003 INSTR_CASE(V, Instr, Z, k) \
2004 INSTR_CASE(V, Instr, Z, kz)
2005
2006#define CASE_ARITH_RM(Instr) \
2007 INSTR_CASE(, Instr, , ) /* SSE */ \
2008 INSTR_CASE(V, Instr, , ) /* AVX-128 */ \
2009 INSTR_CASE(V, Instr, Y, ) /* AVX-256 */ \
2010 INSTR_CASE(V, Instr, Z128, ) \
2011 INSTR_CASE(V, Instr, Z128, k) \
2012 INSTR_CASE(V, Instr, Z128, kz) \
2013 INSTR_CASE(V, Instr, Z256, ) \
2014 INSTR_CASE(V, Instr, Z256, k) \
2015 INSTR_CASE(V, Instr, Z256, kz) \
2016 INSTR_CASE(V, Instr, Z, ) \
2017 INSTR_CASE(V, Instr, Z, k) \
2018 INSTR_CASE(V, Instr, Z, kz)
2019
2020 // TODO: Add additional instructions when useful.
2021 CASE_ARITH_RM(PADDB)
2022 CASE_ARITH_RM(PADDW)
2023 CASE_ARITH_RM(PADDD)
2024 CASE_ARITH_RM(PADDQ)
2025 CASE_ARITH_RM(PSUBB)
2026 CASE_ARITH_RM(PSUBW)
2027 CASE_ARITH_RM(PSUBD)
2028 CASE_ARITH_RM(PSUBQ)
2029 CASE_ARITH_RM(PMADDUBSW)
2030 CASE_ARITH_RM(PMADDWD)
2031 CASE_ARITH_RM(PMULDQ)
2032 CASE_ARITH_RM(PMULUDQ)
2033 CASE_ARITH_RM(PMULLD)
2034 CASE_AVX512_ARITH_RM(PMULLQ)
2035 CASE_ARITH_RM(PMULLW)
2036 CASE_ARITH_RM(PMULHW)
2037 CASE_ARITH_RM(PMULHUW)
2038 CASE_ARITH_RM(PMULHRSW) {
2039 unsigned SrcIdx = getSrcIdx(MI, SrcIdx: 1);
2040 unsigned VectorWidth =
2041 X86::getVectorRegisterWidth(Info: MI->getDesc().operands()[0]);
2042 addConstantComment(MI, OutStreamer, OpNo: SrcIdx + 1, BitWidth: VectorWidth);
2043 break;
2044 }
2045
2046#define MASK_AVX512_CASE(Instr) \
2047 case Instr: \
2048 case Instr##k: \
2049 case Instr##kz:
2050
2051 case X86::MOVSDrm:
2052 case X86::VMOVSDrm:
2053 MASK_AVX512_CASE(X86::VMOVSDZrm)
2054 case X86::MOVSDrm_alt:
2055 case X86::VMOVSDrm_alt:
2056 case X86::VMOVSDZrm_alt:
2057 case X86::MOVQI2PQIrm:
2058 case X86::VMOVQI2PQIrm:
2059 case X86::VMOVQI2PQIZrm:
2060 printZeroUpperMove(MI, OutStreamer, SclWidth: 64, VecWidth: 128, ShuffleComment: "mem[0],zero");
2061 break;
2062
2063 MASK_AVX512_CASE(X86::VMOVSHZrm)
2064 case X86::VMOVSHZrm_alt:
2065 printZeroUpperMove(MI, OutStreamer, SclWidth: 16, VecWidth: 128,
2066 ShuffleComment: "mem[0],zero,zero,zero,zero,zero,zero,zero");
2067 break;
2068
2069 case X86::MOVSSrm:
2070 case X86::VMOVSSrm:
2071 MASK_AVX512_CASE(X86::VMOVSSZrm)
2072 case X86::MOVSSrm_alt:
2073 case X86::VMOVSSrm_alt:
2074 case X86::VMOVSSZrm_alt:
2075 case X86::MOVDI2PDIrm:
2076 case X86::VMOVDI2PDIrm:
2077 case X86::VMOVDI2PDIZrm:
2078 printZeroUpperMove(MI, OutStreamer, SclWidth: 32, VecWidth: 128, ShuffleComment: "mem[0],zero,zero,zero");
2079 break;
2080
2081#define MOV_CASE(Prefix, Suffix) \
2082 case X86::Prefix##MOVAPD##Suffix##rm: \
2083 case X86::Prefix##MOVAPS##Suffix##rm: \
2084 case X86::Prefix##MOVUPD##Suffix##rm: \
2085 case X86::Prefix##MOVUPS##Suffix##rm: \
2086 case X86::Prefix##MOVDQA##Suffix##rm: \
2087 case X86::Prefix##MOVDQU##Suffix##rm:
2088
2089#define MOV_AVX512_CASE(Suffix, Postfix) \
2090 case X86::VMOVDQA64##Suffix##rm##Postfix: \
2091 case X86::VMOVDQA32##Suffix##rm##Postfix: \
2092 case X86::VMOVDQU64##Suffix##rm##Postfix: \
2093 case X86::VMOVDQU32##Suffix##rm##Postfix: \
2094 case X86::VMOVDQU16##Suffix##rm##Postfix: \
2095 case X86::VMOVDQU8##Suffix##rm##Postfix: \
2096 case X86::VMOVAPS##Suffix##rm##Postfix: \
2097 case X86::VMOVAPD##Suffix##rm##Postfix: \
2098 case X86::VMOVUPS##Suffix##rm##Postfix: \
2099 case X86::VMOVUPD##Suffix##rm##Postfix:
2100
2101#define CASE_128_MOV_RM() \
2102 MOV_CASE(, ) /* SSE */ \
2103 MOV_CASE(V, ) /* AVX-128 */ \
2104 MOV_AVX512_CASE(Z128, ) \
2105 MOV_AVX512_CASE(Z128, k) \
2106 MOV_AVX512_CASE(Z128, kz)
2107
2108#define CASE_256_MOV_RM() \
2109 MOV_CASE(V, Y) /* AVX-256 */ \
2110 MOV_AVX512_CASE(Z256, ) \
2111 MOV_AVX512_CASE(Z256, k) \
2112 MOV_AVX512_CASE(Z256, kz) \
2113
2114#define CASE_512_MOV_RM() \
2115 MOV_AVX512_CASE(Z, ) \
2116 MOV_AVX512_CASE(Z, k) \
2117 MOV_AVX512_CASE(Z, kz) \
2118
2119 // For loads from a constant pool to a vector register, print the constant
2120 // loaded.
2121 CASE_128_MOV_RM()
2122 printBroadcast(MI, OutStreamer, Repeats: 1, BitWidth: 128);
2123 break;
2124 CASE_256_MOV_RM()
2125 printBroadcast(MI, OutStreamer, Repeats: 1, BitWidth: 256);
2126 break;
2127 CASE_512_MOV_RM()
2128 printBroadcast(MI, OutStreamer, Repeats: 1, BitWidth: 512);
2129 break;
2130 case X86::VBROADCASTF128rm:
2131 case X86::VBROADCASTI128rm:
2132 MASK_AVX512_CASE(X86::VBROADCASTF32X4Z256rm)
2133 MASK_AVX512_CASE(X86::VBROADCASTF64X2Z256rm)
2134 MASK_AVX512_CASE(X86::VBROADCASTI32X4Z256rm)
2135 MASK_AVX512_CASE(X86::VBROADCASTI64X2Z256rm)
2136 printBroadcast(MI, OutStreamer, Repeats: 2, BitWidth: 128);
2137 break;
2138 MASK_AVX512_CASE(X86::VBROADCASTF32X4Zrm)
2139 MASK_AVX512_CASE(X86::VBROADCASTF64X2Zrm)
2140 MASK_AVX512_CASE(X86::VBROADCASTI32X4Zrm)
2141 MASK_AVX512_CASE(X86::VBROADCASTI64X2Zrm)
2142 printBroadcast(MI, OutStreamer, Repeats: 4, BitWidth: 128);
2143 break;
2144 MASK_AVX512_CASE(X86::VBROADCASTF32X8Zrm)
2145 MASK_AVX512_CASE(X86::VBROADCASTF64X4Zrm)
2146 MASK_AVX512_CASE(X86::VBROADCASTI32X8Zrm)
2147 MASK_AVX512_CASE(X86::VBROADCASTI64X4Zrm)
2148 printBroadcast(MI, OutStreamer, Repeats: 2, BitWidth: 256);
2149 break;
2150
2151 // For broadcast loads from a constant pool to a vector register, repeatedly
2152 // print the constant loaded.
2153 case X86::MOVDDUPrm:
2154 case X86::VMOVDDUPrm:
2155 MASK_AVX512_CASE(X86::VMOVDDUPZ128rm)
2156 case X86::VPBROADCASTQrm:
2157 MASK_AVX512_CASE(X86::VPBROADCASTQZ128rm)
2158 printBroadcast(MI, OutStreamer, Repeats: 2, BitWidth: 64);
2159 break;
2160 case X86::VBROADCASTSDYrm:
2161 MASK_AVX512_CASE(X86::VBROADCASTSDZ256rm)
2162 case X86::VPBROADCASTQYrm:
2163 MASK_AVX512_CASE(X86::VPBROADCASTQZ256rm)
2164 printBroadcast(MI, OutStreamer, Repeats: 4, BitWidth: 64);
2165 break;
2166 MASK_AVX512_CASE(X86::VBROADCASTSDZrm)
2167 MASK_AVX512_CASE(X86::VPBROADCASTQZrm)
2168 printBroadcast(MI, OutStreamer, Repeats: 8, BitWidth: 64);
2169 break;
2170 case X86::VBROADCASTSSrm:
2171 MASK_AVX512_CASE(X86::VBROADCASTSSZ128rm)
2172 case X86::VPBROADCASTDrm:
2173 MASK_AVX512_CASE(X86::VPBROADCASTDZ128rm)
2174 printBroadcast(MI, OutStreamer, Repeats: 4, BitWidth: 32);
2175 break;
2176 case X86::VBROADCASTSSYrm:
2177 MASK_AVX512_CASE(X86::VBROADCASTSSZ256rm)
2178 case X86::VPBROADCASTDYrm:
2179 MASK_AVX512_CASE(X86::VPBROADCASTDZ256rm)
2180 printBroadcast(MI, OutStreamer, Repeats: 8, BitWidth: 32);
2181 break;
2182 MASK_AVX512_CASE(X86::VBROADCASTSSZrm)
2183 MASK_AVX512_CASE(X86::VPBROADCASTDZrm)
2184 printBroadcast(MI, OutStreamer, Repeats: 16, BitWidth: 32);
2185 break;
2186 case X86::VPBROADCASTWrm:
2187 MASK_AVX512_CASE(X86::VPBROADCASTWZ128rm)
2188 printBroadcast(MI, OutStreamer, Repeats: 8, BitWidth: 16);
2189 break;
2190 case X86::VPBROADCASTWYrm:
2191 MASK_AVX512_CASE(X86::VPBROADCASTWZ256rm)
2192 printBroadcast(MI, OutStreamer, Repeats: 16, BitWidth: 16);
2193 break;
2194 MASK_AVX512_CASE(X86::VPBROADCASTWZrm)
2195 printBroadcast(MI, OutStreamer, Repeats: 32, BitWidth: 16);
2196 break;
2197 case X86::VPBROADCASTBrm:
2198 MASK_AVX512_CASE(X86::VPBROADCASTBZ128rm)
2199 printBroadcast(MI, OutStreamer, Repeats: 16, BitWidth: 8);
2200 break;
2201 case X86::VPBROADCASTBYrm:
2202 MASK_AVX512_CASE(X86::VPBROADCASTBZ256rm)
2203 printBroadcast(MI, OutStreamer, Repeats: 32, BitWidth: 8);
2204 break;
2205 MASK_AVX512_CASE(X86::VPBROADCASTBZrm)
2206 printBroadcast(MI, OutStreamer, Repeats: 64, BitWidth: 8);
2207 break;
2208
2209#define MOVX_CASE(Prefix, Ext, Type, Suffix, Postfix) \
2210 case X86::Prefix##PMOV##Ext##Type##Suffix##rm##Postfix:
2211
2212#define CASE_MOVX_RM(Ext, Type) \
2213 MOVX_CASE(, Ext, Type, , ) \
2214 MOVX_CASE(V, Ext, Type, , ) \
2215 MOVX_CASE(V, Ext, Type, Y, ) \
2216 MOVX_CASE(V, Ext, Type, Z128, ) \
2217 MOVX_CASE(V, Ext, Type, Z128, k ) \
2218 MOVX_CASE(V, Ext, Type, Z128, kz ) \
2219 MOVX_CASE(V, Ext, Type, Z256, ) \
2220 MOVX_CASE(V, Ext, Type, Z256, k ) \
2221 MOVX_CASE(V, Ext, Type, Z256, kz ) \
2222 MOVX_CASE(V, Ext, Type, Z, ) \
2223 MOVX_CASE(V, Ext, Type, Z, k ) \
2224 MOVX_CASE(V, Ext, Type, Z, kz )
2225
2226 CASE_MOVX_RM(SX, BD)
2227 printSignExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 32);
2228 break;
2229 CASE_MOVX_RM(SX, BQ)
2230 printSignExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 64);
2231 break;
2232 CASE_MOVX_RM(SX, BW)
2233 printSignExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 16);
2234 break;
2235 CASE_MOVX_RM(SX, DQ)
2236 printSignExtend(MI, OutStreamer, SrcEltBits: 32, DstEltBits: 64);
2237 break;
2238 CASE_MOVX_RM(SX, WD)
2239 printSignExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 32);
2240 break;
2241 CASE_MOVX_RM(SX, WQ)
2242 printSignExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 64);
2243 break;
2244
2245 CASE_MOVX_RM(ZX, BD)
2246 printZeroExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 32);
2247 break;
2248 CASE_MOVX_RM(ZX, BQ)
2249 printZeroExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 64);
2250 break;
2251 CASE_MOVX_RM(ZX, BW)
2252 printZeroExtend(MI, OutStreamer, SrcEltBits: 8, DstEltBits: 16);
2253 break;
2254 CASE_MOVX_RM(ZX, DQ)
2255 printZeroExtend(MI, OutStreamer, SrcEltBits: 32, DstEltBits: 64);
2256 break;
2257 CASE_MOVX_RM(ZX, WD)
2258 printZeroExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 32);
2259 break;
2260 CASE_MOVX_RM(ZX, WQ)
2261 printZeroExtend(MI, OutStreamer, SrcEltBits: 16, DstEltBits: 64);
2262 break;
2263 }
2264}
2265
2266// Does the given operand refer to a DLLIMPORT function?
2267bool isImportedFunction(const MachineOperand &MO) {
2268 return MO.isGlobal() && (MO.getTargetFlags() == X86II::MO_DLLIMPORT);
2269}
2270
2271// Is the given instruction a call to a CFGuard function?
2272bool isCallToCFGuardFunction(const MachineInstr *MI) {
2273 assert(MI->getOpcode() == X86::TAILJMPm64_REX ||
2274 MI->getOpcode() == X86::CALL64m);
2275 const MachineOperand &MO = MI->getOperand(i: 3);
2276 return MO.isGlobal() && (MO.getTargetFlags() == X86II::MO_NO_FLAG) &&
2277 isCFGuardFunction(GV: MO.getGlobal());
2278}
2279
2280// Does the containing block for the given instruction contain any jump table
2281// info (indicating that the block is a dispatch for a jump table)?
2282bool hasJumpTableInfoInBlock(const llvm::MachineInstr *MI) {
2283 const MachineBasicBlock &MBB = *MI->getParent();
2284 for (auto I = MBB.instr_rbegin(), E = MBB.instr_rend(); I != E; ++I)
2285 if (I->isJumpTableDebugInfo())
2286 return true;
2287
2288 return false;
2289}
2290
2291void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
2292 // FIXME: Enable feature predicate checks once all the test pass.
2293 // X86_MC::verifyInstructionPredicates(MI->getOpcode(),
2294 // Subtarget->getFeatureBits());
2295
2296 X86MCInstLower MCInstLowering(*MF, *this);
2297 const X86RegisterInfo *RI =
2298 MF->getSubtarget<X86Subtarget>().getRegisterInfo();
2299
2300 if (MI->getOpcode() == X86::OR64rm) {
2301 for (auto &Opd : MI->operands()) {
2302 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
2303 "swift_async_extendedFramePointerFlags") {
2304 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
2305 }
2306 }
2307 }
2308
2309 // Add comments for values loaded from constant pool.
2310 if (OutStreamer->isVerboseAsm())
2311 addConstantComments(MI, OutStreamer&: *OutStreamer);
2312
2313 // Add a comment about EVEX compression
2314 if (TM.Options.MCOptions.ShowMCEncoding) {
2315 if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_LEGACY)
2316 OutStreamer->AddComment(T: "EVEX TO LEGACY Compression ", EOL: false);
2317 else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX)
2318 OutStreamer->AddComment(T: "EVEX TO VEX Compression ", EOL: false);
2319 else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_EVEX)
2320 OutStreamer->AddComment(T: "EVEX TO EVEX Compression ", EOL: false);
2321 }
2322
2323 // We use this to suppress NOP padding for Windows EH.
2324 bool IsTailJump = false;
2325
2326 switch (MI->getOpcode()) {
2327 case TargetOpcode::DBG_VALUE:
2328 llvm_unreachable("Should be handled target independently");
2329
2330 case X86::EH_RETURN:
2331 case X86::EH_RETURN64: {
2332 // Lower these as normal, but add some comments.
2333 Register Reg = MI->getOperand(i: 0).getReg();
2334 OutStreamer->AddComment(T: StringRef("eh_return, addr: %") +
2335 X86ATTInstPrinter::getRegisterName(Reg));
2336 break;
2337 }
2338 case X86::CLEANUPRET: {
2339 // Lower these as normal, but add some comments.
2340 OutStreamer->AddComment(T: "CLEANUPRET");
2341 break;
2342 }
2343
2344 case X86::CATCHRET: {
2345 // Lower these as normal, but add some comments.
2346 OutStreamer->AddComment(T: "CATCHRET");
2347 break;
2348 }
2349
2350 case X86::ENDBR32:
2351 case X86::ENDBR64: {
2352 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
2353 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2354 // non-empty. If MI is the initial ENDBR, place the
2355 // __patchable_function_entries label after ENDBR.
2356 if (CurrentPatchableFunctionEntrySym &&
2357 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
2358 MI == &MF->front().front()) {
2359 MCInst Inst;
2360 MCInstLowering.Lower(MI, OutMI&: Inst);
2361 EmitAndCountInstruction(Inst);
2362 CurrentPatchableFunctionEntrySym = createTempSymbol(Name: "patch");
2363 OutStreamer->emitLabel(Symbol: CurrentPatchableFunctionEntrySym);
2364 return;
2365 }
2366 break;
2367 }
2368
2369 case X86::TAILJMPd64:
2370 if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(Reg: X86::R11))
2371 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CS_PREFIX));
2372
2373 if (EnableImportCallOptimization && isImportedFunction(MO: MI->getOperand(i: 0))) {
2374 emitLabelAndRecordForImportCallOptimization(
2375 Kind: IMAGE_RETPOLINE_AMD64_IMPORT_BR);
2376 }
2377
2378 // Lower this as normal, but add a comment.
2379 OutStreamer->AddComment(T: "TAILCALL");
2380 IsTailJump = true;
2381 break;
2382
2383 case X86::TAILJMPr:
2384 case X86::TAILJMPm:
2385 case X86::TAILJMPd:
2386 case X86::TAILJMPd_CC:
2387 case X86::TAILJMPr64:
2388 case X86::TAILJMPm64:
2389 case X86::TAILJMPd64_CC:
2390 if (EnableImportCallOptimization)
2391 report_fatal_error(reason: "Unexpected TAILJMP instruction was emitted when "
2392 "import call optimization was enabled");
2393
2394 // Lower these as normal, but add some comments.
2395 OutStreamer->AddComment(T: "TAILCALL");
2396 IsTailJump = true;
2397 break;
2398
2399 case X86::TAILJMPm64_REX:
2400 if (EnableImportCallOptimization && isCallToCFGuardFunction(MI)) {
2401 emitLabelAndRecordForImportCallOptimization(
2402 Kind: IMAGE_RETPOLINE_AMD64_CFG_BR_REX);
2403 }
2404
2405 OutStreamer->AddComment(T: "TAILCALL");
2406 IsTailJump = true;
2407 break;
2408
2409 case X86::TAILJMPr64_REX: {
2410 if (EnableImportCallOptimization) {
2411 assert(MI->getOperand(0).getReg() == X86::RAX &&
2412 "Indirect tail calls with impcall enabled must go through RAX (as "
2413 "enforced by TCRETURNImpCallri64)");
2414 emitLabelAndRecordForImportCallOptimization(
2415 Kind: IMAGE_RETPOLINE_AMD64_INDIR_BR);
2416 }
2417
2418 OutStreamer->AddComment(T: "TAILCALL");
2419 IsTailJump = true;
2420 break;
2421 }
2422
2423 case X86::JMP64r:
2424 if (EnableImportCallOptimization && hasJumpTableInfoInBlock(MI)) {
2425 uint16_t EncodedReg =
2426 this->getSubtarget().getRegisterInfo()->getEncodingValue(
2427 Reg: MI->getOperand(i: 0).getReg().asMCReg());
2428 emitLabelAndRecordForImportCallOptimization(
2429 Kind: (ImportCallKind)(IMAGE_RETPOLINE_AMD64_SWITCHTABLE_FIRST +
2430 EncodedReg));
2431 }
2432 break;
2433
2434 case X86::JMP16r:
2435 case X86::JMP16m:
2436 case X86::JMP32r:
2437 case X86::JMP32m:
2438 case X86::JMP64m:
2439 if (EnableImportCallOptimization && hasJumpTableInfoInBlock(MI))
2440 report_fatal_error(
2441 reason: "Unexpected JMP instruction was emitted for a jump-table when import "
2442 "call optimization was enabled");
2443 break;
2444
2445 case X86::TLS_addr32:
2446 case X86::TLS_addr64:
2447 case X86::TLS_addrX32:
2448 case X86::TLS_base_addr32:
2449 case X86::TLS_base_addr64:
2450 case X86::TLS_base_addrX32:
2451 case X86::TLS_desc32:
2452 case X86::TLS_desc64:
2453 return LowerTlsAddr(MCInstLowering, MI: *MI);
2454
2455 case X86::MOVPC32r: {
2456 // This is a pseudo op for a two instruction sequence with a label, which
2457 // looks like:
2458 // call "L1$pb"
2459 // "L1$pb":
2460 // popl %esi
2461
2462 // Emit the call.
2463 MCSymbol *PICBase = MF->getPICBaseSymbol();
2464 // FIXME: We would like an efficient form for this, so we don't have to do a
2465 // lot of extra uniquing.
2466 EmitAndCountInstruction(
2467 Inst&: MCInstBuilder(X86::CALLpcrel32)
2468 .addExpr(Val: MCSymbolRefExpr::create(Symbol: PICBase, Ctx&: OutContext)));
2469
2470 const X86FrameLowering *FrameLowering =
2471 MF->getSubtarget<X86Subtarget>().getFrameLowering();
2472 bool hasFP = FrameLowering->hasFP(MF: *MF);
2473
2474 // TODO: This is needed only if we require precise CFA.
2475 bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() &&
2476 !OutStreamer->getDwarfFrameInfos().back().End;
2477
2478 int stackGrowth = -RI->getSlotSize();
2479
2480 if (HasActiveDwarfFrame && !hasFP) {
2481 OutStreamer->emitCFIAdjustCfaOffset(Adjustment: -stackGrowth);
2482 MF->getInfo<X86MachineFunctionInfo>()->setHasCFIAdjustCfa(true);
2483 }
2484
2485 // Emit the label.
2486 OutStreamer->emitLabel(Symbol: PICBase);
2487
2488 // popl $reg
2489 EmitAndCountInstruction(
2490 Inst&: MCInstBuilder(X86::POP32r).addReg(Reg: MI->getOperand(i: 0).getReg()));
2491
2492 if (HasActiveDwarfFrame && !hasFP) {
2493 OutStreamer->emitCFIAdjustCfaOffset(Adjustment: stackGrowth);
2494 }
2495 return;
2496 }
2497
2498 case X86::ADD32ri: {
2499 // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri.
2500 if (MI->getOperand(i: 2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS)
2501 break;
2502
2503 // Okay, we have something like:
2504 // EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL)
2505
2506 // For this, we want to print something like:
2507 // MYGLOBAL + (. - PICBASE)
2508 // However, we can't generate a ".", so just emit a new label here and refer
2509 // to it.
2510 MCSymbol *DotSym = OutContext.createTempSymbol();
2511 OutStreamer->emitLabel(Symbol: DotSym);
2512
2513 // Now that we have emitted the label, lower the complex operand expression.
2514 MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MO: MI->getOperand(i: 2));
2515
2516 const MCExpr *DotExpr = MCSymbolRefExpr::create(Symbol: DotSym, Ctx&: OutContext);
2517 const MCExpr *PICBase =
2518 MCSymbolRefExpr::create(Symbol: MF->getPICBaseSymbol(), Ctx&: OutContext);
2519 DotExpr = MCBinaryExpr::createSub(LHS: DotExpr, RHS: PICBase, Ctx&: OutContext);
2520
2521 DotExpr = MCBinaryExpr::createAdd(
2522 LHS: MCSymbolRefExpr::create(Symbol: OpSym, Ctx&: OutContext), RHS: DotExpr, Ctx&: OutContext);
2523
2524 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::ADD32ri)
2525 .addReg(Reg: MI->getOperand(i: 0).getReg())
2526 .addReg(Reg: MI->getOperand(i: 1).getReg())
2527 .addExpr(Val: DotExpr));
2528 return;
2529 }
2530 case TargetOpcode::STATEPOINT:
2531 return LowerSTATEPOINT(MI: *MI, MCIL&: MCInstLowering);
2532
2533 case TargetOpcode::FAULTING_OP:
2534 return LowerFAULTING_OP(FaultingMI: *MI, MCIL&: MCInstLowering);
2535
2536 case TargetOpcode::FENTRY_CALL:
2537 return LowerFENTRY_CALL(MI: *MI, MCIL&: MCInstLowering);
2538
2539 case TargetOpcode::PATCHABLE_OP:
2540 return LowerPATCHABLE_OP(MI: *MI, MCIL&: MCInstLowering);
2541
2542 case TargetOpcode::STACKMAP:
2543 return LowerSTACKMAP(MI: *MI);
2544
2545 case TargetOpcode::PATCHPOINT:
2546 return LowerPATCHPOINT(MI: *MI, MCIL&: MCInstLowering);
2547
2548 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2549 return LowerPATCHABLE_FUNCTION_ENTER(MI: *MI, MCIL&: MCInstLowering);
2550
2551 case TargetOpcode::PATCHABLE_RET:
2552 return LowerPATCHABLE_RET(MI: *MI, MCIL&: MCInstLowering);
2553
2554 case TargetOpcode::PATCHABLE_TAIL_CALL:
2555 return LowerPATCHABLE_TAIL_CALL(MI: *MI, MCIL&: MCInstLowering);
2556
2557 case TargetOpcode::PATCHABLE_EVENT_CALL:
2558 return LowerPATCHABLE_EVENT_CALL(MI: *MI, MCIL&: MCInstLowering);
2559
2560 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
2561 return LowerPATCHABLE_TYPED_EVENT_CALL(MI: *MI, MCIL&: MCInstLowering);
2562
2563 case X86::MORESTACK_RET:
2564 EmitAndCountInstruction(Inst&: MCInstBuilder(getRetOpcode(Subtarget: *Subtarget)));
2565 return;
2566
2567 case X86::KCFI_CHECK:
2568 return LowerKCFI_CHECK(MI: *MI);
2569
2570 case X86::ASAN_CHECK_MEMACCESS:
2571 return LowerASAN_CHECK_MEMACCESS(MI: *MI);
2572
2573 case X86::MORESTACK_RET_RESTORE_R10:
2574 // Return, then restore R10.
2575 EmitAndCountInstruction(Inst&: MCInstBuilder(getRetOpcode(Subtarget: *Subtarget)));
2576 EmitAndCountInstruction(
2577 Inst&: MCInstBuilder(X86::MOV64rr).addReg(Reg: X86::R10).addReg(Reg: X86::RAX));
2578 return;
2579
2580 case X86::SEH_PushReg:
2581 case X86::SEH_Push2Regs:
2582 case X86::SEH_SaveReg:
2583 case X86::SEH_SaveXMM:
2584 case X86::SEH_StackAlloc:
2585 case X86::SEH_StackAlign:
2586 case X86::SEH_SetFrame:
2587 case X86::SEH_PushFrame:
2588 case X86::SEH_EndPrologue:
2589 case X86::SEH_EndEpilogue:
2590 case X86::SEH_UnwindV2Start:
2591 case X86::SEH_UnwindVersion:
2592 EmitSEHInstruction(MI);
2593 return;
2594
2595 case X86::SEH_SplitChainedAtEndOfBlock:
2596 assert(!SplitChainedAtEndOfBlock &&
2597 "Duplicate SEH_SplitChainedAtEndOfBlock in a current block");
2598 SplitChainedAtEndOfBlock = true;
2599 return;
2600
2601 case X86::SEH_SplitChained:
2602 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2603 OutStreamer->emitWinCFISplitChained();
2604 return;
2605
2606 case X86::SEH_BeginEpilogue: {
2607 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2608 EmitSEHInstruction(MI);
2609 return;
2610 }
2611 case X86::UBSAN_UD1:
2612 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::UD1Lm)
2613 .addReg(Reg: X86::EAX)
2614 .addReg(Reg: X86::EAX)
2615 .addImm(Val: 1)
2616 .addReg(Reg: X86::NoRegister)
2617 .addImm(Val: MI->getOperand(i: 0).getImm())
2618 .addReg(Reg: X86::NoRegister));
2619 return;
2620 case X86::CALL64pcrel32:
2621 if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(Reg: X86::R11))
2622 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::CS_PREFIX));
2623
2624 if (EnableImportCallOptimization && isImportedFunction(MO: MI->getOperand(i: 0))) {
2625 emitLabelAndRecordForImportCallOptimization(
2626 Kind: IMAGE_RETPOLINE_AMD64_IMPORT_CALL);
2627
2628 MCInst TmpInst;
2629 MCInstLowering.Lower(MI, OutMI&: TmpInst);
2630
2631 // For Import Call Optimization to work, we need a the call instruction
2632 // with a rex prefix, and a 5-byte nop after the call instruction.
2633 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::REX64_PREFIX));
2634 emitCallInstruction(MCI: TmpInst);
2635 emitNop(OS&: *OutStreamer, NumBytes: 5, Subtarget);
2636 maybeEmitNopAfterCallForWindowsEH(MI);
2637 return;
2638 }
2639
2640 break;
2641
2642 case X86::CALL64r:
2643 if (EnableImportCallOptimization) {
2644 assert(MI->getOperand(0).getReg() == X86::RAX &&
2645 "Indirect calls with impcall enabled must go through RAX (as "
2646 "enforced by CALL64r_ImpCall)");
2647
2648 emitLabelAndRecordForImportCallOptimization(
2649 Kind: IMAGE_RETPOLINE_AMD64_INDIR_CALL);
2650 MCInst TmpInst;
2651 MCInstLowering.Lower(MI, OutMI&: TmpInst);
2652 emitCallInstruction(MCI: TmpInst);
2653
2654 // For Import Call Optimization to work, we need a 3-byte nop after the
2655 // call instruction.
2656 emitNop(OS&: *OutStreamer, NumBytes: 3, Subtarget);
2657 maybeEmitNopAfterCallForWindowsEH(MI);
2658 return;
2659 }
2660 break;
2661
2662 case X86::CALL64m:
2663 if (EnableImportCallOptimization && isCallToCFGuardFunction(MI)) {
2664 emitLabelAndRecordForImportCallOptimization(
2665 Kind: IMAGE_RETPOLINE_AMD64_CFG_CALL);
2666 }
2667 break;
2668
2669 case X86::JCC_1:
2670 // Two instruction prefixes (2EH for branch not-taken and 3EH for branch
2671 // taken) are used as branch hints. Here we add branch taken prefix for
2672 // jump instruction with higher probability than threshold.
2673 if (getSubtarget().hasBranchHint() &&
2674 getSubtarget().getCLOpts().enable_branch_hint) {
2675 const MachineBranchProbabilityInfo *MBPI = GetMBPI(*MF);
2676 MachineBasicBlock *DestBB = MI->getOperand(i: 0).getMBB();
2677 BranchProbability EdgeProb =
2678 MBPI->getEdgeProbability(Src: MI->getParent(), Dst: DestBB);
2679 BranchProbability Threshold(
2680 getSubtarget().getCLOpts().branch_hint_probability_threshold, 100);
2681 if (EdgeProb > Threshold)
2682 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::DS_PREFIX));
2683 }
2684 break;
2685
2686 case X86::JCC_SELF:
2687 MCSymbol *Sym = OutContext.createTempSymbol();
2688 OutStreamer->emitLabel(Symbol: Sym);
2689 EmitAndCountInstruction(
2690 Inst&: MCInstBuilder(X86::JCC_1)
2691 .addExpr(Val: MCSymbolRefExpr::create(Symbol: Sym, Ctx&: OutContext))
2692 .addImm(Val: MI->getOperand(i: 0).getImm()));
2693 return;
2694 }
2695
2696 MCInst TmpInst;
2697 MCInstLowering.Lower(MI, OutMI&: TmpInst);
2698
2699 if (MI->isCall()) {
2700 emitCallInstruction(MCI: TmpInst);
2701 // Since tail calls transfer control without leaving a stack frame, there is
2702 // never a need for NOP padding tail calls.
2703 if (!IsTailJump)
2704 maybeEmitNopAfterCallForWindowsEH(MI);
2705 return;
2706 }
2707
2708 EmitAndCountInstruction(Inst&: TmpInst);
2709}
2710
2711void X86AsmPrinter::emitInlineAsmEnd(const MCSubtargetInfo &StartInfo,
2712 const MCSubtargetInfo *EndInfo,
2713 const MachineInstr *MI) {
2714 if (MI) {
2715 // If unwinding inline asm ends on a call, wineh may require insertion of
2716 // a nop.
2717 unsigned ExtraInfo = MI->getOperand(i: InlineAsm::MIOp_ExtraInfo).getImm();
2718 if (ExtraInfo & InlineAsm::Extra_MayUnwind)
2719 maybeEmitNopAfterCallForWindowsEH(MI);
2720 }
2721}
2722
2723void X86AsmPrinter::emitCallInstruction(const llvm::MCInst &MCI) {
2724 // Stackmap shadows cannot include branch targets, so we can count the bytes
2725 // in a call towards the shadow, but must ensure that the no thread returns
2726 // in to the stackmap shadow. The only way to achieve this is if the call
2727 // is at the end of the shadow.
2728
2729 // Count then size of the call towards the shadow
2730 SMShadowTracker.count(Inst: MCI, STI: getSubtargetInfo(), CodeEmitter: CodeEmitter.get());
2731 // Then flush the shadow so that we fill with nops before the call, not
2732 // after it.
2733 SMShadowTracker.emitShadowPadding(OutStreamer&: *OutStreamer, STI: getSubtargetInfo());
2734 // Then emit the call
2735 OutStreamer->emitInstruction(Inst: MCI, STI: getSubtargetInfo());
2736}
2737
2738// Determines whether a NOP is required after a CALL, so that Windows EH
2739// IP2State tables have the correct information.
2740//
2741// On most Windows platforms (AMD64, ARM64, ARM32, IA64, but *not* x86-32),
2742// exception handling works by looking up instruction pointers in lookup
2743// tables. These lookup tables are stored in .xdata sections in executables.
2744// One element of the lookup tables are the "IP2State" tables (Instruction
2745// Pointer to State).
2746//
2747// If a function has any instructions that require cleanup during exception
2748// unwinding, then it will have an IP2State table. Each entry in the IP2State
2749// table describes a range of bytes in the function's instruction stream, and
2750// associates an "EH state number" with that range of instructions. A value of
2751// -1 means "the null state", which does not require any code to execute.
2752// A value other than -1 is an index into the State table.
2753//
2754// The entries in the IP2State table contain byte offsets within the instruction
2755// stream of the function. The Windows ABI requires that these offsets are
2756// aligned to instruction boundaries; they are not permitted to point to a byte
2757// that is not the first byte of an instruction.
2758//
2759// Unfortunately, CALL instructions present a problem during unwinding. CALL
2760// instructions push the address of the instruction after the CALL instruction,
2761// so that execution can resume after the CALL. If the CALL is the last
2762// instruction within an IP2State region, then the return address (on the stack)
2763// points to the *next* IP2State region. This means that the unwinder will
2764// use the wrong cleanup funclet during unwinding.
2765//
2766// To fix this problem, the Windows AMD64 ABI requires that CALL instructions
2767// are never placed at the end of an IP2State region. Stated equivalently, the
2768// end of a CALL instruction cannot be aligned to an IP2State boundary. If a
2769// CALL instruction would occur at the end of an IP2State region, then the
2770// compiler must insert a NOP instruction after the CALL. The NOP instruction
2771// is placed in the same EH region as the CALL instruction, so that the return
2772// address points to the NOP and the unwinder will locate the correct region.
2773//
2774// NOP padding is only necessary on Windows AMD64 targets. On ARM64 and ARM32,
2775// instructions have a fixed size so the unwinder knows how to "back up" by
2776// one instruction.
2777//
2778// Interaction with Import Call Optimization (ICO):
2779//
2780// Import Call Optimization (ICO) is a compiler + OS feature on Windows which
2781// improves the performance and security of DLL imports. ICO relies on using a
2782// specific CALL idiom that can be replaced by the OS DLL loader. This removes
2783// a load and indirect CALL and replaces it with a single direct CALL.
2784//
2785// To achieve this, ICO also inserts NOPs after the CALL instruction. If the
2786// end of the CALL is aligned with an EH state transition, we *also* insert
2787// a single-byte NOP. **Both forms of NOPs must be preserved.** They cannot
2788// be combined into a single larger NOP; nor can the second NOP be removed.
2789//
2790// This is necessary because, if ICO is active and the call site is modified
2791// by the loader, the loader will end up overwriting the NOPs that were inserted
2792// for ICO. That means that those NOPs cannot be used for the correct
2793// termination of the exception handling region (the IP2State transition),
2794// so we still need an additional NOP instruction. The NOPs cannot be combined
2795// into a longer NOP (which is ordinarily desirable) because then ICO would
2796// split one instruction, producing a malformed instruction after the ICO call.
2797void X86AsmPrinter::maybeEmitNopAfterCallForWindowsEH(const MachineInstr *MI) {
2798 // We only need to insert NOPs after CALLs when targeting Windows on AMD64.
2799 // (Don't let the name fool you: Itanium refers to table-based exception
2800 // handling, not the Itanium architecture.)
2801 if (getModuleExceptionModel() != ExceptionHandling::WinEH ||
2802 MAI.getWinEHEncodingType() != WinEH::EncodingType::Itanium) {
2803 return;
2804 }
2805
2806 bool HasEHPersonality = MF->getWinEHFuncInfo() != nullptr;
2807
2808 // Set up MBB iterator, initially positioned on the same MBB as MI.
2809 MachineFunction::const_iterator MFI(MI->getParent());
2810 MachineFunction::const_iterator MFE(MF->end());
2811
2812 // Set up instruction iterator, positioned immediately *after* MI.
2813 MachineBasicBlock::const_iterator MBBI(MI);
2814 MachineBasicBlock::const_iterator MBBE = MI->getParent()->end();
2815 ++MBBI; // Step over MI
2816
2817 // This loop iterates MBBs
2818 for (;;) {
2819 // This loop iterates instructions
2820 for (; MBBI != MBBE; ++MBBI) {
2821 // Check the instruction that follows this CALL.
2822 const MachineInstr &NextMI = *MBBI;
2823
2824 // If there is an EH_LABEL after this CALL, then there is an EH state
2825 // transition after this CALL. This is exactly the situation which
2826 // requires NOP padding.
2827 if (NextMI.isEHLabel()) {
2828 if (HasEHPersonality) {
2829 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::NOOP));
2830 return;
2831 }
2832 // We actually want to continue, in case there is an SEH_BeginEpilogue
2833 // instruction after the EH_LABEL. In some situations, IR is produced
2834 // that contains EH_LABEL pseudo-instructions, even when we are not
2835 // generating IP2State tables. We still need to insert a NOP before
2836 // SEH_BeginEpilogue in that case.
2837 continue;
2838 }
2839
2840 // Somewhat similarly, if the CALL is the last instruction before the
2841 // SEH prologue, then we also need a NOP. This is necessary because the
2842 // Windows stack unwinder will not invoke a function's exception handler
2843 // if the instruction pointer is in the function prologue or epilogue.
2844 //
2845 // We always emit a NOP before SEH_BeginEpilogue, even if there is no
2846 // personality function (unwind info) for this frame. This is the same
2847 // behavior as MSVC.
2848 if (NextMI.getOpcode() == X86::SEH_BeginEpilogue) {
2849 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::NOOP));
2850 return;
2851 }
2852
2853 if (!NextMI.isPseudo() && !NextMI.isMetaInstruction()) {
2854 // We found a real instruction. During the CALL, the return IP will
2855 // point to this instruction. Since this instruction has the same EH
2856 // state as the call itself (because there is no intervening EH_LABEL),
2857 // the IP2State table will be accurate; there is no need to insert a
2858 // NOP.
2859 return;
2860 }
2861
2862 // The next instruction is a pseudo-op. Ignore it and keep searching.
2863 // Because these instructions do not generate any machine code, they
2864 // cannot prevent the IP2State table from pointing at the wrong
2865 // instruction during a CALL.
2866 }
2867
2868 // We've reached the end of this MBB. Find the next MBB in program order.
2869 // MBB order should be finalized by this point, so falling across MBBs is
2870 // expected.
2871 ++MFI;
2872 if (MFI == MFE) {
2873 // No more blocks; we've reached the end of the function. This should
2874 // only happen with no-return functions, but double-check to be sure.
2875 if (HasEHPersonality) {
2876 // If the CALL has no successors, then it is a noreturn function.
2877 // Insert an INT3 instead of a NOP. This accomplishes the same purpose,
2878 // but is more clear to read. Also, analysis tools will understand
2879 // that they should not continue disassembling after the CALL (unless
2880 // there are other branches to that label).
2881 if (MI->getParent()->succ_empty())
2882 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::INT3));
2883 else
2884 EmitAndCountInstruction(Inst&: MCInstBuilder(X86::NOOP));
2885 }
2886 return;
2887 }
2888
2889 // Set up iterator to scan the next basic block.
2890 const MachineBasicBlock *NextMBB = &*MFI;
2891 MBBI = NextMBB->instr_begin();
2892 MBBE = NextMBB->instr_end();
2893 }
2894}
2895
2896void X86AsmPrinter::emitLabelAndRecordForImportCallOptimization(
2897 ImportCallKind Kind) {
2898 assert(EnableImportCallOptimization);
2899
2900 MCSymbol *CallSiteSymbol = MMI->getContext().createNamedTempSymbol(Name: "impcall");
2901 OutStreamer->emitLabel(Symbol: CallSiteSymbol);
2902
2903 SectionToImportedFunctionCalls[OutStreamer->getCurrentSectionOnly()]
2904 .push_back(x: {.CalleeSymbol: CallSiteSymbol, .Kind: Kind});
2905}
2906