1//===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the X86MCCodeEmitter class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "MCTargetDesc/X86BaseInfo.h"
14#include "MCTargetDesc/X86FixupKinds.h"
15#include "MCTargetDesc/X86MCAsmInfo.h"
16#include "MCTargetDesc/X86MCTargetDesc.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/BinaryFormat/ELF.h"
19#include "llvm/MC/MCCodeEmitter.h"
20#include "llvm/MC/MCContext.h"
21#include "llvm/MC/MCExpr.h"
22#include "llvm/MC/MCFixup.h"
23#include "llvm/MC/MCInst.h"
24#include "llvm/MC/MCInstrDesc.h"
25#include "llvm/MC/MCInstrInfo.h"
26#include "llvm/MC/MCRegisterInfo.h"
27#include "llvm/MC/MCSubtargetInfo.h"
28#include "llvm/MC/MCSymbol.h"
29#include "llvm/Support/Casting.h"
30#include "llvm/Support/ErrorHandling.h"
31#include <cassert>
32#include <cstdint>
33
34using namespace llvm;
35
36#define DEBUG_TYPE "mccodeemitter"
37
38namespace {
39
40enum PrefixKind { None, REX, REX2, XOP, VEX2, VEX3, EVEX };
41
42static void emitByte(uint8_t C, SmallVectorImpl<char> &CB) { CB.push_back(Elt: C); }
43
44class X86OpcodePrefixHelper {
45 // REX (1 byte)
46 // +-----+ +------+
47 // | 40H | | WRXB |
48 // +-----+ +------+
49
50 // REX2 (2 bytes)
51 // +-----+ +-------------------+
52 // | D5H | | M | R'X'B' | WRXB |
53 // +-----+ +-------------------+
54
55 // XOP (3-byte)
56 // +-----+ +--------------+ +-------------------+
57 // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
58 // +-----+ +--------------+ +-------------------+
59
60 // VEX2 (2 bytes)
61 // +-----+ +-------------------+
62 // | C5h | | R | vvvv | L | pp |
63 // +-----+ +-------------------+
64
65 // VEX3 (3 bytes)
66 // +-----+ +--------------+ +-------------------+
67 // | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
68 // +-----+ +--------------+ +-------------------+
69
70 // VEX_R: opcode externsion equivalent to REX.R in
71 // 1's complement (inverted) form
72 //
73 // 1: Same as REX_R=0 (must be 1 in 32-bit mode)
74 // 0: Same as REX_R=1 (64 bit mode only)
75
76 // VEX_X: equivalent to REX.X, only used when a
77 // register is used for index in SIB Byte.
78 //
79 // 1: Same as REX.X=0 (must be 1 in 32-bit mode)
80 // 0: Same as REX.X=1 (64-bit mode only)
81
82 // VEX_B:
83 // 1: Same as REX_B=0 (ignored in 32-bit mode)
84 // 0: Same as REX_B=1 (64 bit mode only)
85
86 // VEX_W: opcode specific (use like REX.W, or used for
87 // opcode extension, or ignored, depending on the opcode byte)
88
89 // VEX_5M (VEX m-mmmmm field):
90 //
91 // 0b00000: Reserved for future use
92 // 0b00001: implied 0F leading opcode
93 // 0b00010: implied 0F 38 leading opcode bytes
94 // 0b00011: implied 0F 3A leading opcode bytes
95 // 0b00100: Reserved for future use
96 // 0b00101: VEX MAP5
97 // 0b00110: VEX MAP6
98 // 0b00111: VEX MAP7
99 // 0b00111-0b11111: Reserved for future use
100 // 0b01000: XOP map select - 08h instructions with imm byte
101 // 0b01001: XOP map select - 09h instructions with no imm byte
102 // 0b01010: XOP map select - 0Ah instructions with imm dword
103
104 // VEX_4V (VEX vvvv field): a register specifier
105 // (in 1's complement form) or 1111 if unused.
106
107 // VEX_PP: opcode extension providing equivalent
108 // functionality of a SIMD prefix
109 // 0b00: None
110 // 0b01: 66
111 // 0b10: F3
112 // 0b11: F2
113
114 // EVEX (4 bytes)
115 // +-----+ +---------------+ +-------------------+ +------------------------+
116 // | 62h | | RXBR' | B'mmm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa |
117 // +-----+ +---------------+ +-------------------+ +------------------------+
118
119 // EVEX_L2/VEX_L (Vector Length):
120 // L2 L
121 // 0 0: scalar or 128-bit vector
122 // 0 1: 256-bit vector
123 // 1 0: 512-bit vector
124
125 // 32-Register Support in 64-bit Mode Using EVEX with Embedded REX/REX2 Bits:
126 //
127 // +----------+---------+--------+-----------+---------+--------------+
128 // | | 4 | 3 | [2:0] | Type | Common Usage |
129 // +----------+---------+--------+-----------+---------+--------------+
130 // | REG | EVEX_R' | EVEX_R | modrm.reg | GPR, VR | Dest or Src |
131 // | VVVV | EVEX_v' | EVEX.vvvv | GPR, VR | Dest or Src |
132 // | RM (VR) | EVEX_X | EVEX_B | modrm.r/m | VR | Dest or Src |
133 // | RM (GPR) | EVEX_B' | EVEX_B | modrm.r/m | GPR | Dest or Src |
134 // | BASE | EVEX_B' | EVEX_B | modrm.r/m | GPR | MA |
135 // | INDEX | EVEX_U | EVEX_X | sib.index | GPR | MA |
136 // | VIDX | EVEX_v' | EVEX_X | sib.index | VR | VSIB MA |
137 // +----------+---------+--------+-----------+---------+--------------+
138 //
139 // * GPR - General-purpose register
140 // * VR - Vector register
141 // * VIDX - Vector index
142 // * VSIB - Vector SIB
143 // * MA - Memory addressing
144
145private:
146 unsigned W : 1;
147 unsigned R : 1;
148 unsigned X : 1;
149 unsigned B : 1;
150 unsigned M : 1;
151 unsigned R2 : 1;
152 unsigned X2 : 1;
153 unsigned B2 : 1;
154 unsigned VEX_4V : 4;
155 unsigned VEX_L : 1;
156 unsigned VEX_PP : 2;
157 unsigned VEX_5M : 5;
158 unsigned EVEX_z : 1;
159 unsigned EVEX_L2 : 1;
160 unsigned EVEX_b : 1;
161 unsigned EVEX_V2 : 1;
162 unsigned EVEX_aaa : 3;
163 PrefixKind Kind = None;
164 const MCRegisterInfo &MRI;
165
166 unsigned getRegEncoding(const MCInst &MI, unsigned OpNum) const {
167 return MRI.getEncodingValue(Reg: MI.getOperand(i: OpNum).getReg());
168 }
169
170 void setR(unsigned Encoding) { R = Encoding >> 3 & 1; }
171 void setR2(unsigned Encoding) {
172 R2 = Encoding >> 4 & 1;
173 assert((!R2 || (Kind <= REX2 || Kind == EVEX)) && "invalid setting");
174 }
175 void setX(unsigned Encoding) { X = Encoding >> 3 & 1; }
176 void setX2(unsigned Encoding) {
177 assert((Kind <= REX2 || Kind == EVEX) && "invalid setting");
178 X2 = Encoding >> 4 & 1;
179 }
180 void setB(unsigned Encoding) { B = Encoding >> 3 & 1; }
181 void setB2(unsigned Encoding) {
182 assert((Kind <= REX2 || Kind == EVEX) && "invalid setting");
183 B2 = Encoding >> 4 & 1;
184 }
185 void set4V(unsigned Encoding) { VEX_4V = Encoding & 0xf; }
186 void setV2(unsigned Encoding) { EVEX_V2 = Encoding >> 4 & 1; }
187
188public:
189 void setW(bool V) { W = V; }
190 void setR(const MCInst &MI, unsigned OpNum) {
191 setR(getRegEncoding(MI, OpNum));
192 }
193 void setX(const MCInst &MI, unsigned OpNum, unsigned Shift = 3) {
194 MCRegister Reg = MI.getOperand(i: OpNum).getReg();
195 // X is used to extend vector register only when shift is not 3.
196 if (Shift != 3 && X86II::isApxExtendedReg(Reg))
197 return;
198 unsigned Encoding = MRI.getEncodingValue(Reg);
199 X = Encoding >> Shift & 1;
200 }
201 void setB(const MCInst &MI, unsigned OpNum) {
202 B = getRegEncoding(MI, OpNum) >> 3 & 1;
203 }
204 void set4V(const MCInst &MI, unsigned OpNum, bool IsImm = false) {
205 // OF, SF, ZF and CF reuse VEX_4V bits but are not reversed
206 if (IsImm)
207 set4V(~(MI.getOperand(i: OpNum).getImm()));
208 else
209 set4V(getRegEncoding(MI, OpNum));
210 }
211 void setL(bool V) { VEX_L = V; }
212 void setPP(unsigned V) { VEX_PP = V; }
213 void set5M(unsigned V) { VEX_5M = V; }
214 void setR2(const MCInst &MI, unsigned OpNum) {
215 setR2(getRegEncoding(MI, OpNum));
216 }
217 void setRR2(const MCInst &MI, unsigned OpNum) {
218 unsigned Encoding = getRegEncoding(MI, OpNum);
219 setR(Encoding);
220 setR2(Encoding);
221 }
222 void setM(bool V) { M = V; }
223 void setXX2(const MCInst &MI, unsigned OpNum) {
224 MCRegister Reg = MI.getOperand(i: OpNum).getReg();
225 unsigned Encoding = MRI.getEncodingValue(Reg);
226 setX(Encoding);
227 // Index can be a vector register while X2 is used to extend GPR only.
228 if (Kind <= REX2 || X86II::isApxExtendedReg(Reg))
229 setX2(Encoding);
230 }
231 void setBB2(const MCInst &MI, unsigned OpNum) {
232 MCRegister Reg = MI.getOperand(i: OpNum).getReg();
233 unsigned Encoding = MRI.getEncodingValue(Reg);
234 setB(Encoding);
235 // Base can be a vector register while B2 is used to extend GPR only
236 if (Kind <= REX2 || X86II::isApxExtendedReg(Reg))
237 setB2(Encoding);
238 }
239 void setZ(bool V) { EVEX_z = V; }
240 void setL2(bool V) { EVEX_L2 = V; }
241 void setEVEX_b(bool V) { EVEX_b = V; }
242 void setEVEX_U(bool V) { X2 = V; }
243 void setV2(const MCInst &MI, unsigned OpNum, bool HasVEX_4V) {
244 // Only needed with VSIB which don't use VVVV.
245 if (HasVEX_4V)
246 return;
247 MCRegister Reg = MI.getOperand(i: OpNum).getReg();
248 if (X86II::isApxExtendedReg(Reg))
249 return;
250 setV2(MRI.getEncodingValue(Reg));
251 }
252 void set4VV2(const MCInst &MI, unsigned OpNum) {
253 unsigned Encoding = getRegEncoding(MI, OpNum);
254 set4V(Encoding);
255 setV2(Encoding);
256 }
257 void setAAA(const MCInst &MI, unsigned OpNum) {
258 EVEX_aaa = getRegEncoding(MI, OpNum);
259 }
260 void setNF(bool V) { EVEX_aaa |= V << 2; }
261 void setSC(const MCInst &MI, unsigned OpNum) {
262 unsigned Encoding = MI.getOperand(i: OpNum).getImm();
263 EVEX_V2 = ~(Encoding >> 3) & 0x1;
264 EVEX_aaa = Encoding & 0x7;
265 }
266
267 X86OpcodePrefixHelper(const MCRegisterInfo &MRI)
268 : W(0), R(0), X(0), B(0), M(0), R2(0), X2(0), B2(0), VEX_4V(0), VEX_L(0),
269 VEX_PP(0), VEX_5M(0), EVEX_z(0), EVEX_L2(0), EVEX_b(0), EVEX_V2(0),
270 EVEX_aaa(0), MRI(MRI) {}
271
272 void setLowerBound(PrefixKind K) { Kind = K; }
273
274 PrefixKind determineOptimalKind() {
275 switch (Kind) {
276 case None:
277 // Not M bit here by intention b/c
278 // 1. No guarantee that REX2 is supported by arch w/o explict EGPR
279 // 2. REX2 is longer than 0FH
280 Kind = (R2 | X2 | B2) ? REX2 : (W | R | X | B) ? REX : None;
281 break;
282 case REX:
283 Kind = (R2 | X2 | B2) ? REX2 : REX;
284 break;
285 case REX2:
286 case XOP:
287 case VEX3:
288 case EVEX:
289 break;
290 case VEX2:
291 Kind = (W | X | B | (VEX_5M != 1)) ? VEX3 : VEX2;
292 break;
293 }
294 return Kind;
295 }
296
297 void emit(SmallVectorImpl<char> &CB) const {
298 uint8_t FirstPayload =
299 ((~R) & 0x1) << 7 | ((~X) & 0x1) << 6 | ((~B) & 0x1) << 5;
300 uint8_t LastPayload = ((~VEX_4V) & 0xf) << 3 | VEX_L << 2 | VEX_PP;
301 switch (Kind) {
302 case None:
303 return;
304 case REX:
305 emitByte(C: 0x40 | W << 3 | R << 2 | X << 1 | B, CB);
306 return;
307 case REX2:
308 emitByte(C: 0xD5, CB);
309 emitByte(C: M << 7 | R2 << 6 | X2 << 5 | B2 << 4 | W << 3 | R << 2 | X << 1 |
310 B,
311 CB);
312 return;
313 case VEX2:
314 emitByte(C: 0xC5, CB);
315 emitByte(C: ((~R) & 1) << 7 | LastPayload, CB);
316 return;
317 case VEX3:
318 case XOP:
319 emitByte(C: Kind == VEX3 ? 0xC4 : 0x8F, CB);
320 emitByte(C: FirstPayload | VEX_5M, CB);
321 emitByte(C: W << 7 | LastPayload, CB);
322 return;
323 case EVEX:
324 assert(VEX_5M && !(VEX_5M & 0x8) && "invalid mmm fields for EVEX!");
325 emitByte(C: 0x62, CB);
326 emitByte(C: FirstPayload | ((~R2) & 0x1) << 4 | B2 << 3 | VEX_5M, CB);
327 emitByte(C: W << 7 | ((~VEX_4V) & 0xf) << 3 | ((~X2) & 0x1) << 2 | VEX_PP,
328 CB);
329 emitByte(C: EVEX_z << 7 | EVEX_L2 << 6 | VEX_L << 5 | EVEX_b << 4 |
330 ((~EVEX_V2) & 0x1) << 3 | EVEX_aaa,
331 CB);
332 return;
333 }
334 }
335};
336
337class X86MCCodeEmitter : public MCCodeEmitter {
338 const MCInstrInfo &MCII;
339 MCContext &Ctx;
340
341public:
342 X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
343 : MCII(mcii), Ctx(ctx) {}
344 X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;
345 X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;
346 ~X86MCCodeEmitter() override = default;
347
348 void emitPrefix(const MCInst &MI, SmallVectorImpl<char> &CB,
349 const MCSubtargetInfo &STI) const;
350
351 void encodeInstruction(const MCInst &MI, SmallVectorImpl<char> &CB,
352 SmallVectorImpl<MCFixup> &Fixups,
353 const MCSubtargetInfo &STI) const override;
354
355private:
356 unsigned getX86RegNum(const MCOperand &MO) const;
357
358 unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;
359
360 void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned FixupKind,
361 bool IsPCRel, uint64_t StartByte,
362 SmallVectorImpl<char> &CB,
363 SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
364
365 void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,
366 SmallVectorImpl<char> &CB) const;
367
368 void emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
369 SmallVectorImpl<char> &CB) const;
370
371 void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
372 uint64_t TSFlags, PrefixKind Kind, uint64_t StartByte,
373 SmallVectorImpl<char> &CB,
374 SmallVectorImpl<MCFixup> &Fixups,
375 const MCSubtargetInfo &STI,
376 bool ForceSIB = false) const;
377
378 PrefixKind emitPrefixImpl(const MCInst &MI, const MCSubtargetInfo &STI,
379 SmallVectorImpl<char> &CB) const;
380
381 PrefixKind emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
382 const MCSubtargetInfo &STI,
383 SmallVectorImpl<char> &CB) const;
384
385 void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI,
386 SmallVectorImpl<char> &CB) const;
387
388 PrefixKind emitOpcodePrefix(int MemOperand, const MCInst &MI,
389 const MCSubtargetInfo &STI,
390 SmallVectorImpl<char> &CB) const;
391
392 PrefixKind emitREXPrefix(int MemOperand, const MCInst &MI,
393 const MCSubtargetInfo &STI,
394 SmallVectorImpl<char> &CB) const;
395};
396
397} // end anonymous namespace
398
399static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {
400 assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
401 return RM | (RegOpcode << 3) | (Mod << 6);
402}
403
404static void emitConstant(uint64_t Val, unsigned Size,
405 SmallVectorImpl<char> &CB) {
406 // Output the constant in little endian byte order.
407 for (unsigned i = 0; i != Size; ++i) {
408 emitByte(C: Val & 255, CB);
409 Val >>= 8;
410 }
411}
412
413/// \returns the appropriate fixup kind to use for an immediate in an
414/// instruction with the specified TSFlags.
415static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
416 unsigned Size = X86II::getSizeOfImm(TSFlags);
417 if (X86II::isImmSigned(TSFlags)) {
418 switch (Size) {
419 default:
420 llvm_unreachable("Unsupported signed fixup size!");
421 case 4:
422 return X86::reloc_signed_4byte;
423 }
424 }
425 switch (Size) {
426 default:
427 llvm_unreachable("Invalid generic fixup size!");
428 case 1:
429 return FK_Data_1;
430 case 2:
431 return FK_Data_2;
432 case 4:
433 return FK_Data_4;
434 case 8:
435 return FK_Data_8;
436 }
437}
438
439enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };
440
441/// Check if this expression starts with _GLOBAL_OFFSET_TABLE_ and if it is
442/// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on
443/// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
444/// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a
445/// binary expression.
446///
447/// TODO: Move this to X86AsmBackend.cpp at relocation decision phase so that we
448/// don't have to mess with MCExpr.
449static GlobalOffsetTableExprKind
450startsWithGlobalOffsetTable(const MCExpr *Expr) {
451 const MCExpr *RHS = nullptr;
452 if (Expr->getKind() == MCExpr::Binary) {
453 const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);
454 Expr = BE->getLHS();
455 RHS = BE->getRHS();
456 }
457
458 if (Expr->getKind() != MCExpr::SymbolRef)
459 return GOT_None;
460
461 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
462 const MCSymbol &S = Ref->getSymbol();
463 if (S.getName() != "_GLOBAL_OFFSET_TABLE_")
464 return GOT_None;
465 if (RHS && RHS->getKind() == MCExpr::SymbolRef)
466 return GOT_SymDiff;
467 return GOT_Normal;
468}
469
470static bool hasSecRelSymbolRef(const MCExpr *Expr) {
471 if (Expr->getKind() == MCExpr::SymbolRef) {
472 auto *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
473 return Ref->getSpecifier() == X86::S_COFF_SECREL;
474 }
475 return false;
476}
477
478static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
479 unsigned Opcode = MI.getOpcode();
480 const MCInstrDesc &Desc = MCII.get(Opcode);
481 if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
482 Opcode != X86::JCC_4) ||
483 !(getImmFixupKind(TSFlags: Desc.TSFlags) == FK_Data_4 &&
484 X86II::isImmPCRel(TSFlags: Desc.TSFlags)))
485 return false;
486
487 unsigned CurOp = X86II::getOperandBias(Desc);
488 const MCOperand &Op = MI.getOperand(i: CurOp);
489 if (!Op.isExpr())
490 return false;
491
492 auto *Ref = dyn_cast<MCSymbolRefExpr>(Val: Op.getExpr());
493 return Ref && Ref->getSpecifier() == X86::S_None;
494}
495
496unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {
497 return Ctx.getRegisterInfo()->getEncodingValue(Reg: MO.getReg()) & 0x7;
498}
499
500unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,
501 unsigned OpNum) const {
502 return Ctx.getRegisterInfo()->getEncodingValue(Reg: MI.getOperand(i: OpNum).getReg());
503}
504
505void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
506 unsigned FixupKind, bool PCRel,
507 uint64_t StartByte,
508 SmallVectorImpl<char> &CB,
509 SmallVectorImpl<MCFixup> &Fixups,
510 int ImmOffset) const {
511 unsigned Size = 4;
512 switch (FixupKind) {
513 case FK_Data_1:
514 Size = 1;
515 break;
516 case FK_Data_2:
517 Size = 2;
518 break;
519 case FK_Data_8:
520 Size = 8;
521 break;
522 }
523 const MCExpr *Expr = nullptr;
524 if (DispOp.isImm()) {
525 // If this is a simple integer displacement that doesn't require a
526 // relocation, emit it now.
527 if (!(is_contained(Set: {FK_Data_1, FK_Data_2, FK_Data_4}, Element: FixupKind) &&
528 PCRel)) {
529 emitConstant(Val: DispOp.getImm() + ImmOffset, Size, CB);
530 return;
531 }
532 Expr = MCConstantExpr::create(Value: DispOp.getImm(), Ctx);
533 } else {
534 Expr = DispOp.getExpr();
535 }
536
537 // If we have an immoffset, add it to the expression.
538 if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
539 FixupKind == X86::reloc_signed_4byte)) {
540 GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
541 if (Kind != GOT_None) {
542 assert(ImmOffset == 0);
543
544 if (Size == 8) {
545 FixupKind = FirstLiteralRelocationKind + ELF::R_X86_64_GOTPC64;
546 } else {
547 assert(Size == 4);
548 FixupKind = X86::reloc_global_offset_table;
549 }
550
551 if (Kind == GOT_Normal)
552 ImmOffset = static_cast<int>(CB.size() - StartByte);
553 } else if (Expr->getKind() == MCExpr::SymbolRef) {
554 if (hasSecRelSymbolRef(Expr)) {
555 FixupKind = FK_SecRel_4;
556 }
557 } else if (Expr->getKind() == MCExpr::Binary) {
558 const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
559 if (hasSecRelSymbolRef(Expr: Bin->getLHS()) ||
560 hasSecRelSymbolRef(Expr: Bin->getRHS())) {
561 FixupKind = FK_SecRel_4;
562 }
563 }
564 }
565
566 if (ImmOffset)
567 Expr = MCBinaryExpr::createAdd(LHS: Expr, RHS: MCConstantExpr::create(Value: ImmOffset, Ctx),
568 Ctx, Loc: Expr->getLoc());
569
570 // Emit a symbolic constant as a fixup and a few zero bytes.
571 Fixups.push_back(Elt: MCFixup::create(Offset: static_cast<uint32_t>(CB.size() - StartByte),
572 Value: Expr, Kind: FixupKind, PCRel));
573 emitConstant(Val: 0, Size, CB);
574}
575
576void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg,
577 unsigned RegOpcodeFld,
578 SmallVectorImpl<char> &CB) const {
579 emitByte(C: modRMByte(Mod: 3, RegOpcode: RegOpcodeFld, RM: getX86RegNum(MO: ModRMReg)), CB);
580}
581
582void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
583 SmallVectorImpl<char> &CB) const {
584 // SIB byte is in the same format as the modRMByte.
585 emitByte(C: modRMByte(Mod: SS, RegOpcode: Index, RM: Base), CB);
586}
587
588void X86MCCodeEmitter::emitMemModRMByte(
589 const MCInst &MI, unsigned Op, unsigned RegOpcodeField, uint64_t TSFlags,
590 PrefixKind Kind, uint64_t StartByte, SmallVectorImpl<char> &CB,
591 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI,
592 bool ForceSIB) const {
593 const MCOperand &Disp = MI.getOperand(i: Op + X86::AddrDisp);
594 const MCOperand &Base = MI.getOperand(i: Op + X86::AddrBaseReg);
595 const MCOperand &Scale = MI.getOperand(i: Op + X86::AddrScaleAmt);
596 const MCOperand &IndexReg = MI.getOperand(i: Op + X86::AddrIndexReg);
597 MCRegister BaseReg = Base.getReg();
598
599 // Handle %rip relative addressing.
600 if (BaseReg == X86::RIP ||
601 BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode
602 assert(STI.hasFeature(X86::Is64Bit) &&
603 "Rip-relative addressing requires 64-bit mode");
604 assert(!IndexReg.getReg() && !ForceSIB && "Invalid rip-relative address");
605 emitByte(C: modRMByte(Mod: 0, RegOpcode: RegOpcodeField, RM: 5), CB);
606
607 unsigned Opcode = MI.getOpcode();
608 unsigned FixupKind = [&]() {
609 // Enable relaxed relocation only for a MCSymbolRefExpr. We cannot use a
610 // relaxed relocation if an offset is present (e.g. x@GOTPCREL+4).
611 if (!(Disp.isExpr() && isa<MCSymbolRefExpr>(Val: Disp.getExpr())))
612 return X86::reloc_riprel_4byte;
613
614 // Certain loads for GOT references can be relocated against the symbol
615 // directly if the symbol ends up in the same linkage unit.
616 switch (Opcode) {
617 default:
618 return X86::reloc_riprel_4byte;
619 case X86::MOV64rm:
620 // movq loads is a subset of reloc_riprel_4byte_relax_rex/rex2. It is a
621 // special case because COFF and Mach-O don't support ELF's more
622 // flexible R_X86_64_REX_GOTPCRELX/R_X86_64_CODE_4_GOTPCRELX relaxation.
623 return Kind == REX2 ? X86::reloc_riprel_4byte_movq_load_rex2
624 : X86::reloc_riprel_4byte_movq_load;
625 case X86::ADC32rm:
626 case X86::ADD32rm:
627 case X86::AND32rm:
628 case X86::CMP32rm:
629 case X86::MOV32rm:
630 case X86::OR32rm:
631 case X86::SBB32rm:
632 case X86::SUB32rm:
633 case X86::TEST32mr:
634 case X86::XOR32rm:
635 case X86::CALL64m:
636 case X86::JMP64m:
637 case X86::TAILJMPm64:
638 case X86::TEST64mr:
639 case X86::ADC64rm:
640 case X86::ADD64rm:
641 case X86::AND64rm:
642 case X86::CMP64rm:
643 case X86::OR64rm:
644 case X86::SBB64rm:
645 case X86::SUB64rm:
646 case X86::XOR64rm:
647 case X86::LEA64r:
648 return Kind == REX2 ? X86::reloc_riprel_4byte_relax_rex2
649 : Kind == REX ? X86::reloc_riprel_4byte_relax_rex
650 : X86::reloc_riprel_4byte_relax;
651 case X86::ADD64rm_NF:
652 case X86::ADD64rm_ND:
653 case X86::ADD64mr_ND:
654 case X86::ADD64mr_NF_ND:
655 case X86::ADD64rm_NF_ND:
656 return X86::reloc_riprel_4byte_relax_evex;
657 }
658 }();
659
660 // rip-relative addressing is actually relative to the *next* instruction.
661 // Since an immediate can follow the mod/rm byte for an instruction, this
662 // means that we need to bias the displacement field of the instruction with
663 // the size of the immediate field. If we have this case, add it into the
664 // expression to emit.
665 // Note: rip-relative addressing using immediate displacement values should
666 // not be adjusted, assuming it was the user's intent.
667 int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags)
668 ? X86II::getSizeOfImm(TSFlags)
669 : 0;
670
671 emitImmediate(DispOp: Disp, Loc: MI.getLoc(), FixupKind, PCRel: true, StartByte, CB, Fixups,
672 ImmOffset: -ImmSize);
673 return;
674 }
675
676 unsigned BaseRegNo = BaseReg ? getX86RegNum(MO: Base) : -1U;
677
678 bool IsAdSize16 = STI.hasFeature(Feature: X86::Is32Bit) &&
679 (TSFlags & X86II::AdSizeMask) == X86II::AdSize16;
680
681 // 16-bit addressing forms of the ModR/M byte have a different encoding for
682 // the R/M field and are far more limited in which registers can be used.
683 if (IsAdSize16 || X86_MC::is16BitMemOperand(MI, Op, STI)) {
684 if (BaseReg) {
685 // For 32-bit addressing, the row and column values in Table 2-2 are
686 // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
687 // some special cases. And getX86RegNum reflects that numbering.
688 // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,
689 // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only
690 // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,
691 // while values 0-3 indicate the allowed combinations (base+index) of
692 // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.
693 //
694 // R16Table[] is a lookup from the normal RegNo, to the row values from
695 // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.
696 static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5};
697 unsigned RMfield = R16Table[BaseRegNo];
698
699 assert(RMfield && "invalid 16-bit base register");
700
701 if (IndexReg.getReg()) {
702 unsigned IndexReg16 = R16Table[getX86RegNum(MO: IndexReg)];
703
704 assert(IndexReg16 && "invalid 16-bit index register");
705 // We must have one of SI/DI (4,5), and one of BP/BX (6,7).
706 assert(((IndexReg16 ^ RMfield) & 2) &&
707 "invalid 16-bit base/index register combination");
708 assert(Scale.getImm() == 1 &&
709 "invalid scale for 16-bit memory reference");
710
711 // Allow base/index to appear in either order (although GAS doesn't).
712 if (IndexReg16 & 2)
713 RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);
714 else
715 RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
716 }
717
718 if (Disp.isImm() && isInt<8>(x: Disp.getImm())) {
719 if (Disp.getImm() == 0 && RMfield != 6) {
720 // There is no displacement; just the register.
721 emitByte(C: modRMByte(Mod: 0, RegOpcode: RegOpcodeField, RM: RMfield), CB);
722 return;
723 }
724 // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
725 emitByte(C: modRMByte(Mod: 1, RegOpcode: RegOpcodeField, RM: RMfield), CB);
726 emitImmediate(DispOp: Disp, Loc: MI.getLoc(), FixupKind: FK_Data_1, PCRel: false, StartByte, CB,
727 Fixups);
728 return;
729 }
730 // This is the [REG]+disp16 case.
731 emitByte(C: modRMByte(Mod: 2, RegOpcode: RegOpcodeField, RM: RMfield), CB);
732 } else {
733 assert(!IndexReg.getReg() && "Unexpected index register!");
734 // There is no BaseReg; this is the plain [disp16] case.
735 emitByte(C: modRMByte(Mod: 0, RegOpcode: RegOpcodeField, RM: 6), CB);
736 }
737
738 // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
739 emitImmediate(DispOp: Disp, Loc: MI.getLoc(), FixupKind: FK_Data_2, PCRel: false, StartByte, CB, Fixups);
740 return;
741 }
742
743 // Check for presence of {disp8} or {disp32} pseudo prefixes.
744 bool UseDisp8 = MI.getFlags() & X86::IP_USE_DISP8;
745 bool UseDisp32 = MI.getFlags() & X86::IP_USE_DISP32;
746
747 // We only allow no displacement if no pseudo prefix is present.
748 bool AllowNoDisp = !UseDisp8 && !UseDisp32;
749 // Disp8 is allowed unless the {disp32} prefix is present.
750 bool AllowDisp8 = !UseDisp32;
751
752 // Determine whether a SIB byte is needed.
753 if (!ForceSIB && !X86II::needSIB(BaseReg, IndexReg: IndexReg.getReg(),
754 In64BitMode: STI.hasFeature(Feature: X86::Is64Bit))) {
755 if (!BaseReg) { // [disp32] in X86-32 mode
756 emitByte(C: modRMByte(Mod: 0, RegOpcode: RegOpcodeField, RM: 5), CB);
757 emitImmediate(DispOp: Disp, Loc: MI.getLoc(), FixupKind: FK_Data_4, PCRel: false, StartByte, CB, Fixups);
758 return;
759 }
760
761 // If the base is not EBP/ESP/R12/R13/R20/R21/R28/R29 and there is no
762 // displacement, use simple indirect register encoding, this handles
763 // addresses like [EAX]. The encoding for [EBP], [R13], [R20], [R21], [R28]
764 // or [R29] with no displacement means [disp32] so we handle it by emitting
765 // a displacement of 0 later.
766 if (BaseRegNo != N86::EBP) {
767 if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp) {
768 emitByte(C: modRMByte(Mod: 0, RegOpcode: RegOpcodeField, RM: BaseRegNo), CB);
769 return;
770 }
771
772 // If the displacement is @tlscall, treat it as a zero.
773 if (Disp.isExpr()) {
774 auto *Sym = dyn_cast<MCSymbolRefExpr>(Val: Disp.getExpr());
775 if (Sym && Sym->getSpecifier() == X86::S_TLSCALL) {
776 // This is exclusively used by call *a@tlscall(base). The relocation
777 // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
778 Fixups.push_back(Elt: MCFixup::create(Offset: 0, Value: Sym, Kind: FK_NONE));
779 emitByte(C: modRMByte(Mod: 0, RegOpcode: RegOpcodeField, RM: BaseRegNo), CB);
780 return;
781 }
782 }
783 }
784
785 // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
786 // Including a compressed disp8 for EVEX instructions that support it.
787 // This also handles the 0 displacement for [EBP], [R13], [R21] or [R29]. We
788 // can't use disp8 if the {disp32} pseudo prefix is present.
789 if (Disp.isImm() && AllowDisp8) {
790 int ImmOffset = 0;
791 if (X86II::isDispOrCDisp8(TSFlags, Value: Disp.getImm(), ImmOffset: &ImmOffset)) {
792 emitByte(C: modRMByte(Mod: 1, RegOpcode: RegOpcodeField, RM: BaseRegNo), CB);
793 emitImmediate(DispOp: Disp, Loc: MI.getLoc(), FixupKind: FK_Data_1, PCRel: false, StartByte, CB,
794 Fixups, ImmOffset);
795 return;
796 }
797 }
798
799 // Otherwise, emit the most general non-SIB encoding: [REG+disp32].
800 // Displacement may be 0 for [EBP], [R13], [R21], [R29] case if {disp32}
801 // pseudo prefix prevented using disp8 above.
802 emitByte(C: modRMByte(Mod: 2, RegOpcode: RegOpcodeField, RM: BaseRegNo), CB);
803 unsigned Opcode = MI.getOpcode();
804 unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
805 : X86::reloc_signed_4byte;
806 emitImmediate(DispOp: Disp, Loc: MI.getLoc(), FixupKind: MCFixupKind(FixupKind), PCRel: false, StartByte,
807 CB, Fixups);
808 return;
809 }
810
811 // We need a SIB byte, so start by outputting the ModR/M byte first
812 assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP &&
813 "Cannot use ESP as index reg!");
814
815 bool ForceDisp32 = false;
816 bool ForceDisp8 = false;
817 int ImmOffset = 0;
818 if (!BaseReg) {
819 // If there is no base register, we emit the special case SIB byte with
820 // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
821 BaseRegNo = 5;
822 emitByte(C: modRMByte(Mod: 0, RegOpcode: RegOpcodeField, RM: 4), CB);
823 ForceDisp32 = true;
824 } else if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp &&
825 // Base reg can't be EBP/RBP/R13/R21/R29 as that would end up with
826 // '5' as the base field, but that is the magic [*] nomenclature
827 // that indicates no base when mod=0. For these cases we'll emit a
828 // 0 displacement instead.
829 BaseRegNo != N86::EBP) {
830 // Emit no displacement ModR/M byte
831 emitByte(C: modRMByte(Mod: 0, RegOpcode: RegOpcodeField, RM: 4), CB);
832 } else if (Disp.isImm() && AllowDisp8 &&
833 X86II::isDispOrCDisp8(TSFlags, Value: Disp.getImm(), ImmOffset: &ImmOffset)) {
834 // Displacement fits in a byte or matches an EVEX compressed disp8, use
835 // disp8 encoding. This also handles EBP/R13/R21/R29 base with 0
836 // displacement unless {disp32} pseudo prefix was used.
837 emitByte(C: modRMByte(Mod: 1, RegOpcode: RegOpcodeField, RM: 4), CB);
838 ForceDisp8 = true;
839 } else {
840 // Otherwise, emit the normal disp32 encoding.
841 emitByte(C: modRMByte(Mod: 2, RegOpcode: RegOpcodeField, RM: 4), CB);
842 ForceDisp32 = true;
843 }
844
845 // Calculate what the SS field value should be...
846 static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};
847 unsigned SS = SSTable[Scale.getImm()];
848
849 unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(MO: IndexReg) : 4;
850
851 emitSIBByte(SS, Index: IndexRegNo, Base: BaseRegNo, CB);
852
853 // Do we need to output a displacement?
854 if (ForceDisp8)
855 emitImmediate(DispOp: Disp, Loc: MI.getLoc(), FixupKind: FK_Data_1, PCRel: false, StartByte, CB, Fixups,
856 ImmOffset);
857 else if (ForceDisp32)
858 emitImmediate(DispOp: Disp, Loc: MI.getLoc(), FixupKind: X86::reloc_signed_4byte, PCRel: false, StartByte,
859 CB, Fixups);
860}
861
862/// Emit all instruction prefixes.
863///
864/// \returns one of the REX, XOP, VEX2, VEX3, EVEX if any of them is used,
865/// otherwise returns None.
866PrefixKind X86MCCodeEmitter::emitPrefixImpl(const MCInst &MI,
867 const MCSubtargetInfo &STI,
868 SmallVectorImpl<char> &CB) const {
869 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
870 uint64_t TSFlags = Desc.TSFlags;
871 // Determine where the memory operand starts, if present.
872 int MemoryOperand = X86II::getMemoryOperandIdx(Desc);
873 // Emit segment override opcode prefix as needed.
874 if (MemoryOperand != -1)
875 emitSegmentOverridePrefix(SegOperand: MemoryOperand + X86::AddrSegmentReg, MI, CB);
876
877 // Emit the repeat opcode prefix as needed.
878 unsigned Flags = MI.getFlags();
879 if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT)
880 emitByte(C: 0xF3, CB);
881 if (Flags & X86::IP_HAS_REPEAT_NE)
882 emitByte(C: 0xF2, CB);
883
884 // Emit the address size opcode prefix as needed.
885 if (X86_MC::needsAddressSizeOverride(MI, STI, MemoryOperand, TSFlags) ||
886 Flags & X86::IP_HAS_AD_SIZE)
887 emitByte(C: 0x67, CB);
888
889 uint64_t Form = TSFlags & X86II::FormMask;
890 switch (Form) {
891 default:
892 break;
893 case X86II::RawFrmDstSrc:
894 // Emit segment override opcode prefix as needed (not for %ds).
895 if (MI.getOperand(i: 2).getReg() != X86::DS)
896 emitSegmentOverridePrefix(SegOperand: 2, MI, CB);
897 break;
898 case X86II::RawFrmSrc:
899 // Emit segment override opcode prefix as needed (not for %ds).
900 if (MI.getOperand(i: 1).getReg() != X86::DS)
901 emitSegmentOverridePrefix(SegOperand: 1, MI, CB);
902 break;
903 case X86II::RawFrmMemOffs:
904 // Emit segment override opcode prefix as needed.
905 emitSegmentOverridePrefix(SegOperand: 1, MI, CB);
906 break;
907 }
908
909 // REX prefix is optional, but if used must be immediately before the opcode
910 // Encoding type for this instruction.
911 return (TSFlags & X86II::EncodingMask)
912 ? emitVEXOpcodePrefix(MemOperand: MemoryOperand, MI, STI, CB)
913 : emitOpcodePrefix(MemOperand: MemoryOperand, MI, STI, CB);
914}
915
916// AVX instructions are encoded using an encoding scheme that combines
917// prefix bytes, opcode extension field, operand encoding fields, and vector
918// length encoding capability into a new prefix, referred to as VEX.
919
920// The majority of the AVX-512 family of instructions (operating on
921// 512/256/128-bit vector register operands) are encoded using a new prefix
922// (called EVEX).
923
924// XOP is a revised subset of what was originally intended as SSE5. It was
925// changed to be similar but not overlapping with AVX.
926
927/// Emit XOP, VEX2, VEX3 or EVEX prefix.
928/// \returns the used prefix.
929PrefixKind
930X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
931 const MCSubtargetInfo &STI,
932 SmallVectorImpl<char> &CB) const {
933 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
934 uint64_t TSFlags = Desc.TSFlags;
935
936 assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");
937
938#ifndef NDEBUG
939 unsigned NumOps = MI.getNumOperands();
940 for (unsigned I = NumOps ? X86II::getOperandBias(Desc) : 0; I != NumOps;
941 ++I) {
942 const MCOperand &MO = MI.getOperand(I);
943 if (!MO.isReg())
944 continue;
945 MCRegister Reg = MO.getReg();
946 if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
947 report_fatal_error(
948 "Cannot encode high byte register in VEX/EVEX-prefixed instruction");
949 }
950#endif
951
952 X86OpcodePrefixHelper Prefix(*Ctx.getRegisterInfo());
953 switch (TSFlags & X86II::EncodingMask) {
954 default:
955 break;
956 case X86II::XOP:
957 Prefix.setLowerBound(XOP);
958 break;
959 case X86II::VEX:
960 // VEX can be 2 byte or 3 byte, not determined yet if not explicit
961 Prefix.setLowerBound((MI.getFlags() & X86::IP_USE_VEX3) ? VEX3 : VEX2);
962 break;
963 case X86II::EVEX:
964 Prefix.setLowerBound(EVEX);
965 break;
966 }
967
968 Prefix.setW(TSFlags & X86II::REX_W);
969 Prefix.setNF(TSFlags & X86II::EVEX_NF);
970
971 bool HasEVEX_K = TSFlags & X86II::EVEX_K;
972 bool HasVEX_4V = TSFlags & X86II::VEX_4V;
973 bool IsND = X86II::hasNewDataDest(TSFlags); // IsND implies HasVEX_4V
974 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
975
976 switch (TSFlags & X86II::OpMapMask) {
977 default:
978 llvm_unreachable("Invalid prefix!");
979 case X86II::TB:
980 Prefix.set5M(0x1); // 0F
981 break;
982 case X86II::T8:
983 Prefix.set5M(0x2); // 0F 38
984 break;
985 case X86II::TA:
986 Prefix.set5M(0x3); // 0F 3A
987 break;
988 case X86II::XOP8:
989 Prefix.set5M(0x8);
990 break;
991 case X86II::XOP9:
992 Prefix.set5M(0x9);
993 break;
994 case X86II::XOPA:
995 Prefix.set5M(0xA);
996 break;
997 case X86II::T_MAP4:
998 Prefix.set5M(0x4);
999 break;
1000 case X86II::T_MAP5:
1001 Prefix.set5M(0x5);
1002 break;
1003 case X86II::T_MAP6:
1004 Prefix.set5M(0x6);
1005 break;
1006 case X86II::T_MAP7:
1007 Prefix.set5M(0x7);
1008 break;
1009 }
1010
1011 Prefix.setL(TSFlags & X86II::VEX_L);
1012 Prefix.setL2(TSFlags & X86II::EVEX_L2);
1013 switch (TSFlags & X86II::OpPrefixMask) {
1014 case X86II::PD:
1015 Prefix.setPP(0x1); // 66
1016 break;
1017 case X86II::XS:
1018 Prefix.setPP(0x2); // F3
1019 break;
1020 case X86II::XD:
1021 Prefix.setPP(0x3); // F2
1022 break;
1023 }
1024
1025 Prefix.setZ(HasEVEX_K && (TSFlags & X86II::EVEX_Z));
1026 Prefix.setEVEX_b(TSFlags & X86II::EVEX_B);
1027 Prefix.setEVEX_U(TSFlags & X86II::EVEX_U);
1028
1029 bool EncodeRC = false;
1030 uint8_t EVEX_rc = 0;
1031
1032 unsigned CurOp = X86II::getOperandBias(Desc);
1033 bool HasTwoConditionalOps = TSFlags & X86II::TwoConditionalOps;
1034
1035 switch (TSFlags & X86II::FormMask) {
1036 default:
1037 llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
1038 case X86II::MRMDestMem4VOp3CC: {
1039 // src1(ModR/M), MemAddr, src2(VEX_4V)
1040 Prefix.setRR2(MI, OpNum: CurOp++);
1041 Prefix.setBB2(MI, OpNum: MemOperand + X86::AddrBaseReg);
1042 Prefix.setXX2(MI, OpNum: MemOperand + X86::AddrIndexReg);
1043 CurOp += X86::AddrNumOperands;
1044 Prefix.set4VV2(MI, OpNum: CurOp++);
1045 break;
1046 }
1047 case X86II::MRM_C0:
1048 case X86II::RawFrm:
1049 break;
1050 case X86II::MRMDestMemCC:
1051 case X86II::MRMDestMemFSIB:
1052 case X86II::MRMDestMem: {
1053 // MRMDestMem instructions forms:
1054 // MemAddr, src1(ModR/M)
1055 // MemAddr, src1(VEX_4V), src2(ModR/M)
1056 // MemAddr, src1(ModR/M), imm8
1057 //
1058 // NDD:
1059 // dst(VEX_4V), MemAddr, src1(ModR/M)
1060 Prefix.setBB2(MI, OpNum: MemOperand + X86::AddrBaseReg);
1061 Prefix.setXX2(MI, OpNum: MemOperand + X86::AddrIndexReg);
1062 Prefix.setV2(MI, OpNum: MemOperand + X86::AddrIndexReg, HasVEX_4V);
1063
1064 if (IsND)
1065 Prefix.set4VV2(MI, OpNum: CurOp++);
1066
1067 CurOp += X86::AddrNumOperands;
1068
1069 if (HasEVEX_K)
1070 Prefix.setAAA(MI, OpNum: CurOp++);
1071
1072 if (!IsND && HasVEX_4V)
1073 Prefix.set4VV2(MI, OpNum: CurOp++);
1074
1075 Prefix.setRR2(MI, OpNum: CurOp++);
1076 if (HasTwoConditionalOps) {
1077 Prefix.set4V(MI, OpNum: CurOp++, /*IsImm=*/true);
1078 Prefix.setSC(MI, OpNum: CurOp++);
1079 }
1080 break;
1081 }
1082 case X86II::MRMSrcMemCC:
1083 case X86II::MRMSrcMemFSIB:
1084 case X86II::MRMSrcMem: {
1085 // MRMSrcMem instructions forms:
1086 // src1(ModR/M), MemAddr
1087 // src1(ModR/M), src2(VEX_4V), MemAddr
1088 // src1(ModR/M), MemAddr, imm8
1089 // src1(ModR/M), MemAddr, src2(Imm[7:4])
1090 //
1091 // FMA4:
1092 // dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
1093 //
1094 // NDD:
1095 // dst(VEX_4V), src1(ModR/M), MemAddr
1096 if (IsND)
1097 Prefix.set4VV2(MI, OpNum: CurOp++);
1098
1099 Prefix.setRR2(MI, OpNum: CurOp++);
1100
1101 if (HasEVEX_K)
1102 Prefix.setAAA(MI, OpNum: CurOp++);
1103
1104 if (!IsND && HasVEX_4V)
1105 Prefix.set4VV2(MI, OpNum: CurOp++);
1106
1107 Prefix.setBB2(MI, OpNum: MemOperand + X86::AddrBaseReg);
1108 Prefix.setXX2(MI, OpNum: MemOperand + X86::AddrIndexReg);
1109 Prefix.setV2(MI, OpNum: MemOperand + X86::AddrIndexReg, HasVEX_4V);
1110 CurOp += X86::AddrNumOperands;
1111 if (HasTwoConditionalOps) {
1112 Prefix.set4V(MI, OpNum: CurOp++, /*IsImm=*/true);
1113 Prefix.setSC(MI, OpNum: CurOp++);
1114 }
1115 break;
1116 }
1117 case X86II::MRMSrcMem4VOp3: {
1118 // Instruction format for 4VOp3:
1119 // src1(ModR/M), MemAddr, src3(VEX_4V)
1120 Prefix.setRR2(MI, OpNum: CurOp++);
1121 Prefix.setBB2(MI, OpNum: MemOperand + X86::AddrBaseReg);
1122 Prefix.setXX2(MI, OpNum: MemOperand + X86::AddrIndexReg);
1123 Prefix.set4VV2(MI, OpNum: CurOp + X86::AddrNumOperands);
1124 break;
1125 }
1126 case X86II::MRMSrcMemOp4: {
1127 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1128 Prefix.setR(MI, OpNum: CurOp++);
1129 Prefix.set4V(MI, OpNum: CurOp++);
1130 Prefix.setBB2(MI, OpNum: MemOperand + X86::AddrBaseReg);
1131 Prefix.setXX2(MI, OpNum: MemOperand + X86::AddrIndexReg);
1132 break;
1133 }
1134 case X86II::MRMXmCC:
1135 case X86II::MRM0m:
1136 case X86II::MRM1m:
1137 case X86II::MRM2m:
1138 case X86II::MRM3m:
1139 case X86II::MRM4m:
1140 case X86II::MRM5m:
1141 case X86II::MRM6m:
1142 case X86II::MRM7m: {
1143 // MRM[0-9]m instructions forms:
1144 // MemAddr
1145 // src1(VEX_4V), MemAddr
1146 if (HasVEX_4V)
1147 Prefix.set4VV2(MI, OpNum: CurOp++);
1148
1149 if (HasEVEX_K)
1150 Prefix.setAAA(MI, OpNum: CurOp++);
1151
1152 Prefix.setBB2(MI, OpNum: MemOperand + X86::AddrBaseReg);
1153 Prefix.setXX2(MI, OpNum: MemOperand + X86::AddrIndexReg);
1154 Prefix.setV2(MI, OpNum: MemOperand + X86::AddrIndexReg, HasVEX_4V);
1155 CurOp += X86::AddrNumOperands + 1; // Skip first imm.
1156 if (HasTwoConditionalOps) {
1157 Prefix.set4V(MI, OpNum: CurOp++, /*IsImm=*/true);
1158 Prefix.setSC(MI, OpNum: CurOp++);
1159 }
1160 break;
1161 }
1162 case X86II::MRMSrcRegCC:
1163 case X86II::MRMSrcReg: {
1164 // MRMSrcReg instructions forms:
1165 // dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
1166 // dst(ModR/M), src1(ModR/M)
1167 // dst(ModR/M), src1(ModR/M), imm8
1168 //
1169 // FMA4:
1170 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1171 //
1172 // NDD:
1173 // dst(VEX_4V), src1(ModR/M.reg), src2(ModR/M)
1174 if (IsND)
1175 Prefix.set4VV2(MI, OpNum: CurOp++);
1176 Prefix.setRR2(MI, OpNum: CurOp++);
1177
1178 if (HasEVEX_K)
1179 Prefix.setAAA(MI, OpNum: CurOp++);
1180
1181 if (!IsND && HasVEX_4V)
1182 Prefix.set4VV2(MI, OpNum: CurOp++);
1183
1184 Prefix.setBB2(MI, OpNum: CurOp);
1185 Prefix.setX(MI, OpNum: CurOp, Shift: 4);
1186 ++CurOp;
1187
1188 if (HasTwoConditionalOps) {
1189 Prefix.set4V(MI, OpNum: CurOp++, /*IsImm=*/true);
1190 Prefix.setSC(MI, OpNum: CurOp++);
1191 }
1192
1193 if (TSFlags & X86II::EVEX_B) {
1194 if (HasEVEX_RC) {
1195 unsigned NumOps = Desc.getNumOperands();
1196 unsigned RcOperand = NumOps - 1;
1197 assert(RcOperand >= CurOp);
1198 EVEX_rc = MI.getOperand(i: RcOperand).getImm();
1199 assert(EVEX_rc <= 3 && "Invalid rounding control!");
1200 }
1201 EncodeRC = true;
1202 }
1203 break;
1204 }
1205 case X86II::MRMSrcReg4VOp3: {
1206 // Instruction format for 4VOp3:
1207 // src1(ModR/M), src2(ModR/M), src3(VEX_4V)
1208 Prefix.setRR2(MI, OpNum: CurOp++);
1209 Prefix.setBB2(MI, OpNum: CurOp++);
1210 Prefix.set4VV2(MI, OpNum: CurOp++);
1211 break;
1212 }
1213 case X86II::MRMSrcRegOp4: {
1214 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1215 Prefix.setR(MI, OpNum: CurOp++);
1216 Prefix.set4V(MI, OpNum: CurOp++);
1217 // Skip second register source (encoded in Imm[7:4])
1218 ++CurOp;
1219
1220 Prefix.setB(MI, OpNum: CurOp);
1221 Prefix.setX(MI, OpNum: CurOp, Shift: 4);
1222 ++CurOp;
1223 break;
1224 }
1225 case X86II::MRMDestRegCC:
1226 case X86II::MRMDestReg: {
1227 // MRMDestReg instructions forms:
1228 // dst(ModR/M), src(ModR/M)
1229 // dst(ModR/M), src(ModR/M), imm8
1230 // dst(ModR/M), src1(VEX_4V), src2(ModR/M)
1231 //
1232 // NDD:
1233 // dst(VEX_4V), src1(ModR/M), src2(ModR/M)
1234 if (IsND)
1235 Prefix.set4VV2(MI, OpNum: CurOp++);
1236 Prefix.setBB2(MI, OpNum: CurOp);
1237 Prefix.setX(MI, OpNum: CurOp, Shift: 4);
1238 ++CurOp;
1239
1240 if (HasEVEX_K)
1241 Prefix.setAAA(MI, OpNum: CurOp++);
1242
1243 if (!IsND && HasVEX_4V)
1244 Prefix.set4VV2(MI, OpNum: CurOp++);
1245
1246 Prefix.setRR2(MI, OpNum: CurOp++);
1247 if (HasTwoConditionalOps) {
1248 Prefix.set4V(MI, OpNum: CurOp++, /*IsImm=*/true);
1249 Prefix.setSC(MI, OpNum: CurOp++);
1250 }
1251 if (TSFlags & X86II::EVEX_B)
1252 EncodeRC = true;
1253 break;
1254 }
1255 case X86II::MRMr0: {
1256 // MRMr0 instructions forms:
1257 // 11:rrr:000
1258 // dst(ModR/M)
1259 Prefix.setRR2(MI, OpNum: CurOp++);
1260 break;
1261 }
1262 case X86II::MRMXrCC:
1263 case X86II::MRM0r:
1264 case X86II::MRM1r:
1265 case X86II::MRM2r:
1266 case X86II::MRM3r:
1267 case X86II::MRM4r:
1268 case X86II::MRM5r:
1269 case X86II::MRM6r:
1270 case X86II::MRM7r: {
1271 // MRM0r-MRM7r instructions forms:
1272 // dst(VEX_4V), src(ModR/M), imm8
1273 if (HasVEX_4V)
1274 Prefix.set4VV2(MI, OpNum: CurOp++);
1275
1276 if (HasEVEX_K)
1277 Prefix.setAAA(MI, OpNum: CurOp++);
1278
1279 Prefix.setBB2(MI, OpNum: CurOp);
1280 Prefix.setX(MI, OpNum: CurOp, Shift: 4);
1281 ++CurOp;
1282 if (HasTwoConditionalOps) {
1283 Prefix.set4V(MI, OpNum: ++CurOp, /*IsImm=*/true);
1284 Prefix.setSC(MI, OpNum: ++CurOp);
1285 }
1286 break;
1287 }
1288 }
1289 if (EncodeRC) {
1290 Prefix.setL(EVEX_rc & 0x1);
1291 Prefix.setL2(EVEX_rc & 0x2);
1292 }
1293 PrefixKind Kind = Prefix.determineOptimalKind();
1294 Prefix.emit(CB);
1295 return Kind;
1296}
1297
1298/// Emit REX prefix which specifies
1299/// 1) 64-bit instructions,
1300/// 2) non-default operand size, and
1301/// 3) use of X86-64 extended registers.
1302///
1303/// \returns the used prefix (REX or None).
1304PrefixKind X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,
1305 const MCSubtargetInfo &STI,
1306 SmallVectorImpl<char> &CB) const {
1307 if (!STI.hasFeature(Feature: X86::Is64Bit))
1308 return None;
1309 X86OpcodePrefixHelper Prefix(*Ctx.getRegisterInfo());
1310 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
1311 uint64_t TSFlags = Desc.TSFlags;
1312 Prefix.setW(TSFlags & X86II::REX_W);
1313 unsigned NumOps = MI.getNumOperands();
1314 bool UsesHighByteReg = false;
1315#ifndef NDEBUG
1316 bool HasRegOp = false;
1317#endif
1318 unsigned CurOp = NumOps ? X86II::getOperandBias(Desc) : 0;
1319 for (unsigned i = CurOp; i != NumOps; ++i) {
1320 const MCOperand &MO = MI.getOperand(i);
1321 if (MO.isReg()) {
1322#ifndef NDEBUG
1323 HasRegOp = true;
1324#endif
1325 MCRegister Reg = MO.getReg();
1326 if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
1327 UsesHighByteReg = true;
1328 // If it accesses SPL, BPL, SIL, or DIL, then it requires a REX prefix.
1329 if (X86II::isX86_64NonExtLowByteReg(Reg))
1330 Prefix.setLowerBound(REX);
1331 } else if (MO.isExpr() && STI.getTargetTriple().isX32()) {
1332 // GOTTPOFF and TLSDESC relocations require a REX prefix to allow
1333 // linker optimizations: even if the instructions we see may not require
1334 // any prefix, they may be replaced by instructions that do. This is
1335 // handled as a special case here so that it also works for hand-written
1336 // assembly without the user needing to write REX, as with GNU as.
1337 const auto *Ref = dyn_cast<MCSymbolRefExpr>(Val: MO.getExpr());
1338 if (Ref && (Ref->getSpecifier() == X86::S_GOTTPOFF ||
1339 Ref->getSpecifier() == X86::S_TLSDESC)) {
1340 Prefix.setLowerBound(REX);
1341 }
1342 }
1343 }
1344 if (MI.getFlags() & X86::IP_USE_REX)
1345 Prefix.setLowerBound(REX);
1346 if ((TSFlags & X86II::ExplicitOpPrefixMask) == X86II::ExplicitREX2Prefix ||
1347 MI.getFlags() & X86::IP_USE_REX2)
1348 Prefix.setLowerBound(REX2);
1349 switch (TSFlags & X86II::FormMask) {
1350 default:
1351 assert(!HasRegOp && "Unexpected form in emitREXPrefix!");
1352 break;
1353 case X86II::RawFrm:
1354 case X86II::RawFrmMemOffs:
1355 case X86II::RawFrmSrc:
1356 case X86II::RawFrmDst:
1357 case X86II::RawFrmDstSrc:
1358 break;
1359 case X86II::AddRegFrm:
1360 Prefix.setBB2(MI, OpNum: CurOp++);
1361 break;
1362 case X86II::MRMSrcReg:
1363 case X86II::MRMSrcRegCC:
1364 Prefix.setRR2(MI, OpNum: CurOp++);
1365 Prefix.setBB2(MI, OpNum: CurOp++);
1366 break;
1367 case X86II::MRMSrcMem:
1368 case X86II::MRMSrcMemCC:
1369 Prefix.setRR2(MI, OpNum: CurOp++);
1370 Prefix.setBB2(MI, OpNum: MemOperand + X86::AddrBaseReg);
1371 Prefix.setXX2(MI, OpNum: MemOperand + X86::AddrIndexReg);
1372 CurOp += X86::AddrNumOperands;
1373 break;
1374 case X86II::MRMDestReg:
1375 Prefix.setBB2(MI, OpNum: CurOp++);
1376 Prefix.setRR2(MI, OpNum: CurOp++);
1377 break;
1378 case X86II::MRMDestMem:
1379 Prefix.setBB2(MI, OpNum: MemOperand + X86::AddrBaseReg);
1380 Prefix.setXX2(MI, OpNum: MemOperand + X86::AddrIndexReg);
1381 CurOp += X86::AddrNumOperands;
1382 Prefix.setRR2(MI, OpNum: CurOp++);
1383 break;
1384 case X86II::MRMXmCC:
1385 case X86II::MRMXm:
1386 case X86II::MRM0m:
1387 case X86II::MRM1m:
1388 case X86II::MRM2m:
1389 case X86II::MRM3m:
1390 case X86II::MRM4m:
1391 case X86II::MRM5m:
1392 case X86II::MRM6m:
1393 case X86II::MRM7m:
1394 Prefix.setBB2(MI, OpNum: MemOperand + X86::AddrBaseReg);
1395 Prefix.setXX2(MI, OpNum: MemOperand + X86::AddrIndexReg);
1396 break;
1397 case X86II::MRMXrCC:
1398 case X86II::MRMXr:
1399 case X86II::MRM0r:
1400 case X86II::MRM1r:
1401 case X86II::MRM2r:
1402 case X86II::MRM3r:
1403 case X86II::MRM4r:
1404 case X86II::MRM5r:
1405 case X86II::MRM6r:
1406 case X86II::MRM7r:
1407 Prefix.setBB2(MI, OpNum: CurOp++);
1408 break;
1409 }
1410 Prefix.setM((TSFlags & X86II::OpMapMask) == X86II::TB);
1411 PrefixKind Kind = Prefix.determineOptimalKind();
1412 if (Kind && UsesHighByteReg)
1413 report_fatal_error(
1414 reason: "Cannot encode high byte register in REX-prefixed instruction");
1415 Prefix.emit(CB);
1416 return Kind;
1417}
1418
1419/// Emit segment override opcode prefix as needed.
1420void X86MCCodeEmitter::emitSegmentOverridePrefix(
1421 unsigned SegOperand, const MCInst &MI, SmallVectorImpl<char> &CB) const {
1422 // Check for explicit segment override on memory operand.
1423 if (MCRegister Reg = MI.getOperand(i: SegOperand).getReg())
1424 emitByte(C: X86::getSegmentOverridePrefixForReg(Reg), CB);
1425}
1426
1427/// Emit all instruction prefixes prior to the opcode.
1428///
1429/// \param MemOperand the operand # of the start of a memory operand if present.
1430/// If not present, it is -1.
1431///
1432/// \returns the used prefix (REX or None).
1433PrefixKind X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI,
1434 const MCSubtargetInfo &STI,
1435 SmallVectorImpl<char> &CB) const {
1436 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
1437 uint64_t TSFlags = Desc.TSFlags;
1438
1439 // Emit the operand size opcode prefix as needed.
1440 if ((TSFlags & X86II::OpSizeMask) ==
1441 (STI.hasFeature(Feature: X86::Is16Bit) ? X86II::OpSize32 : X86II::OpSize16))
1442 emitByte(C: 0x66, CB);
1443
1444 // Emit the LOCK opcode prefix.
1445 if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK)
1446 emitByte(C: 0xF0, CB);
1447
1448 // Emit the NOTRACK opcode prefix.
1449 if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK)
1450 emitByte(C: 0x3E, CB);
1451
1452 switch (TSFlags & X86II::OpPrefixMask) {
1453 case X86II::PD: // 66
1454 emitByte(C: 0x66, CB);
1455 break;
1456 case X86II::XS: // F3
1457 emitByte(C: 0xF3, CB);
1458 break;
1459 case X86II::XD: // F2
1460 emitByte(C: 0xF2, CB);
1461 break;
1462 }
1463
1464 // Handle REX prefix.
1465 assert((STI.hasFeature(X86::Is64Bit) || !(TSFlags & X86II::REX_W)) &&
1466 "REX.W requires 64bit mode.");
1467 PrefixKind Kind = emitREXPrefix(MemOperand, MI, STI, CB);
1468
1469 // 0x0F escape code must be emitted just before the opcode.
1470 switch (TSFlags & X86II::OpMapMask) {
1471 case X86II::TB: // Two-byte opcode map
1472 // Encoded by M bit in REX2
1473 if (Kind == REX2)
1474 break;
1475 [[fallthrough]];
1476 case X86II::T8: // 0F 38
1477 case X86II::TA: // 0F 3A
1478 case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller.
1479 emitByte(C: 0x0F, CB);
1480 break;
1481 }
1482
1483 switch (TSFlags & X86II::OpMapMask) {
1484 case X86II::T8: // 0F 38
1485 emitByte(C: 0x38, CB);
1486 break;
1487 case X86II::TA: // 0F 3A
1488 emitByte(C: 0x3A, CB);
1489 break;
1490 }
1491
1492 return Kind;
1493}
1494
1495void X86MCCodeEmitter::emitPrefix(const MCInst &MI, SmallVectorImpl<char> &CB,
1496 const MCSubtargetInfo &STI) const {
1497 uint64_t TSFlags = MCII.get(Opcode: MI.getOpcode()).TSFlags;
1498
1499 // Pseudo instructions don't get encoded.
1500 if (X86II::isPseudo(TSFlags))
1501 return;
1502
1503 emitPrefixImpl(MI, STI, CB);
1504}
1505
1506void X86_MC::emitPrefix(MCCodeEmitter &MCE, const MCInst &MI,
1507 SmallVectorImpl<char> &CB, const MCSubtargetInfo &STI) {
1508 static_cast<X86MCCodeEmitter &>(MCE).emitPrefix(MI, CB, STI);
1509}
1510
1511void X86MCCodeEmitter::encodeInstruction(const MCInst &MI,
1512 SmallVectorImpl<char> &CB,
1513 SmallVectorImpl<MCFixup> &Fixups,
1514 const MCSubtargetInfo &STI) const {
1515 unsigned Opcode = MI.getOpcode();
1516 const MCInstrDesc &Desc = MCII.get(Opcode);
1517 uint64_t TSFlags = Desc.TSFlags;
1518
1519 // Pseudo instructions don't get encoded.
1520 if (X86II::isPseudo(TSFlags))
1521 return;
1522
1523 unsigned NumOps = Desc.getNumOperands();
1524 unsigned CurOp = X86II::getOperandBias(Desc);
1525
1526 uint64_t StartByte = CB.size();
1527
1528 PrefixKind Kind = emitPrefixImpl(MI, STI, CB);
1529
1530 // It uses the VEX.VVVV field?
1531 bool HasVEX_4V = TSFlags & X86II::VEX_4V;
1532 bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;
1533
1534 // It uses the EVEX.aaa field?
1535 bool HasEVEX_K = TSFlags & X86II::EVEX_K;
1536 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
1537
1538 // Used if a register is encoded in 7:4 of immediate.
1539 unsigned I8RegNum = 0;
1540
1541 uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
1542
1543 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1544 BaseOpcode = 0x0F; // Weird 3DNow! encoding.
1545
1546 unsigned OpcodeOffset = 0;
1547
1548 bool IsND = X86II::hasNewDataDest(TSFlags);
1549 bool HasTwoConditionalOps = TSFlags & X86II::TwoConditionalOps;
1550
1551 uint64_t Form = TSFlags & X86II::FormMask;
1552 switch (Form) {
1553 default:
1554 errs() << "FORM: " << Form << "\n";
1555 llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
1556 case X86II::Pseudo:
1557 llvm_unreachable("Pseudo instruction shouldn't be emitted");
1558 case X86II::RawFrmDstSrc:
1559 emitByte(C: BaseOpcode, CB);
1560 CurOp += 3; // Consume operands.
1561 break;
1562 case X86II::RawFrmSrc:
1563 emitByte(C: BaseOpcode, CB);
1564 CurOp += 2; // Consume operands.
1565 break;
1566 case X86II::RawFrmDst:
1567 emitByte(C: BaseOpcode, CB);
1568 ++CurOp; // Consume operand.
1569 break;
1570 case X86II::PrefixByte:
1571 emitByte(C: BaseOpcode, CB);
1572 break;
1573 case X86II::AddCCFrm: {
1574 // This will be added to the opcode in the fallthrough.
1575 OpcodeOffset = MI.getOperand(i: NumOps - 1).getImm();
1576 assert(OpcodeOffset < 16 && "Unexpected opcode offset!");
1577 --NumOps; // Drop the operand from the end.
1578 [[fallthrough]];
1579 case X86II::RawFrm:
1580 emitByte(C: BaseOpcode + OpcodeOffset, CB);
1581
1582 if (!STI.hasFeature(Feature: X86::Is64Bit) || !isPCRel32Branch(MI, MCII))
1583 break;
1584
1585 const MCOperand &Op = MI.getOperand(i: CurOp++);
1586 emitImmediate(DispOp: Op, Loc: MI.getLoc(), FixupKind: X86::reloc_branch_4byte_pcrel, PCRel: true,
1587 StartByte, CB, Fixups);
1588 break;
1589 }
1590 case X86II::RawFrmMemOffs:
1591 emitByte(C: BaseOpcode, CB);
1592 emitImmediate(DispOp: MI.getOperand(i: CurOp++), Loc: MI.getLoc(), FixupKind: getImmFixupKind(TSFlags),
1593 PCRel: X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);
1594 ++CurOp; // skip segment operand
1595 break;
1596 case X86II::RawFrmImm8:
1597 emitByte(C: BaseOpcode, CB);
1598 emitImmediate(DispOp: MI.getOperand(i: CurOp++), Loc: MI.getLoc(), FixupKind: getImmFixupKind(TSFlags),
1599 PCRel: X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);
1600 emitImmediate(DispOp: MI.getOperand(i: CurOp++), Loc: MI.getLoc(), FixupKind: FK_Data_1, PCRel: false,
1601 StartByte, CB, Fixups);
1602 break;
1603 case X86II::RawFrmImm16:
1604 emitByte(C: BaseOpcode, CB);
1605 emitImmediate(DispOp: MI.getOperand(i: CurOp++), Loc: MI.getLoc(), FixupKind: getImmFixupKind(TSFlags),
1606 PCRel: X86II::isImmPCRel(TSFlags), StartByte, CB, Fixups);
1607 emitImmediate(DispOp: MI.getOperand(i: CurOp++), Loc: MI.getLoc(), FixupKind: FK_Data_2, PCRel: false,
1608 StartByte, CB, Fixups);
1609 break;
1610
1611 case X86II::AddRegFrm:
1612 emitByte(C: BaseOpcode + getX86RegNum(MO: MI.getOperand(i: CurOp++)), CB);
1613 break;
1614
1615 case X86II::MRMDestReg: {
1616 emitByte(C: BaseOpcode, CB);
1617 unsigned SrcRegNum = CurOp + 1;
1618
1619 if (HasEVEX_K) // Skip writemask
1620 ++SrcRegNum;
1621
1622 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1623 ++SrcRegNum;
1624 if (IsND) // Skip the NDD operand encoded in EVEX_VVVV
1625 ++CurOp;
1626
1627 emitRegModRMByte(ModRMReg: MI.getOperand(i: CurOp),
1628 RegOpcodeFld: getX86RegNum(MO: MI.getOperand(i: SrcRegNum)), CB);
1629 CurOp = SrcRegNum + 1;
1630 break;
1631 }
1632 case X86II::MRMDestRegCC: {
1633 unsigned FirstOp = CurOp++;
1634 unsigned SecondOp = CurOp++;
1635 unsigned CC = MI.getOperand(i: CurOp++).getImm();
1636 emitByte(C: BaseOpcode + CC, CB);
1637 emitRegModRMByte(ModRMReg: MI.getOperand(i: FirstOp),
1638 RegOpcodeFld: getX86RegNum(MO: MI.getOperand(i: SecondOp)), CB);
1639 break;
1640 }
1641 case X86II::MRMDestMem4VOp3CC: {
1642 unsigned CC = MI.getOperand(i: 8).getImm();
1643 emitByte(C: BaseOpcode + CC, CB);
1644 unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1645 emitMemModRMByte(MI, Op: CurOp + 1, RegOpcodeField: getX86RegNum(MO: MI.getOperand(i: 0)), TSFlags,
1646 Kind, StartByte, CB, Fixups, STI, ForceSIB: false);
1647 CurOp = SrcRegNum + 3; // skip reg, VEX_V4 and CC
1648 break;
1649 }
1650 case X86II::MRMDestMemFSIB:
1651 case X86II::MRMDestMem: {
1652 emitByte(C: BaseOpcode, CB);
1653 unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1654
1655 if (HasEVEX_K) // Skip writemask
1656 ++SrcRegNum;
1657
1658 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1659 ++SrcRegNum;
1660
1661 if (IsND) // Skip new data destination
1662 ++CurOp;
1663
1664 bool ForceSIB = (Form == X86II::MRMDestMemFSIB);
1665 emitMemModRMByte(MI, Op: CurOp, RegOpcodeField: getX86RegNum(MO: MI.getOperand(i: SrcRegNum)), TSFlags,
1666 Kind, StartByte, CB, Fixups, STI, ForceSIB);
1667 CurOp = SrcRegNum + 1;
1668 break;
1669 }
1670 case X86II::MRMDestMemCC: {
1671 unsigned MemOp = CurOp;
1672 CurOp = MemOp + X86::AddrNumOperands;
1673 unsigned RegOp = CurOp++;
1674 unsigned CC = MI.getOperand(i: CurOp++).getImm();
1675 emitByte(C: BaseOpcode + CC, CB);
1676 emitMemModRMByte(MI, Op: MemOp, RegOpcodeField: getX86RegNum(MO: MI.getOperand(i: RegOp)), TSFlags,
1677 Kind, StartByte, CB, Fixups, STI);
1678 break;
1679 }
1680 case X86II::MRMSrcReg: {
1681 emitByte(C: BaseOpcode, CB);
1682 unsigned SrcRegNum = CurOp + 1;
1683
1684 if (HasEVEX_K) // Skip writemask
1685 ++SrcRegNum;
1686
1687 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1688 ++SrcRegNum;
1689
1690 if (IsND) // Skip new data destination
1691 ++CurOp;
1692
1693 emitRegModRMByte(ModRMReg: MI.getOperand(i: SrcRegNum),
1694 RegOpcodeFld: getX86RegNum(MO: MI.getOperand(i: CurOp)), CB);
1695 CurOp = SrcRegNum + 1;
1696 if (HasVEX_I8Reg)
1697 I8RegNum = getX86RegEncoding(MI, OpNum: CurOp++);
1698 // do not count the rounding control operand
1699 if (HasEVEX_RC)
1700 --NumOps;
1701 break;
1702 }
1703 case X86II::MRMSrcReg4VOp3: {
1704 emitByte(C: BaseOpcode, CB);
1705 unsigned SrcRegNum = CurOp + 1;
1706
1707 emitRegModRMByte(ModRMReg: MI.getOperand(i: SrcRegNum),
1708 RegOpcodeFld: getX86RegNum(MO: MI.getOperand(i: CurOp)), CB);
1709 CurOp = SrcRegNum + 1;
1710 ++CurOp; // Encoded in VEX.VVVV
1711 break;
1712 }
1713 case X86II::MRMSrcRegOp4: {
1714 emitByte(C: BaseOpcode, CB);
1715 unsigned SrcRegNum = CurOp + 1;
1716
1717 // Skip 1st src (which is encoded in VEX_VVVV)
1718 ++SrcRegNum;
1719
1720 // Capture 2nd src (which is encoded in Imm[7:4])
1721 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1722 I8RegNum = getX86RegEncoding(MI, OpNum: SrcRegNum++);
1723
1724 emitRegModRMByte(ModRMReg: MI.getOperand(i: SrcRegNum),
1725 RegOpcodeFld: getX86RegNum(MO: MI.getOperand(i: CurOp)), CB);
1726 CurOp = SrcRegNum + 1;
1727 break;
1728 }
1729 case X86II::MRMSrcRegCC: {
1730 if (IsND) // Skip new data destination
1731 ++CurOp;
1732 unsigned FirstOp = CurOp++;
1733 unsigned SecondOp = CurOp++;
1734
1735 unsigned CC = MI.getOperand(i: CurOp++).getImm();
1736 emitByte(C: BaseOpcode + CC, CB);
1737
1738 emitRegModRMByte(ModRMReg: MI.getOperand(i: SecondOp),
1739 RegOpcodeFld: getX86RegNum(MO: MI.getOperand(i: FirstOp)), CB);
1740 break;
1741 }
1742 case X86II::MRMSrcMemFSIB:
1743 case X86II::MRMSrcMem: {
1744 unsigned FirstMemOp = CurOp + 1;
1745
1746 if (IsND) // Skip new data destination
1747 CurOp++;
1748
1749 if (HasEVEX_K) // Skip writemask
1750 ++FirstMemOp;
1751
1752 if (HasVEX_4V)
1753 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1754
1755 emitByte(C: BaseOpcode, CB);
1756
1757 bool ForceSIB = (Form == X86II::MRMSrcMemFSIB);
1758 emitMemModRMByte(MI, Op: FirstMemOp, RegOpcodeField: getX86RegNum(MO: MI.getOperand(i: CurOp)),
1759 TSFlags, Kind, StartByte, CB, Fixups, STI, ForceSIB);
1760 CurOp = FirstMemOp + X86::AddrNumOperands;
1761 if (HasVEX_I8Reg)
1762 I8RegNum = getX86RegEncoding(MI, OpNum: CurOp++);
1763 break;
1764 }
1765 case X86II::MRMSrcMem4VOp3: {
1766 unsigned FirstMemOp = CurOp + 1;
1767
1768 emitByte(C: BaseOpcode, CB);
1769
1770 emitMemModRMByte(MI, Op: FirstMemOp, RegOpcodeField: getX86RegNum(MO: MI.getOperand(i: CurOp)),
1771 TSFlags, Kind, StartByte, CB, Fixups, STI);
1772 CurOp = FirstMemOp + X86::AddrNumOperands;
1773 ++CurOp; // Encoded in VEX.VVVV.
1774 break;
1775 }
1776 case X86II::MRMSrcMemOp4: {
1777 unsigned FirstMemOp = CurOp + 1;
1778
1779 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1780
1781 // Capture second register source (encoded in Imm[7:4])
1782 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1783 I8RegNum = getX86RegEncoding(MI, OpNum: FirstMemOp++);
1784
1785 emitByte(C: BaseOpcode, CB);
1786
1787 emitMemModRMByte(MI, Op: FirstMemOp, RegOpcodeField: getX86RegNum(MO: MI.getOperand(i: CurOp)),
1788 TSFlags, Kind, StartByte, CB, Fixups, STI);
1789 CurOp = FirstMemOp + X86::AddrNumOperands;
1790 break;
1791 }
1792 case X86II::MRMSrcMemCC: {
1793 if (IsND) // Skip new data destination
1794 ++CurOp;
1795 unsigned RegOp = CurOp++;
1796 unsigned FirstMemOp = CurOp;
1797 CurOp = FirstMemOp + X86::AddrNumOperands;
1798
1799 unsigned CC = MI.getOperand(i: CurOp++).getImm();
1800 emitByte(C: BaseOpcode + CC, CB);
1801
1802 emitMemModRMByte(MI, Op: FirstMemOp, RegOpcodeField: getX86RegNum(MO: MI.getOperand(i: RegOp)),
1803 TSFlags, Kind, StartByte, CB, Fixups, STI);
1804 break;
1805 }
1806
1807 case X86II::MRMXrCC: {
1808 unsigned RegOp = CurOp++;
1809
1810 unsigned CC = MI.getOperand(i: CurOp++).getImm();
1811 emitByte(C: BaseOpcode + CC, CB);
1812 emitRegModRMByte(ModRMReg: MI.getOperand(i: RegOp), RegOpcodeFld: 0, CB);
1813 break;
1814 }
1815
1816 case X86II::MRMXr:
1817 case X86II::MRM0r:
1818 case X86II::MRM1r:
1819 case X86II::MRM2r:
1820 case X86II::MRM3r:
1821 case X86II::MRM4r:
1822 case X86II::MRM5r:
1823 case X86II::MRM6r:
1824 case X86II::MRM7r:
1825 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1826 ++CurOp;
1827 if (HasEVEX_K) // Skip writemask
1828 ++CurOp;
1829 emitByte(C: BaseOpcode, CB);
1830 emitRegModRMByte(ModRMReg: MI.getOperand(i: CurOp++),
1831 RegOpcodeFld: (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, CB);
1832 break;
1833 case X86II::MRMr0:
1834 emitByte(C: BaseOpcode, CB);
1835 emitByte(C: modRMByte(Mod: 3, RegOpcode: getX86RegNum(MO: MI.getOperand(i: CurOp++)), RM: 0), CB);
1836 break;
1837
1838 case X86II::MRMXmCC: {
1839 unsigned FirstMemOp = CurOp;
1840 CurOp = FirstMemOp + X86::AddrNumOperands;
1841
1842 unsigned CC = MI.getOperand(i: CurOp++).getImm();
1843 emitByte(C: BaseOpcode + CC, CB);
1844
1845 emitMemModRMByte(MI, Op: FirstMemOp, RegOpcodeField: 0, TSFlags, Kind, StartByte, CB, Fixups,
1846 STI);
1847 break;
1848 }
1849
1850 case X86II::MRMXm:
1851 case X86II::MRM0m:
1852 case X86II::MRM1m:
1853 case X86II::MRM2m:
1854 case X86II::MRM3m:
1855 case X86II::MRM4m:
1856 case X86II::MRM5m:
1857 case X86II::MRM6m:
1858 case X86II::MRM7m:
1859 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1860 ++CurOp;
1861 if (HasEVEX_K) // Skip writemask
1862 ++CurOp;
1863 emitByte(C: BaseOpcode, CB);
1864 emitMemModRMByte(MI, Op: CurOp,
1865 RegOpcodeField: (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags,
1866 Kind, StartByte, CB, Fixups, STI);
1867 CurOp += X86::AddrNumOperands;
1868 break;
1869
1870 case X86II::MRM0X:
1871 case X86II::MRM1X:
1872 case X86II::MRM2X:
1873 case X86II::MRM3X:
1874 case X86II::MRM4X:
1875 case X86II::MRM5X:
1876 case X86II::MRM6X:
1877 case X86II::MRM7X:
1878 emitByte(C: BaseOpcode, CB);
1879 emitByte(C: 0xC0 + ((Form - X86II::MRM0X) << 3), CB);
1880 break;
1881
1882 case X86II::MRM_C0:
1883 case X86II::MRM_C1:
1884 case X86II::MRM_C2:
1885 case X86II::MRM_C3:
1886 case X86II::MRM_C4:
1887 case X86II::MRM_C5:
1888 case X86II::MRM_C6:
1889 case X86II::MRM_C7:
1890 case X86II::MRM_C8:
1891 case X86II::MRM_C9:
1892 case X86II::MRM_CA:
1893 case X86II::MRM_CB:
1894 case X86II::MRM_CC:
1895 case X86II::MRM_CD:
1896 case X86II::MRM_CE:
1897 case X86II::MRM_CF:
1898 case X86II::MRM_D0:
1899 case X86II::MRM_D1:
1900 case X86II::MRM_D2:
1901 case X86II::MRM_D3:
1902 case X86II::MRM_D4:
1903 case X86II::MRM_D5:
1904 case X86II::MRM_D6:
1905 case X86II::MRM_D7:
1906 case X86II::MRM_D8:
1907 case X86II::MRM_D9:
1908 case X86II::MRM_DA:
1909 case X86II::MRM_DB:
1910 case X86II::MRM_DC:
1911 case X86II::MRM_DD:
1912 case X86II::MRM_DE:
1913 case X86II::MRM_DF:
1914 case X86II::MRM_E0:
1915 case X86II::MRM_E1:
1916 case X86II::MRM_E2:
1917 case X86II::MRM_E3:
1918 case X86II::MRM_E4:
1919 case X86II::MRM_E5:
1920 case X86II::MRM_E6:
1921 case X86II::MRM_E7:
1922 case X86II::MRM_E8:
1923 case X86II::MRM_E9:
1924 case X86II::MRM_EA:
1925 case X86II::MRM_EB:
1926 case X86II::MRM_EC:
1927 case X86II::MRM_ED:
1928 case X86II::MRM_EE:
1929 case X86II::MRM_EF:
1930 case X86II::MRM_F0:
1931 case X86II::MRM_F1:
1932 case X86II::MRM_F2:
1933 case X86II::MRM_F3:
1934 case X86II::MRM_F4:
1935 case X86II::MRM_F5:
1936 case X86II::MRM_F6:
1937 case X86II::MRM_F7:
1938 case X86II::MRM_F8:
1939 case X86II::MRM_F9:
1940 case X86II::MRM_FA:
1941 case X86II::MRM_FB:
1942 case X86II::MRM_FC:
1943 case X86II::MRM_FD:
1944 case X86II::MRM_FE:
1945 case X86II::MRM_FF:
1946 emitByte(C: BaseOpcode, CB);
1947 emitByte(C: 0xC0 + Form - X86II::MRM_C0, CB);
1948 break;
1949 }
1950
1951 if (HasVEX_I8Reg) {
1952 // The last source register of a 4 operand instruction in AVX is encoded
1953 // in bits[7:4] of a immediate byte.
1954 assert(I8RegNum < 16 && "Register encoding out of range");
1955 I8RegNum <<= 4;
1956 if (CurOp != NumOps) {
1957 unsigned Val = MI.getOperand(i: CurOp++).getImm();
1958 assert(Val < 16 && "Immediate operand value out of range");
1959 I8RegNum |= Val;
1960 }
1961 emitImmediate(DispOp: MCOperand::createImm(Val: I8RegNum), Loc: MI.getLoc(), FixupKind: FK_Data_1, PCRel: false,
1962 StartByte, CB, Fixups);
1963 } else {
1964 // If there is a remaining operand, it must be a trailing immediate. Emit it
1965 // according to the right size for the instruction. Some instructions
1966 // (SSE4a extrq and insertq) have two trailing immediates.
1967
1968 // Skip two trainling conditional operands encoded in EVEX prefix
1969 unsigned RemainingOps = NumOps - CurOp - 2 * HasTwoConditionalOps;
1970 // Verify that hasImm(TSFlags) matches the presence of remaining operands.
1971 // Exclude forms that emit immediates in the switch above (RawFrm and
1972 // AddCCFrm may consume a PC-relative operand; RawFrmImm8/16 and
1973 // RawFrmMemOffs always consume their immediates there).
1974 assert((!X86II::hasImm(TSFlags) || RemainingOps || Form == X86II::RawFrm ||
1975 Form == X86II::AddCCFrm || Form == X86II::RawFrmImm8 ||
1976 Form == X86II::RawFrmImm16 || Form == X86II::RawFrmMemOffs) &&
1977 "TSFlags indicates immediate but no operand provides it");
1978 while (RemainingOps) {
1979 emitImmediate(DispOp: MI.getOperand(i: CurOp++), Loc: MI.getLoc(),
1980 FixupKind: getImmFixupKind(TSFlags: Desc.TSFlags),
1981 PCRel: X86II::isImmPCRel(TSFlags: Desc.TSFlags), StartByte, CB, Fixups);
1982 --RemainingOps;
1983 }
1984 CurOp += 2 * HasTwoConditionalOps;
1985 }
1986
1987 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1988 emitByte(C: X86II::getBaseOpcodeFor(TSFlags), CB);
1989
1990 if (CB.size() - StartByte > 15)
1991 Ctx.reportError(L: MI.getLoc(), Msg: "instruction length exceeds the limit of 15");
1992#ifndef NDEBUG
1993 // FIXME: Verify.
1994 if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {
1995 errs() << "Cannot encode all operands of: ";
1996 MI.dump();
1997 errs() << '\n';
1998 abort();
1999 }
2000#endif
2001}
2002
2003MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,
2004 MCContext &Ctx) {
2005 return new X86MCCodeEmitter(MCII, Ctx);
2006}
2007