1//===- X86CompressEVEX.cpp ------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass compresses instructions from EVEX space to legacy/VEX/EVEX space
10// when possible in order to reduce code size or facilitate HW decoding.
11//
12// Possible compression:
13// a. AVX512 instruction (EVEX) -> AVX instruction (VEX)
14// b. Promoted instruction (EVEX) -> pre-promotion instruction (legacy/VEX)
15// c. NDD (EVEX) -> non-NDD (legacy)
16// d. NF_ND (EVEX) -> NF (EVEX)
17// e. NonNF (EVEX) -> NF (EVEX)
18// f. SETZUCCm (EVEX) -> SETCCm (legacy)
19// g. VPMOV*2M (EVEX) + KMOV -> VMOVMSK/VPMOVMSKB (VEX)
20// h. VPMOV*2M (EVEX) + masked VMOV* -> VBLENDV* (VEX)
21//
22// Compression a, b and c can always reduce code size, with some exceptions
23// such as promoted 16-bit CRC32 which is as long as the legacy version.
24//
25// legacy:
26// crc32w %si, %eax ## encoding: [0x66,0xf2,0x0f,0x38,0xf1,0xc6]
27// promoted:
28// crc32w %si, %eax ## encoding: [0x62,0xf4,0x7d,0x08,0xf1,0xc6]
29//
30// From performance perspective, these should be same (same uops and same EXE
31// ports). From a FMV perspective, an older legacy encoding is preferred b/c it
32// can execute in more places (broader HW install base). So we will still do
33// the compression.
34//
35// Compression d can help hardware decode (HW may skip reading the NDD
36// register) although the instruction length remains unchanged.
37//
38// Compression e can help hardware skip updating EFLAGS although the instruction
39// length remains unchanged.
40//===----------------------------------------------------------------------===//
41
42#include "MCTargetDesc/X86BaseInfo.h"
43#include "X86.h"
44#include "X86InstrInfo.h"
45#include "X86Subtarget.h"
46#include "llvm/ADT/SmallVector.h"
47#include "llvm/ADT/StringRef.h"
48#include "llvm/CodeGen/MachineFunction.h"
49#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
50#include "llvm/CodeGen/MachineFunctionPass.h"
51#include "llvm/CodeGen/MachineInstr.h"
52#include "llvm/CodeGen/MachineOperand.h"
53#include "llvm/CodeGen/MachinePassManager.h"
54#include "llvm/IR/Analysis.h"
55#include "llvm/MC/MCInstrDesc.h"
56#include "llvm/Pass.h"
57#include <atomic>
58#include <cassert>
59#include <cstdint>
60
61using namespace llvm;
62
63#define COMP_EVEX_DESC "Compressing EVEX instrs when possible"
64#define COMP_EVEX_NAME "x86-compress-evex"
65
66#define DEBUG_TYPE COMP_EVEX_NAME
67
68extern cl::opt<bool> X86EnableAPXForRelocation;
69
70namespace {
71// Including the generated EVEX compression tables.
72#define GET_X86_COMPRESS_EVEX_TABLE
73#include "X86GenInstrMapping.inc"
74
75class CompressEVEXLegacy : public MachineFunctionPass {
76public:
77 static char ID;
78 CompressEVEXLegacy() : MachineFunctionPass(ID) {}
79 StringRef getPassName() const override { return COMP_EVEX_DESC; }
80
81 bool runOnMachineFunction(MachineFunction &MF) override;
82
83 // This pass runs after regalloc and doesn't support VReg operands.
84 MachineFunctionProperties getRequiredProperties() const override {
85 return MachineFunctionProperties().setNoVRegs();
86 }
87};
88
89} // end anonymous namespace
90
91char CompressEVEXLegacy::ID = 0;
92
93static bool usesExtendedRegister(const MachineInstr &MI) {
94 auto isHiRegIdx = [](MCRegister Reg) {
95 // Check for XMM register with indexes between 16 - 31.
96 if (Reg >= X86::XMM16 && Reg <= X86::XMM31)
97 return true;
98 // Check for YMM register with indexes between 16 - 31.
99 if (Reg >= X86::YMM16 && Reg <= X86::YMM31)
100 return true;
101 // Check for GPR with indexes between 16 - 31.
102 if (X86II::isApxExtendedReg(Reg))
103 return true;
104 return false;
105 };
106
107 // Check that operands are not ZMM regs or
108 // XMM/YMM regs with hi indexes between 16 - 31.
109 for (const MachineOperand &MO : MI.explicit_operands()) {
110 if (!MO.isReg())
111 continue;
112
113 MCRegister Reg = MO.getReg().asMCReg();
114 assert(!X86II::isZMMReg(Reg) &&
115 "ZMM instructions should not be in the EVEX->VEX tables");
116 if (isHiRegIdx(Reg))
117 return true;
118 }
119
120 return false;
121}
122
123// Do any custom cleanup needed to finalize the conversion.
124static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) {
125 (void)NewOpc;
126 unsigned Opc = MI.getOpcode();
127 switch (Opc) {
128 case X86::VALIGNDZ128rri:
129 case X86::VALIGNDZ128rmi:
130 case X86::VALIGNQZ128rri:
131 case X86::VALIGNQZ128rmi: {
132 assert((NewOpc == X86::VPALIGNRrri || NewOpc == X86::VPALIGNRrmi) &&
133 "Unexpected new opcode!");
134 unsigned Scale =
135 (Opc == X86::VALIGNQZ128rri || Opc == X86::VALIGNQZ128rmi) ? 8 : 4;
136 MachineOperand &Imm = MI.getOperand(i: MI.getNumExplicitOperands() - 1);
137 Imm.setImm(Imm.getImm() * Scale);
138 break;
139 }
140 case X86::VSHUFF32X4Z256rmi:
141 case X86::VSHUFF32X4Z256rri:
142 case X86::VSHUFF64X2Z256rmi:
143 case X86::VSHUFF64X2Z256rri:
144 case X86::VSHUFI32X4Z256rmi:
145 case X86::VSHUFI32X4Z256rri:
146 case X86::VSHUFI64X2Z256rmi:
147 case X86::VSHUFI64X2Z256rri: {
148 assert((NewOpc == X86::VPERM2F128rri || NewOpc == X86::VPERM2I128rri ||
149 NewOpc == X86::VPERM2F128rmi || NewOpc == X86::VPERM2I128rmi) &&
150 "Unexpected new opcode!");
151 MachineOperand &Imm = MI.getOperand(i: MI.getNumExplicitOperands() - 1);
152 int64_t ImmVal = Imm.getImm();
153 // Set bit 5, move bit 1 to bit 4, copy bit 0.
154 Imm.setImm(0x20 | ((ImmVal & 2) << 3) | (ImmVal & 1));
155 break;
156 }
157 case X86::VRNDSCALEPDZ128rri:
158 case X86::VRNDSCALEPDZ128rmi:
159 case X86::VRNDSCALEPSZ128rri:
160 case X86::VRNDSCALEPSZ128rmi:
161 case X86::VRNDSCALEPDZ256rri:
162 case X86::VRNDSCALEPDZ256rmi:
163 case X86::VRNDSCALEPSZ256rri:
164 case X86::VRNDSCALEPSZ256rmi:
165 case X86::VRNDSCALESDZrri:
166 case X86::VRNDSCALESDZrmi:
167 case X86::VRNDSCALESSZrri:
168 case X86::VRNDSCALESSZrmi:
169 case X86::VRNDSCALESDZrri_Int:
170 case X86::VRNDSCALESDZrmi_Int:
171 case X86::VRNDSCALESSZrri_Int:
172 case X86::VRNDSCALESSZrmi_Int:
173 const MachineOperand &Imm = MI.getOperand(i: MI.getNumExplicitOperands() - 1);
174 int64_t ImmVal = Imm.getImm();
175 // Ensure that only bits 3:0 of the immediate are used.
176 if ((ImmVal & 0xf) != ImmVal)
177 return false;
178 break;
179 }
180
181 return true;
182}
183
184static bool isKMovNarrowing(unsigned VPMOVOpc, unsigned KMOVOpc) {
185 unsigned VPMOVBits = 0;
186 switch (VPMOVOpc) {
187 case X86::VPMOVQ2MZ128kr:
188 VPMOVBits = 2;
189 break;
190 case X86::VPMOVQ2MZ256kr:
191 case X86::VPMOVD2MZ128kr:
192 VPMOVBits = 4;
193 break;
194 case X86::VPMOVD2MZ256kr:
195 VPMOVBits = 8;
196 break;
197 case X86::VPMOVB2MZ128kr:
198 VPMOVBits = 16;
199 break;
200 case X86::VPMOVB2MZ256kr:
201 VPMOVBits = 32;
202 break;
203 default:
204 llvm_unreachable("Unknown VPMOV opcode");
205 }
206
207 unsigned KMOVSize = 0;
208 switch (KMOVOpc) {
209 case X86::KMOVBrk:
210 KMOVSize = 8;
211 break;
212 case X86::KMOVWrk:
213 KMOVSize = 16;
214 break;
215 case X86::KMOVDrk:
216 KMOVSize = 32;
217 break;
218 default:
219 llvm_unreachable("Unknown KMOV opcode");
220 }
221
222 return KMOVSize < VPMOVBits;
223}
224
225static bool isCompressibleBlendVUse(unsigned BlendOpc, unsigned UseOpc) {
226 switch (BlendOpc) {
227 case X86::VBLENDVPSrrr:
228 switch (UseOpc) {
229 case X86::VMOVAPSZ128rrk:
230 case X86::VMOVUPSZ128rrk:
231 case X86::VMOVDQA32Z128rrk:
232 case X86::VMOVDQU32Z128rrk:
233 return true;
234 default:
235 return false;
236 }
237 case X86::VBLENDVPSYrrr:
238 switch (UseOpc) {
239 case X86::VMOVAPSZ256rrk:
240 case X86::VMOVUPSZ256rrk:
241 case X86::VMOVDQA32Z256rrk:
242 case X86::VMOVDQU32Z256rrk:
243 return true;
244 default:
245 return false;
246 }
247 case X86::VBLENDVPDrrr:
248 switch (UseOpc) {
249 case X86::VMOVAPDZ128rrk:
250 case X86::VMOVUPDZ128rrk:
251 case X86::VMOVDQA64Z128rrk:
252 case X86::VMOVDQU64Z128rrk:
253 return true;
254 default:
255 return false;
256 }
257 case X86::VBLENDVPDYrrr:
258 switch (UseOpc) {
259 case X86::VMOVAPDZ256rrk:
260 case X86::VMOVUPDZ256rrk:
261 case X86::VMOVDQA64Z256rrk:
262 case X86::VMOVDQU64Z256rrk:
263 return true;
264 default:
265 return false;
266 }
267 case X86::VPBLENDVBrrr:
268 return UseOpc == X86::VMOVDQU8Z128rrk;
269 case X86::VPBLENDVBYrrr:
270 return UseOpc == X86::VMOVDQU8Z256rrk;
271 default:
272 return false;
273 }
274}
275
276// Try to compress VPMOV*2M chain patterns:
277// vpmov*2m %xmm0, %k0 -> (erase this)
278// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
279// and:
280// vpmov*2m %xmm0, %k1 -> (erase this)
281// vmov* %xmm1, %xmm2 {%k1} -> vblendv* %xmm0, %xmm2, %xmm1, %xmm2
282static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB,
283 const X86Subtarget &ST,
284 SmallVectorImpl<MachineInstr *> &ToErase) {
285 const X86InstrInfo *TII = ST.getInstrInfo();
286 const TargetRegisterInfo *TRI = ST.getRegisterInfo();
287 MachineRegisterInfo *MRI = &MBB.getParent()->getRegInfo();
288
289 unsigned Opc = MI.getOpcode();
290 if (Opc != X86::VPMOVD2MZ128kr && Opc != X86::VPMOVD2MZ256kr &&
291 Opc != X86::VPMOVQ2MZ128kr && Opc != X86::VPMOVQ2MZ256kr &&
292 Opc != X86::VPMOVB2MZ128kr && Opc != X86::VPMOVB2MZ256kr)
293 return false;
294
295 if (usesExtendedRegister(MI))
296 return false;
297
298 Register MaskReg = MI.getOperand(i: 0).getReg();
299 Register SrcVecReg = MI.getOperand(i: 1).getReg();
300
301 unsigned MovMskOpc = 0;
302 unsigned BlendOpc = 0;
303 switch (Opc) {
304 case X86::VPMOVD2MZ128kr:
305 MovMskOpc = X86::VMOVMSKPSrr;
306 BlendOpc = X86::VBLENDVPSrrr;
307 break;
308 case X86::VPMOVD2MZ256kr:
309 MovMskOpc = X86::VMOVMSKPSYrr;
310 BlendOpc = X86::VBLENDVPSYrrr;
311 break;
312 case X86::VPMOVQ2MZ128kr:
313 MovMskOpc = X86::VMOVMSKPDrr;
314 BlendOpc = X86::VBLENDVPDrrr;
315 break;
316 case X86::VPMOVQ2MZ256kr:
317 MovMskOpc = X86::VMOVMSKPDYrr;
318 BlendOpc = X86::VBLENDVPDYrrr;
319 break;
320 case X86::VPMOVB2MZ128kr:
321 MovMskOpc = X86::VPMOVMSKBrr;
322 BlendOpc = X86::VPBLENDVBrrr;
323 break;
324 case X86::VPMOVB2MZ256kr:
325 MovMskOpc = X86::VPMOVMSKBYrr;
326 BlendOpc = X86::VPBLENDVBYrrr;
327 break;
328 default:
329 llvm_unreachable("Unknown VPMOV opcode");
330 }
331
332 MachineInstr *KMovMI = nullptr;
333 MachineInstr *BlendMI = nullptr;
334
335 for (MachineInstr &CurMI : llvm::make_range(
336 x: std::next(x: MachineBasicBlock::iterator(MI)), y: MBB.end())) {
337 if (CurMI.readsRegister(Reg: MaskReg, TRI)) {
338 if (KMovMI || BlendMI)
339 return false; // Fail: Mask has MULTIPLE uses
340
341 unsigned UseOpc = CurMI.getOpcode();
342 bool IsKMOV = UseOpc == X86::KMOVBrk || UseOpc == X86::KMOVWrk ||
343 UseOpc == X86::KMOVDrk;
344 // Only allow non-narrowing KMOV uses of the mask.
345 if (IsKMOV && CurMI.getOperand(i: 1).getReg() == MaskReg &&
346 !isKMovNarrowing(VPMOVOpc: Opc, KMOVOpc: UseOpc)) {
347 KMovMI = &CurMI;
348 // continue scanning to ensure
349 // there are no *other* uses of the mask later in the block.
350 } else if (isCompressibleBlendVUse(BlendOpc, UseOpc) &&
351 CurMI.getOperand(i: 2).getReg() == MaskReg &&
352 !usesExtendedRegister(MI: CurMI) &&
353 checkPredicate(Opc: BlendOpc, Subtarget: &ST)) {
354 BlendMI = &CurMI;
355 } else {
356 return false;
357 }
358 }
359
360 if (CurMI.modifiesRegister(Reg: MaskReg, TRI)) {
361 if (!KMovMI && !BlendMI)
362 return false; // Mask clobbered before use
363 break;
364 }
365
366 if (!KMovMI && !BlendMI && CurMI.modifiesRegister(Reg: SrcVecReg, TRI)) {
367 return false; // SrcVecReg modified before it could be reused
368 }
369 }
370
371 if (!KMovMI && !BlendMI)
372 return false;
373
374 // Check if MaskReg is used in any other basic blocks
375 for (const MachineOperand &MO : MRI->use_operands(Reg: MaskReg))
376 if (MO.getParent()->getParent() != &MBB)
377 return false;
378
379 // Apply the transformation
380 MachineInstr *NewMI = nullptr;
381 if (KMovMI) {
382 KMovMI->setDesc(TII->get(Opcode: MovMskOpc));
383 MachineOperand &NewSrc = KMovMI->getOperand(i: 1);
384 NewSrc.setReg(SrcVecReg);
385 // setReg() keeps the mask operand's kill flag; take the source's kill
386 // state from the VPMOV instead.
387 NewSrc.setIsKill(MI.getOperand(i: 1).isKill());
388 NewMI = KMovMI;
389 } else if (BlendMI) {
390 const MachineOperand &MaskVec = MI.getOperand(i: 1);
391 const MachineOperand &Dst = BlendMI->getOperand(i: 0);
392 const MachineOperand &Passthru = BlendMI->getOperand(i: 1);
393 const MachineOperand &Src = BlendMI->getOperand(i: 3);
394
395 // Build a replacement instead of changing BlendMI in place because
396 // VMOV*rrk has a tied passthrough operand and a different operand order
397 // than VBLENDV.
398 auto MIB =
399 BuildMI(BB&: MBB, I&: *BlendMI, MIMD: BlendMI->getDebugLoc(), MCID: TII->get(Opcode: BlendOpc))
400 .addReg(RegNo: Dst.getReg(), Flags: getRegState(RegOp: Dst))
401 .addReg(RegNo: Passthru.getReg(), Flags: getRegState(RegOp: Passthru))
402 .addReg(RegNo: Src.getReg(), Flags: getRegState(RegOp: Src))
403 .addReg(RegNo: MaskVec.getReg(), Flags: getRegState(RegOp: MaskVec));
404 NewMI = MIB;
405 ToErase.push_back(Elt: BlendMI);
406 }
407 assert(NewMI && "Expected a compressed instruction");
408 NewMI->setAsmPrinterFlag(X86::AC_EVEX_2_VEX);
409 ToErase.push_back(Elt: &MI);
410 return true;
411}
412
413static bool CompressEVEXImpl(MachineInstr &MI, MachineBasicBlock &MBB,
414 const X86Subtarget &ST,
415 SmallVectorImpl<MachineInstr *> &ToErase) {
416 uint64_t TSFlags = MI.getDesc().TSFlags;
417
418 // Check for EVEX instructions only.
419 if ((TSFlags & X86II::EncodingMask) != X86II::EVEX)
420 return false;
421
422 // Instructions with mask or 512-bit vector can't be converted to VEX.
423 if (TSFlags & (X86II::EVEX_K | X86II::EVEX_L2))
424 return false;
425
426 // Specialized VPMOVD2M + KMOV -> MOVMSK fold first.
427 if (tryCompressVPMOVPattern(MI, MBB, ST, ToErase))
428 return true;
429
430 auto IsRedundantNewDataDest = [&](unsigned &Opc) {
431 // $rbx = ADD64rr_ND $rbx, $rax / $rbx = ADD64rr_ND $rax, $rbx
432 // ->
433 // $rbx = ADD64rr $rbx, $rax
434 const MCInstrDesc &Desc = MI.getDesc();
435 Register Reg0 = MI.getOperand(i: 0).getReg();
436 const MachineOperand &Op1 = MI.getOperand(i: 1);
437 if (!Op1.isReg() || X86::getFirstAddrOperandIdx(MI) == 1 ||
438 X86::isCFCMOVCC(Opcode: MI.getOpcode()))
439 return false;
440 Register Reg1 = Op1.getReg();
441 if (Reg1 == Reg0)
442 return true;
443
444 // Op1 and Op2 may be commutable for ND instructions.
445 if (!Desc.isCommutable() || Desc.getNumOperands() < 3 ||
446 !MI.getOperand(i: 2).isReg() || MI.getOperand(i: 2).getReg() != Reg0)
447 return false;
448 // Opcode may change after commute, e.g. SHRD -> SHLD
449 ST.getInstrInfo()->commuteInstruction(MI, NewMI: false, OpIdx1: 1, OpIdx2: 2);
450 Opc = MI.getOpcode();
451 return true;
452 };
453
454 // EVEX_B has several meanings.
455 // AVX512:
456 // register form: rounding control or SAE
457 // memory form: broadcast
458 //
459 // APX:
460 // MAP4: NDD, ZU
461 //
462 // For AVX512 cases, EVEX prefix is needed in order to carry this information
463 // thus preventing the transformation to VEX encoding.
464 bool IsND = X86II::hasNewDataDest(TSFlags);
465 unsigned Opc = MI.getOpcode();
466 bool IsSetZUCCm = Opc == X86::SETZUCCm;
467 if (TSFlags & X86II::EVEX_B && !IsND && !IsSetZUCCm)
468 return false;
469 // MOVBE*rr is special because it has semantic of NDD but not set EVEX_B.
470 bool IsNDLike = IsND || Opc == X86::MOVBE32rr || Opc == X86::MOVBE64rr;
471 bool IsRedundantNDD = IsNDLike ? IsRedundantNewDataDest(Opc) : false;
472
473 auto GetCompressedOpc = [&](unsigned Opc) -> unsigned {
474 ArrayRef<X86TableEntry> Table = ArrayRef(X86CompressEVEXTable);
475 const auto I = llvm::lower_bound(Range&: Table, Value&: Opc);
476 if (I == Table.end() || I->OldOpc != Opc)
477 return 0;
478
479 if (usesExtendedRegister(MI) || !checkPredicate(Opc: I->NewOpc, Subtarget: &ST) ||
480 !performCustomAdjustments(MI, NewOpc: I->NewOpc))
481 return 0;
482 return I->NewOpc;
483 };
484
485 Register Dst = MI.getOperand(i: 0).getReg();
486 if (IsRedundantNDD) {
487 // Redundant NDD ops cannot be safely compressed if either:
488 // - the legacy op would introduce a partial write that BreakFalseDeps
489 // identified as a potential stall, or
490 // - the op is writing to a subregister of a live register, i.e. the
491 // full (zeroed) result is used.
492 // Both cases are indicated by an implicit def of the superregister.
493 if (Dst &&
494 (X86::GR16RegClass.contains(Reg: Dst) || X86::GR8RegClass.contains(Reg: Dst))) {
495 Register Super = getX86SubSuperRegister(Reg: Dst, Size: 64);
496 if (MI.definesRegister(Reg: Super, /*TRI=*/nullptr))
497 IsRedundantNDD = false;
498 }
499
500 // ADDrm/mr instructions with NDD + relocation had been transformed to the
501 // instructions without NDD in X86SuppressAPXForRelocation pass. That is to
502 // keep backward compatibility with linkers without APX support.
503 if (!X86EnableAPXForRelocation)
504 assert(!isAddMemInstrWithRelocation(MI) &&
505 "Unexpected NDD instruction with relocation!");
506 } else if (Opc == X86::ADD32ri_ND || Opc == X86::ADD64ri32_ND ||
507 Opc == X86::ADD32rr_ND || Opc == X86::ADD64rr_ND) {
508 // Non-redundant NDD ADD can be compressed to LEA when:
509 // - No EGPR register used and
510 // - EFLAGS is dead.
511 if (!usesExtendedRegister(MI) &&
512 MI.registerDefIsDead(Reg: X86::EFLAGS, /*TRI=*/nullptr)) {
513 Register Src1 = MI.getOperand(i: 1).getReg();
514 const MachineOperand &Src2 = MI.getOperand(i: 2);
515 bool Is32BitReg = Opc == X86::ADD32ri_ND || Opc == X86::ADD32rr_ND;
516 const MCInstrDesc &NewDesc =
517 ST.getInstrInfo()->get(Opcode: Is32BitReg ? X86::LEA64_32r : X86::LEA64r);
518 if (Is32BitReg)
519 Src1 = getX86SubSuperRegister(Reg: Src1, Size: 64);
520 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: NewDesc, DestReg: Dst)
521 .addReg(RegNo: Src1)
522 .addImm(Val: 1);
523 if (Opc == X86::ADD32ri_ND || Opc == X86::ADD64ri32_ND)
524 MIB.addReg(RegNo: 0).add(MO: Src2);
525 else if (Is32BitReg)
526 MIB.addReg(RegNo: getX86SubSuperRegister(Reg: Src2.getReg(), Size: 64)).addImm(Val: 0);
527 else
528 MIB.add(MO: Src2).addImm(Val: 0);
529 MIB.addReg(RegNo: 0);
530 MI.removeFromParent();
531 return true;
532 }
533 }
534
535 // NonNF -> NF only if it's not a compressible NDD instruction and eflags is
536 // dead.
537 unsigned NewOpc = IsRedundantNDD
538 ? X86::getNonNDVariant(Opc)
539 : ((IsNDLike && ST.hasNF() &&
540 MI.registerDefIsDead(Reg: X86::EFLAGS, /*TRI=*/nullptr))
541 ? X86::getNFVariant(Opc)
542 : GetCompressedOpc(Opc));
543
544 if (!NewOpc)
545 return false;
546 // NF (No Flags) instructions cannot compress to VEX/legacy encoding.
547 // NF_ND can still compress to NF (both remain EVEX).
548 assert((IsND || !(TSFlags & X86II::EVEX_NF)) &&
549 "Unexpected to compress NF instructions without ND.");
550
551 const MCInstrDesc &NewDesc = ST.getInstrInfo()->get(Opcode: NewOpc);
552 MI.setDesc(NewDesc);
553 unsigned AsmComment;
554 switch (NewDesc.TSFlags & X86II::EncodingMask) {
555 case X86II::LEGACY:
556 AsmComment = X86::AC_EVEX_2_LEGACY;
557 break;
558 case X86II::VEX:
559 AsmComment = X86::AC_EVEX_2_VEX;
560 break;
561 case X86II::EVEX:
562 AsmComment = X86::AC_EVEX_2_EVEX;
563 assert(IsND && (NewDesc.TSFlags & X86II::EVEX_NF) &&
564 "Unknown EVEX2EVEX compression");
565 break;
566 default:
567 llvm_unreachable("Unknown EVEX compression");
568 }
569 MI.setAsmPrinterFlag(AsmComment);
570 if (IsRedundantNDD)
571 MI.tieOperands(DefIdx: 0, UseIdx: 1);
572
573 return true;
574}
575
576static bool runOnMF(MachineFunction &MF) {
577 LLVM_DEBUG(dbgs() << "Start X86CompressEVEXPass\n";);
578#ifndef NDEBUG
579 // Make sure the tables are sorted.
580 static std::atomic<bool> TableChecked(false);
581 if (!TableChecked.load(std::memory_order_relaxed)) {
582 assert(llvm::is_sorted(X86CompressEVEXTable) &&
583 "X86CompressEVEXTable is not sorted!");
584 TableChecked.store(true, std::memory_order_relaxed);
585 }
586#endif
587 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
588 if (!ST.hasAVX512() && !ST.hasEGPR() && !ST.hasNDD() && !ST.hasZU())
589 return false;
590
591 bool Changed = false;
592
593 for (MachineBasicBlock &MBB : MF) {
594 SmallVector<MachineInstr *, 4> ToErase;
595
596 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
597 Changed |= CompressEVEXImpl(MI, MBB, ST, ToErase);
598 }
599
600 for (MachineInstr *MI : ToErase) {
601 MI->eraseFromParent();
602 }
603 }
604 LLVM_DEBUG(dbgs() << "End X86CompressEVEXPass\n";);
605 return Changed;
606}
607
608INITIALIZE_PASS(CompressEVEXLegacy, COMP_EVEX_NAME, COMP_EVEX_DESC, false,
609 false)
610
611FunctionPass *llvm::createX86CompressEVEXLegacyPass() {
612 return new CompressEVEXLegacy();
613}
614
615bool CompressEVEXLegacy::runOnMachineFunction(MachineFunction &MF) {
616 return runOnMF(MF);
617}
618
619PreservedAnalyses
620X86CompressEVEXPass::run(MachineFunction &MF,
621 MachineFunctionAnalysisManager &MFAM) {
622 bool Changed = runOnMF(MF);
623 if (!Changed)
624 return PreservedAnalyses::all();
625 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
626 PA.preserveSet<CFGAnalyses>();
627 return PA;
628}
629