1//===- X86CompressEVEX.cpp ------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass compresses instructions from EVEX space to legacy/VEX/EVEX space
10// when possible in order to reduce code size or facilitate HW decoding.
11//
12// Possible compression:
13// a. AVX512 instruction (EVEX) -> AVX instruction (VEX)
14// b. Promoted instruction (EVEX) -> pre-promotion instruction (legacy/VEX)
15// c. NDD (EVEX) -> non-NDD (legacy)
16// d. NF_ND (EVEX) -> NF (EVEX)
17// e. NonNF (EVEX) -> NF (EVEX)
18// f. SETZUCCm (EVEX) -> SETCCm (legacy)
19// g. VPMOV*2M (EVEX) + KMOV -> VMOVMSK/VPMOVMSKB (VEX)
20// h. VPMOV*2M (EVEX) + masked VMOV* -> VBLENDV* (VEX)
21//
22// Compression a, b and c can always reduce code size, with some exceptions
23// such as promoted 16-bit CRC32 which is as long as the legacy version.
24//
25// legacy:
26// crc32w %si, %eax ## encoding: [0x66,0xf2,0x0f,0x38,0xf1,0xc6]
27// promoted:
28// crc32w %si, %eax ## encoding: [0x62,0xf4,0x7d,0x08,0xf1,0xc6]
29//
30// From performance perspective, these should be same (same uops and same EXE
31// ports). From a FMV perspective, an older legacy encoding is preferred b/c it
32// can execute in more places (broader HW install base). So we will still do
33// the compression.
34//
35// Compression d can help hardware decode (HW may skip reading the NDD
36// register) although the instruction length remains unchanged.
37//
38// Compression e can help hardware skip updating EFLAGS although the instruction
39// length remains unchanged.
40//===----------------------------------------------------------------------===//
41
42#include "MCTargetDesc/X86BaseInfo.h"
43#include "X86.h"
44#include "X86InstrInfo.h"
45#include "X86Subtarget.h"
46#include "llvm/ADT/SmallVector.h"
47#include "llvm/ADT/StringRef.h"
48#include "llvm/CodeGen/MachineFunction.h"
49#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
50#include "llvm/CodeGen/MachineFunctionPass.h"
51#include "llvm/CodeGen/MachineInstr.h"
52#include "llvm/CodeGen/MachineOperand.h"
53#include "llvm/CodeGen/MachinePassManager.h"
54#include "llvm/IR/Analysis.h"
55#include "llvm/MC/MCInstrDesc.h"
56#include "llvm/Pass.h"
57#include <atomic>
58#include <cassert>
59#include <cstdint>
60
61using namespace llvm;
62
63#define COMP_EVEX_DESC "Compressing EVEX instrs when possible"
64#define COMP_EVEX_NAME "x86-compress-evex"
65
66#define DEBUG_TYPE COMP_EVEX_NAME
67
68extern cl::opt<bool> X86EnableAPXForRelocation;
69
70namespace {
71// Including the generated EVEX compression tables.
72#define GET_X86_COMPRESS_EVEX_TABLE
73#include "X86GenInstrMapping.inc"
74
75class CompressEVEXLegacy : public MachineFunctionPass {
76public:
77 static char ID;
78 CompressEVEXLegacy() : MachineFunctionPass(ID) {}
79 StringRef getPassName() const override { return COMP_EVEX_DESC; }
80
81 bool runOnMachineFunction(MachineFunction &MF) override;
82
83 // This pass runs after regalloc and doesn't support VReg operands.
84 MachineFunctionProperties getRequiredProperties() const override {
85 return MachineFunctionProperties().setNoVRegs();
86 }
87};
88
89} // end anonymous namespace
90
91char CompressEVEXLegacy::ID = 0;
92
93static bool usesExtendedRegister(const MachineInstr &MI) {
94 auto isHiRegIdx = [](MCRegister Reg) {
95 // Check for XMM register with indexes between 16 - 31.
96 if (Reg >= X86::XMM16 && Reg <= X86::XMM31)
97 return true;
98 // Check for YMM register with indexes between 16 - 31.
99 if (Reg >= X86::YMM16 && Reg <= X86::YMM31)
100 return true;
101 // Check for GPR with indexes between 16 - 31.
102 if (X86II::isApxExtendedReg(Reg))
103 return true;
104 return false;
105 };
106
107 // Check that operands are not ZMM regs or
108 // XMM/YMM regs with hi indexes between 16 - 31.
109 for (const MachineOperand &MO : MI.explicit_operands()) {
110 if (!MO.isReg())
111 continue;
112
113 MCRegister Reg = MO.getReg().asMCReg();
114 assert(!X86II::isZMMReg(Reg) &&
115 "ZMM instructions should not be in the EVEX->VEX tables");
116 if (isHiRegIdx(Reg))
117 return true;
118 }
119
120 return false;
121}
122
123// Return true if the EVEX form of \p MI can encode its memory displacement as
124// a compressed disp8*N (1 byte) while the VEX/legacy twin would be forced to
125// spend a full disp32 (4 bytes). In that window the EVEX encoding is strictly
126// shorter overall, despite its 1-2 byte larger prefix, so compressing it to
127// VEX would grow code size.
128static bool hasShorterEVEXViaCDisp8(const MachineInstr &MI) {
129 int MemOpIdx = X86::getFirstAddrOperandIdx(MI);
130 if (MemOpIdx < 0)
131 return false;
132
133 const MachineOperand &Disp = MI.getOperand(i: MemOpIdx + X86::AddrDisp);
134 // Only a constant displacement can be range-checked here; symbolic ones
135 // (globals, constant pool, jump tables, ...) are resolved later.
136 if (!Disp.isImm())
137 return false;
138
139 int64_t Val = Disp.getImm();
140 return !isInt<8>(x: Val) && X86II::isDispOrCDisp8(TSFlags: MI.getDesc().TSFlags, Value: Val);
141}
142
143// Do any custom cleanup needed to finalize the conversion.
144static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) {
145 (void)NewOpc;
146 unsigned Opc = MI.getOpcode();
147 switch (Opc) {
148 case X86::VALIGNDZ128rri:
149 case X86::VALIGNDZ128rmi:
150 case X86::VALIGNQZ128rri:
151 case X86::VALIGNQZ128rmi: {
152 assert((NewOpc == X86::VPALIGNRrri || NewOpc == X86::VPALIGNRrmi) &&
153 "Unexpected new opcode!");
154 unsigned Scale =
155 (Opc == X86::VALIGNQZ128rri || Opc == X86::VALIGNQZ128rmi) ? 8 : 4;
156 MachineOperand &Imm = MI.getOperand(i: MI.getNumExplicitOperands() - 1);
157 Imm.setImm(Imm.getImm() * Scale);
158 break;
159 }
160 case X86::VSHUFF32X4Z256rmi:
161 case X86::VSHUFF32X4Z256rri:
162 case X86::VSHUFF64X2Z256rmi:
163 case X86::VSHUFF64X2Z256rri:
164 case X86::VSHUFI32X4Z256rmi:
165 case X86::VSHUFI32X4Z256rri:
166 case X86::VSHUFI64X2Z256rmi:
167 case X86::VSHUFI64X2Z256rri: {
168 assert((NewOpc == X86::VPERM2F128rri || NewOpc == X86::VPERM2I128rri ||
169 NewOpc == X86::VPERM2F128rmi || NewOpc == X86::VPERM2I128rmi) &&
170 "Unexpected new opcode!");
171 MachineOperand &Imm = MI.getOperand(i: MI.getNumExplicitOperands() - 1);
172 int64_t ImmVal = Imm.getImm();
173 // Set bit 5, move bit 1 to bit 4, copy bit 0.
174 Imm.setImm(0x20 | ((ImmVal & 2) << 3) | (ImmVal & 1));
175 break;
176 }
177 case X86::VRNDSCALEPDZ128rri:
178 case X86::VRNDSCALEPDZ128rmi:
179 case X86::VRNDSCALEPSZ128rri:
180 case X86::VRNDSCALEPSZ128rmi:
181 case X86::VRNDSCALEPDZ256rri:
182 case X86::VRNDSCALEPDZ256rmi:
183 case X86::VRNDSCALEPSZ256rri:
184 case X86::VRNDSCALEPSZ256rmi:
185 case X86::VRNDSCALESDZrri:
186 case X86::VRNDSCALESDZrmi:
187 case X86::VRNDSCALESSZrri:
188 case X86::VRNDSCALESSZrmi:
189 case X86::VRNDSCALESDZrri_Int:
190 case X86::VRNDSCALESDZrmi_Int:
191 case X86::VRNDSCALESSZrri_Int:
192 case X86::VRNDSCALESSZrmi_Int:
193 const MachineOperand &Imm = MI.getOperand(i: MI.getNumExplicitOperands() - 1);
194 int64_t ImmVal = Imm.getImm();
195 // Ensure that only bits 3:0 of the immediate are used.
196 if ((ImmVal & 0xf) != ImmVal)
197 return false;
198 break;
199 }
200
201 return true;
202}
203
204static unsigned getMovMskBits(unsigned Opc) {
205 switch (Opc) {
206 case X86::VPMOVQ2MZ128kr:
207 case X86::VPCMPQZ128rri:
208 return 2;
209 case X86::VPMOVQ2MZ256kr:
210 case X86::VPMOVD2MZ128kr:
211 case X86::VPCMPQZ256rri:
212 case X86::VPCMPDZ128rri:
213 return 4;
214 case X86::VPMOVD2MZ256kr:
215 case X86::VPCMPDZ256rri:
216 return 8;
217 case X86::VPMOVB2MZ128kr:
218 case X86::VPCMPBZ128rri:
219 return 16;
220 case X86::VPMOVB2MZ256kr:
221 case X86::VPCMPBZ256rri:
222 return 32;
223 default:
224 llvm_unreachable("Unknown opcode");
225 }
226}
227
228static bool isKMovNarrowing(unsigned MaskBits, unsigned KMOVOpc) {
229 unsigned KMOVSize = 0;
230 switch (KMOVOpc) {
231 case X86::KMOVBrk:
232 KMOVSize = 8;
233 break;
234 case X86::KMOVWrk:
235 KMOVSize = 16;
236 break;
237 case X86::KMOVDrk:
238 KMOVSize = 32;
239 break;
240 default:
241 llvm_unreachable("Unknown KMOV opcode");
242 }
243
244 return KMOVSize < MaskBits;
245}
246
247static bool isZeroVector(const MachineInstr &MI) {
248 switch (MI.getOpcode()) {
249 case X86::VPXORrr:
250 case X86::VPXORYrr:
251 case X86::VXORPSrr:
252 case X86::VXORPSYrr:
253 return MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
254 default:
255 return false;
256 }
257}
258
259static bool isAllOnesVector(const MachineInstr &MI, bool Is256Bit) {
260 switch (MI.getOpcode()) {
261 case X86::VPCMPEQDrr:
262 return !Is256Bit && MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
263 case X86::VPCMPEQDYrr:
264 return MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
265 default:
266 return false;
267 }
268}
269
270static MachineInstr *getSignMaskConstantDef(MachineInstr &MI, Register Reg,
271 bool IsZero, bool Is256Bit,
272 const TargetRegisterInfo *TRI) {
273 for (MachineInstr &DefMI : llvm::reverse(C: llvm::make_range(
274 x: MI.getParent()->begin(), y: MachineBasicBlock::iterator(MI)))) {
275 if (!DefMI.modifiesRegister(Reg, TRI))
276 continue;
277 // Stop at the nearest def/clobber; an older matching constant may no
278 // longer be the reaching definition.
279 if (IsZero ? isZeroVector(MI: DefMI) : isAllOnesVector(MI: DefMI, Is256Bit))
280 return &DefMI;
281 break;
282 }
283 return nullptr;
284}
285
286static bool isCompressibleBlendVUse(unsigned BlendOpc, unsigned UseOpc) {
287 switch (BlendOpc) {
288 case X86::VBLENDVPSrrr:
289 switch (UseOpc) {
290 case X86::VMOVAPSZ128rrk:
291 case X86::VMOVUPSZ128rrk:
292 case X86::VMOVDQA32Z128rrk:
293 case X86::VMOVDQU32Z128rrk:
294 return true;
295 default:
296 return false;
297 }
298 case X86::VBLENDVPSYrrr:
299 switch (UseOpc) {
300 case X86::VMOVAPSZ256rrk:
301 case X86::VMOVUPSZ256rrk:
302 case X86::VMOVDQA32Z256rrk:
303 case X86::VMOVDQU32Z256rrk:
304 return true;
305 default:
306 return false;
307 }
308 case X86::VBLENDVPDrrr:
309 switch (UseOpc) {
310 case X86::VMOVAPDZ128rrk:
311 case X86::VMOVUPDZ128rrk:
312 case X86::VMOVDQA64Z128rrk:
313 case X86::VMOVDQU64Z128rrk:
314 return true;
315 default:
316 return false;
317 }
318 case X86::VBLENDVPDYrrr:
319 switch (UseOpc) {
320 case X86::VMOVAPDZ256rrk:
321 case X86::VMOVUPDZ256rrk:
322 case X86::VMOVDQA64Z256rrk:
323 case X86::VMOVDQU64Z256rrk:
324 return true;
325 default:
326 return false;
327 }
328 case X86::VPBLENDVBrrr:
329 return UseOpc == X86::VMOVDQU8Z128rrk;
330 case X86::VPBLENDVBYrrr:
331 return UseOpc == X86::VMOVDQU8Z256rrk;
332 default:
333 return false;
334 }
335}
336
337static bool isCompressibleMaskedBlendUse(unsigned BlendOpc, unsigned UseOpc) {
338 switch (BlendOpc) {
339 case X86::VBLENDVPSrrr:
340 return UseOpc == X86::VPBLENDMDZ128rrk || UseOpc == X86::VBLENDMPSZ128rrk;
341 case X86::VBLENDVPSYrrr:
342 return UseOpc == X86::VPBLENDMDZ256rrk || UseOpc == X86::VBLENDMPSZ256rrk;
343 case X86::VBLENDVPDrrr:
344 return UseOpc == X86::VPBLENDMQZ128rrk || UseOpc == X86::VBLENDMPDZ128rrk;
345 case X86::VBLENDVPDYrrr:
346 return UseOpc == X86::VPBLENDMQZ256rrk || UseOpc == X86::VBLENDMPDZ256rrk;
347 default:
348 return false;
349 }
350}
351
352// Try to compress mask producer chains:
353// vpmov*2m %xmm0, %k0 -> (erase this)
354// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
355//
356// vpcmpge* $0, %xmm0, %k0 -> (erase this) (X >= 0)
357// vpcmpgt* $-1, %xmm0, %k0 -> (erase this) (X > -1)
358// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
359// bounded complement of %eax
360//
361// vpmov*2m %xmm0, %k1 -> (erase this)
362// vmov* %xmm1, %xmm2 {%k1} -> vblendv* %xmm0, %xmm2, %xmm1, %xmm2
363static bool tryCompressMaskProducer(MachineInstr &MI, MachineBasicBlock &MBB,
364 const X86Subtarget &ST,
365 SmallVectorImpl<MachineInstr *> &ToErase) {
366 const X86InstrInfo *TII = ST.getInstrInfo();
367 const TargetRegisterInfo *TRI = ST.getRegisterInfo();
368 MachineRegisterInfo *MRI = &MBB.getParent()->getRegInfo();
369
370 unsigned Opc = MI.getOpcode();
371 bool IsSignMaskCmp = Opc == X86::VPCMPBZ128rri || Opc == X86::VPCMPBZ256rri ||
372 Opc == X86::VPCMPDZ128rri || Opc == X86::VPCMPDZ256rri ||
373 Opc == X86::VPCMPQZ128rri || Opc == X86::VPCMPQZ256rri;
374 if (!IsSignMaskCmp && Opc != X86::VPMOVD2MZ128kr &&
375 Opc != X86::VPMOVD2MZ256kr && Opc != X86::VPMOVQ2MZ128kr &&
376 Opc != X86::VPMOVQ2MZ256kr && Opc != X86::VPMOVB2MZ128kr &&
377 Opc != X86::VPMOVB2MZ256kr)
378 return false;
379
380 if (usesExtendedRegister(MI))
381 return false;
382
383 Register MaskReg = MI.getOperand(i: 0).getReg();
384 Register SrcVecReg = MI.getOperand(i: 1).getReg();
385 MachineInstr *ConstantDef = nullptr;
386 bool ConstantDefOnlyFeedsCmp = false;
387
388 if (IsSignMaskCmp) {
389 int64_t Pred = MI.getOperand(i: 3).getImm();
390 // VPCMP signed predicates: nlt (5) folds X >= 0, nle (6) folds X > -1.
391 if (Pred != 5 && Pred != 6)
392 return false;
393 Register ConstantReg = MI.getOperand(i: 2).getReg();
394 bool Is256Bit = Opc == X86::VPCMPBZ256rri || Opc == X86::VPCMPDZ256rri ||
395 Opc == X86::VPCMPQZ256rri;
396 // The sign-mask fold is valid only for compares against the reaching
397 // zero/all-ones vector definition.
398 ConstantDef =
399 getSignMaskConstantDef(MI, Reg: ConstantReg, IsZero: Pred == 5, Is256Bit, TRI);
400 if (!ConstantDef)
401 return false;
402 // If the constant feeds only this compare, erase it with the compare.
403 ConstantDefOnlyFeedsCmp = !TRI->regsOverlap(RegA: ConstantReg, RegB: SrcVecReg);
404 for (MachineInstr &UseMI :
405 llvm::make_range(x: std::next(x: MachineBasicBlock::iterator(*ConstantDef)),
406 y: MachineBasicBlock::iterator(MI)))
407 if (UseMI.readsRegister(Reg: ConstantReg, TRI)) {
408 ConstantDefOnlyFeedsCmp = false;
409 break;
410 }
411 }
412
413 unsigned MovMskOpc = 0;
414 unsigned BlendOpc = 0;
415 switch (Opc) {
416 case X86::VPCMPDZ128rri:
417 case X86::VPMOVD2MZ128kr:
418 MovMskOpc = X86::VMOVMSKPSrr;
419 BlendOpc = X86::VBLENDVPSrrr;
420 break;
421 case X86::VPCMPDZ256rri:
422 case X86::VPMOVD2MZ256kr:
423 MovMskOpc = X86::VMOVMSKPSYrr;
424 BlendOpc = X86::VBLENDVPSYrrr;
425 break;
426 case X86::VPCMPQZ128rri:
427 case X86::VPMOVQ2MZ128kr:
428 MovMskOpc = X86::VMOVMSKPDrr;
429 BlendOpc = X86::VBLENDVPDrrr;
430 break;
431 case X86::VPCMPQZ256rri:
432 case X86::VPMOVQ2MZ256kr:
433 MovMskOpc = X86::VMOVMSKPDYrr;
434 BlendOpc = X86::VBLENDVPDYrrr;
435 break;
436 case X86::VPCMPBZ128rri:
437 case X86::VPMOVB2MZ128kr:
438 MovMskOpc = X86::VPMOVMSKBrr;
439 BlendOpc = X86::VPBLENDVBrrr;
440 break;
441 case X86::VPCMPBZ256rri:
442 case X86::VPMOVB2MZ256kr:
443 MovMskOpc = X86::VPMOVMSKBYrr;
444 BlendOpc = X86::VPBLENDVBYrrr;
445 break;
446 default:
447 llvm_unreachable("Unknown VPMOV opcode");
448 }
449
450 MachineInstr *KMovMI = nullptr;
451 MachineInstr *BlendMI = nullptr;
452 bool BlendIsMaskedBlend = false;
453
454 for (MachineInstr &CurMI : llvm::make_range(
455 x: std::next(x: MachineBasicBlock::iterator(MI)), y: MBB.end())) {
456 if (CurMI.readsRegister(Reg: MaskReg, TRI)) {
457 if (KMovMI || BlendMI)
458 return false; // Fail: Mask has MULTIPLE uses
459
460 unsigned UseOpc = CurMI.getOpcode();
461 bool IsKMOV = UseOpc == X86::KMOVBrk || UseOpc == X86::KMOVWrk ||
462 UseOpc == X86::KMOVDrk;
463 // Only allow non-narrowing KMOV uses of the mask.
464 if (IsKMOV && CurMI.getOperand(i: 1).getReg() == MaskReg &&
465 !usesExtendedRegister(MI: CurMI) &&
466 !isKMovNarrowing(MaskBits: getMovMskBits(Opc), KMOVOpc: UseOpc)) {
467 KMovMI = &CurMI;
468 // continue scanning to ensure
469 // there are no *other* uses of the mask later in the block.
470 } else {
471 bool IsMaskedMove =
472 !IsSignMaskCmp && isCompressibleBlendVUse(BlendOpc, UseOpc);
473 bool IsMaskedBlend =
474 !IsSignMaskCmp && isCompressibleMaskedBlendUse(BlendOpc, UseOpc);
475
476 if (!IsMaskedMove && !IsMaskedBlend)
477 return false;
478
479 unsigned MaskOpIdx = IsMaskedBlend ? 1 : 2;
480 if (CurMI.getOperand(i: MaskOpIdx).getReg() == MaskReg &&
481 !usesExtendedRegister(MI: CurMI) && checkPredicate(Opc: BlendOpc, Subtarget: &ST)) {
482 BlendMI = &CurMI;
483 BlendIsMaskedBlend = IsMaskedBlend;
484 } else {
485 return false;
486 }
487 }
488 }
489
490 if (CurMI.modifiesRegister(Reg: MaskReg, TRI)) {
491 if (!KMovMI && !BlendMI)
492 return false; // Mask clobbered before use
493 break;
494 }
495
496 if (!KMovMI && !BlendMI && CurMI.modifiesRegister(Reg: SrcVecReg, TRI)) {
497 return false; // SrcVecReg modified before it could be reused
498 }
499 }
500
501 if (!KMovMI && !BlendMI)
502 return false;
503
504 unsigned MovMskBits = getMovMskBits(Opc);
505 // Bounded complements define EFLAGS, unlike VPCMP + KMOV. A 32-bit
506 // complement uses NOT, which does not modify EFLAGS.
507 if (IsSignMaskCmp && KMovMI) {
508 if (KMovMI->getOperand(i: 0).isDead() ||
509 (MovMskBits != 32 &&
510 MBB.computeRegisterLiveness(
511 TRI, Reg: X86::EFLAGS,
512 Before: std::next(x: MachineBasicBlock::const_iterator(*KMovMI)),
513 Neighborhood: MBB.size()) != MachineBasicBlock::LQR_Dead))
514 return false;
515 }
516
517 // Check if MaskReg is used in any other basic blocks
518 for (const MachineInstr &UseMI : MRI->use_instructions(Reg: MaskReg))
519 if (UseMI.getParent() != &MBB)
520 return false;
521
522 // Apply the transformation
523 MachineInstr *NewMI = nullptr;
524 if (KMovMI) {
525 MachineOperand OldDst = KMovMI->getOperand(i: 0);
526 KMovMI->setDesc(TII->get(Opcode: MovMskOpc));
527 MachineOperand &NewSrc = KMovMI->getOperand(i: 1);
528 NewSrc.setReg(SrcVecReg);
529 // setReg() keeps the mask operand's kill flag; take the source's kill
530 // state from the VPMOV instead.
531 NewSrc.setIsKill(MI.getOperand(i: 1).isKill());
532 NewMI = KMovMI;
533 if (IsSignMaskCmp) {
534 Register DstReg = OldDst.getReg();
535 int64_t ComplementMask =
536 APInt::getLowBitsSet(numBits: 32, loBitsSet: MovMskBits).getSExtValue();
537 unsigned ComplementOpc =
538 MovMskBits == 32
539 ? X86::NOT32r
540 : (isInt<8>(x: ComplementMask) ? X86::XOR32ri8 : X86::XOR32ri);
541 auto MIB = BuildMI(BB&: MBB, I: std::next(x: MachineBasicBlock::iterator(*KMovMI)),
542 MIMD: KMovMI->getDebugLoc(), MCID: TII->get(Opcode: ComplementOpc), DestReg: DstReg)
543 .addReg(RegNo: DstReg, Flags: RegState::Kill);
544 if (MovMskBits != 32) {
545 MIB.addImm(Val: ComplementMask);
546 MIB->findRegisterDefOperand(Reg: X86::EFLAGS, TRI)->setIsDead();
547 }
548 MIB->getOperand(i: 0).setIsRenamable(OldDst.isRenamable());
549 }
550 } else if (BlendMI) {
551 const MachineOperand &MaskVec = MI.getOperand(i: 1);
552 const MachineOperand &Dst = BlendMI->getOperand(i: 0);
553 const MachineOperand &Passthru =
554 BlendMI->getOperand(i: BlendIsMaskedBlend ? 2 : 1);
555 const MachineOperand &Src = BlendMI->getOperand(i: 3);
556
557 // Build a replacement instead of changing BlendMI in place because
558 // masked VMOV and VPBLENDM have different operand layouts from VBLENDV.
559 auto MIB =
560 BuildMI(BB&: MBB, I&: *BlendMI, MIMD: BlendMI->getDebugLoc(), MCID: TII->get(Opcode: BlendOpc))
561 .addReg(RegNo: Dst.getReg(), Flags: getRegState(RegOp: Dst))
562 .addReg(RegNo: Passthru.getReg(), Flags: getRegState(RegOp: Passthru))
563 .addReg(RegNo: Src.getReg(), Flags: getRegState(RegOp: Src))
564 .addReg(RegNo: MaskVec.getReg(), Flags: getRegState(RegOp: MaskVec));
565 NewMI = MIB;
566 ToErase.push_back(Elt: BlendMI);
567 }
568 assert(NewMI && "Expected a compressed instruction");
569 NewMI->setAsmPrinterFlag(X86::AC_EVEX_2_VEX);
570 ToErase.push_back(Elt: &MI);
571 if (ConstantDefOnlyFeedsCmp && MI.getOperand(i: 2).isKill())
572 ToErase.push_back(Elt: ConstantDef);
573 return true;
574}
575
576static bool CompressEVEXImpl(MachineInstr &MI, MachineBasicBlock &MBB,
577 const X86Subtarget &ST,
578 SmallVectorImpl<MachineInstr *> &ToErase) {
579 uint64_t TSFlags = MI.getDesc().TSFlags;
580
581 // Check for EVEX instructions only.
582 if ((TSFlags & X86II::EncodingMask) != X86II::EVEX)
583 return false;
584
585 // Instructions with mask or 512-bit vector can't be converted to VEX.
586 if (TSFlags & (X86II::EVEX_K | X86II::EVEX_L2))
587 return false;
588
589 // Keep the EVEX encoding when there's 1-byte compressed disp8*N.
590 if (hasShorterEVEXViaCDisp8(MI))
591 return false;
592
593 // Specialized mask-producing folds to MOVMSK/VBLENDV first.
594 if (tryCompressMaskProducer(MI, MBB, ST, ToErase))
595 return true;
596
597 auto IsRedundantNewDataDest = [&](unsigned &Opc) {
598 // $rbx = ADD64rr_ND $rbx, $rax / $rbx = ADD64rr_ND $rax, $rbx
599 // ->
600 // $rbx = ADD64rr $rbx, $rax
601 const MCInstrDesc &Desc = MI.getDesc();
602 Register Reg0 = MI.getOperand(i: 0).getReg();
603 const MachineOperand &Op1 = MI.getOperand(i: 1);
604 if (!Op1.isReg() || X86::getFirstAddrOperandIdx(MI) == 1 ||
605 X86::isCFCMOVCC(Opcode: MI.getOpcode()))
606 return false;
607 Register Reg1 = Op1.getReg();
608 if (Reg1 == Reg0)
609 return true;
610
611 // Op1 and Op2 may be commutable for ND instructions.
612 if (!Desc.isCommutable() || Desc.getNumOperands() < 3 ||
613 !MI.getOperand(i: 2).isReg() || MI.getOperand(i: 2).getReg() != Reg0)
614 return false;
615 // Opcode may change after commute, e.g. SHRD -> SHLD
616 ST.getInstrInfo()->commuteInstruction(MI, NewMI: false, OpIdx1: 1, OpIdx2: 2);
617 Opc = MI.getOpcode();
618 return true;
619 };
620
621 // EVEX_B has several meanings.
622 // AVX512:
623 // register form: rounding control or SAE
624 // memory form: broadcast
625 //
626 // APX:
627 // MAP4: NDD, ZU
628 //
629 // For AVX512 cases, EVEX prefix is needed in order to carry this information
630 // thus preventing the transformation to VEX encoding.
631 bool IsND = X86II::hasNewDataDest(TSFlags);
632 unsigned Opc = MI.getOpcode();
633 bool IsSetZUCCm = Opc == X86::SETZUCCm;
634 if (TSFlags & X86II::EVEX_B && !IsND && !IsSetZUCCm)
635 return false;
636 // MOVBE*rr is special because it has semantic of NDD but not set EVEX_B.
637 bool IsNDLike = IsND || Opc == X86::MOVBE32rr || Opc == X86::MOVBE64rr;
638 bool IsRedundantNDD = IsNDLike ? IsRedundantNewDataDest(Opc) : false;
639
640 auto GetCompressedOpc = [&](unsigned Opc) -> unsigned {
641 ArrayRef<X86TableEntry> Table = ArrayRef(X86CompressEVEXTable);
642 const auto I = llvm::lower_bound(Range&: Table, Value&: Opc);
643 if (I == Table.end() || I->OldOpc != Opc)
644 return 0;
645
646 if (usesExtendedRegister(MI) || !checkPredicate(Opc: I->NewOpc, Subtarget: &ST) ||
647 !performCustomAdjustments(MI, NewOpc: I->NewOpc))
648 return 0;
649 return I->NewOpc;
650 };
651
652 Register Dst = MI.getOperand(i: 0).getReg();
653 if (IsRedundantNDD) {
654 // Redundant NDD ops cannot be safely compressed if either:
655 // - the legacy op would introduce a partial write that BreakFalseDeps
656 // identified as a potential stall, or
657 // - the op is writing to a subregister of a live register, i.e. the
658 // full (zeroed) result is used.
659 // Both cases are indicated by an implicit def of the superregister.
660 if (Dst &&
661 (X86::GR16RegClass.contains(Reg: Dst) || X86::GR8RegClass.contains(Reg: Dst))) {
662 Register Super = getX86SubSuperRegister(Reg: Dst, Size: 64);
663 if (MI.definesRegister(Reg: Super, /*TRI=*/nullptr))
664 IsRedundantNDD = false;
665 }
666
667 // ADDrm/mr instructions with NDD + relocation had been transformed to the
668 // instructions without NDD in X86SuppressAPXForRelocation pass. That is to
669 // keep backward compatibility with linkers without APX support.
670 if (!X86EnableAPXForRelocation)
671 assert(!isAddMemInstrWithRelocation(MI) &&
672 "Unexpected NDD instruction with relocation!");
673 } else if (Opc == X86::ADD32ri_ND || Opc == X86::ADD64ri32_ND ||
674 Opc == X86::ADD32rr_ND || Opc == X86::ADD64rr_ND) {
675 // Non-redundant NDD ADD can be compressed to LEA when:
676 // - No EGPR register used and
677 // - EFLAGS is dead.
678 if (!usesExtendedRegister(MI) &&
679 MI.registerDefIsDead(Reg: X86::EFLAGS, /*TRI=*/nullptr)) {
680 Register Src1 = MI.getOperand(i: 1).getReg();
681 const MachineOperand &Src2 = MI.getOperand(i: 2);
682 bool Is32BitReg = Opc == X86::ADD32ri_ND || Opc == X86::ADD32rr_ND;
683 const MCInstrDesc &NewDesc =
684 ST.getInstrInfo()->get(Opcode: Is32BitReg ? X86::LEA64_32r : X86::LEA64r);
685 if (Is32BitReg)
686 Src1 = getX86SubSuperRegister(Reg: Src1, Size: 64);
687 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: NewDesc, DestReg: Dst)
688 .addReg(RegNo: Src1)
689 .addImm(Val: 1);
690 if (Opc == X86::ADD32ri_ND || Opc == X86::ADD64ri32_ND)
691 MIB.addReg(RegNo: 0).add(MO: Src2);
692 else if (Is32BitReg)
693 MIB.addReg(RegNo: getX86SubSuperRegister(Reg: Src2.getReg(), Size: 64)).addImm(Val: 0);
694 else
695 MIB.add(MO: Src2).addImm(Val: 0);
696 MIB.addReg(RegNo: 0);
697 MI.removeFromParent();
698 return true;
699 }
700 }
701
702 // NonNF -> NF only if it's not a compressible NDD instruction and eflags is
703 // dead.
704 unsigned NewOpc = IsRedundantNDD
705 ? X86::getNonNDVariant(Opc)
706 : ((IsNDLike && ST.hasNF() &&
707 MI.registerDefIsDead(Reg: X86::EFLAGS, /*TRI=*/nullptr))
708 ? X86::getNFVariant(Opc)
709 : GetCompressedOpc(Opc));
710
711 if (!NewOpc)
712 return false;
713 // NF (No Flags) instructions cannot compress to VEX/legacy encoding.
714 // NF_ND can still compress to NF (both remain EVEX).
715 assert((IsND || !(TSFlags & X86II::EVEX_NF)) &&
716 "Unexpected to compress NF instructions without ND.");
717
718 const MCInstrDesc &NewDesc = ST.getInstrInfo()->get(Opcode: NewOpc);
719 MI.setDesc(NewDesc);
720 unsigned AsmComment;
721 switch (NewDesc.TSFlags & X86II::EncodingMask) {
722 case X86II::LEGACY:
723 AsmComment = X86::AC_EVEX_2_LEGACY;
724 break;
725 case X86II::VEX:
726 AsmComment = X86::AC_EVEX_2_VEX;
727 break;
728 case X86II::EVEX:
729 AsmComment = X86::AC_EVEX_2_EVEX;
730 assert(IsND && (NewDesc.TSFlags & X86II::EVEX_NF) &&
731 "Unknown EVEX2EVEX compression");
732 break;
733 default:
734 llvm_unreachable("Unknown EVEX compression");
735 }
736 MI.setAsmPrinterFlag(AsmComment);
737 if (IsRedundantNDD)
738 MI.tieOperands(DefIdx: 0, UseIdx: 1);
739
740 return true;
741}
742
743static bool runOnMF(MachineFunction &MF) {
744 LLVM_DEBUG(dbgs() << "Start X86CompressEVEXPass\n";);
745#ifndef NDEBUG
746 // Make sure the tables are sorted.
747 static std::atomic<bool> TableChecked(false);
748 if (!TableChecked.load(std::memory_order_relaxed)) {
749 assert(llvm::is_sorted(X86CompressEVEXTable) &&
750 "X86CompressEVEXTable is not sorted!");
751 TableChecked.store(true, std::memory_order_relaxed);
752 }
753#endif
754 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
755 if (!ST.hasAVX512() && !ST.hasEGPR() && !ST.hasNDD() && !ST.hasZU())
756 return false;
757
758 bool Changed = false;
759
760 for (MachineBasicBlock &MBB : MF) {
761 SmallVector<MachineInstr *, 4> ToErase;
762
763 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
764 Changed |= CompressEVEXImpl(MI, MBB, ST, ToErase);
765 }
766
767 for (MachineInstr *MI : ToErase) {
768 MI->eraseFromParent();
769 }
770 }
771 LLVM_DEBUG(dbgs() << "End X86CompressEVEXPass\n";);
772 return Changed;
773}
774
775INITIALIZE_PASS(CompressEVEXLegacy, COMP_EVEX_NAME, COMP_EVEX_DESC, false,
776 false)
777
778FunctionPass *llvm::createX86CompressEVEXLegacyPass() {
779 return new CompressEVEXLegacy();
780}
781
782bool CompressEVEXLegacy::runOnMachineFunction(MachineFunction &MF) {
783 return runOnMF(MF);
784}
785
786PreservedAnalyses
787X86CompressEVEXPass::run(MachineFunction &MF,
788 MachineFunctionAnalysisManager &MFAM) {
789 bool Changed = runOnMF(MF);
790 if (!Changed)
791 return PreservedAnalyses::all();
792 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
793 PA.preserveSet<CFGAnalyses>();
794 return PA;
795}
796