1//===- X86CompressEVEX.cpp ------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass compresses instructions from EVEX space to legacy/VEX/EVEX space
10// when possible in order to reduce code size or facilitate HW decoding.
11//
12// Possible compression:
13// a. AVX512 instruction (EVEX) -> AVX instruction (VEX)
14// b. Promoted instruction (EVEX) -> pre-promotion instruction (legacy/VEX)
15// c. NDD (EVEX) -> non-NDD (legacy)
16// d. NF_ND (EVEX) -> NF (EVEX)
17// e. NonNF (EVEX) -> NF (EVEX)
18// f. SETZUCCm (EVEX) -> SETCCm (legacy)
19// g. VPMOV*2M (EVEX) + KMOV -> VMOVMSK/VPMOVMSKB (VEX)
20// h. VPMOV*2M (EVEX) + masked VMOV* -> VBLENDV* (VEX)
21//
22// Compression a, b and c can always reduce code size, with some exceptions
23// such as promoted 16-bit CRC32 which is as long as the legacy version.
24//
25// legacy:
26// crc32w %si, %eax ## encoding: [0x66,0xf2,0x0f,0x38,0xf1,0xc6]
27// promoted:
28// crc32w %si, %eax ## encoding: [0x62,0xf4,0x7d,0x08,0xf1,0xc6]
29//
30// From performance perspective, these should be same (same uops and same EXE
31// ports). From a FMV perspective, an older legacy encoding is preferred b/c it
32// can execute in more places (broader HW install base). So we will still do
33// the compression.
34//
35// Compression d can help hardware decode (HW may skip reading the NDD
36// register) although the instruction length remains unchanged.
37//
38// Compression e can help hardware skip updating EFLAGS although the instruction
39// length remains unchanged.
40//===----------------------------------------------------------------------===//
41
42#include "MCTargetDesc/X86BaseInfo.h"
43#include "X86.h"
44#include "X86InstrInfo.h"
45#include "X86Subtarget.h"
46#include "llvm/ADT/SmallVector.h"
47#include "llvm/ADT/StringRef.h"
48#include "llvm/CodeGen/MachineFunction.h"
49#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
50#include "llvm/CodeGen/MachineFunctionPass.h"
51#include "llvm/CodeGen/MachineInstr.h"
52#include "llvm/CodeGen/MachineOperand.h"
53#include "llvm/CodeGen/MachinePassManager.h"
54#include "llvm/IR/Analysis.h"
55#include "llvm/MC/MCInstrDesc.h"
56#include "llvm/Pass.h"
57#include <atomic>
58#include <cassert>
59#include <cstdint>
60
61using namespace llvm;
62
63#define COMP_EVEX_DESC "Compressing EVEX instrs when possible"
64#define COMP_EVEX_NAME "x86-compress-evex"
65
66#define DEBUG_TYPE COMP_EVEX_NAME
67
68namespace {
69// Including the generated EVEX compression tables.
70#define GET_X86_COMPRESS_EVEX_TABLE
71#include "X86GenInstrMapping.inc"
72
73class CompressEVEXLegacy : public MachineFunctionPass {
74public:
75 static char ID;
76 CompressEVEXLegacy() : MachineFunctionPass(ID) {}
77 StringRef getPassName() const override { return COMP_EVEX_DESC; }
78
79 bool runOnMachineFunction(MachineFunction &MF) override;
80
81 // This pass runs after regalloc and doesn't support VReg operands.
82 MachineFunctionProperties getRequiredProperties() const override {
83 return MachineFunctionProperties().setNoVRegs();
84 }
85};
86
87} // end anonymous namespace
88
89char CompressEVEXLegacy::ID = 0;
90
91static bool usesExtendedRegister(const MachineInstr &MI) {
92 auto isHiRegIdx = [](MCRegister Reg) {
93 // Check for XMM register with indexes between 16 - 31.
94 if (Reg >= X86::XMM16 && Reg <= X86::XMM31)
95 return true;
96 // Check for YMM register with indexes between 16 - 31.
97 if (Reg >= X86::YMM16 && Reg <= X86::YMM31)
98 return true;
99 // Check for GPR with indexes between 16 - 31.
100 if (X86II::isApxExtendedReg(Reg))
101 return true;
102 return false;
103 };
104
105 // Check that operands are not ZMM regs or
106 // XMM/YMM regs with hi indexes between 16 - 31.
107 for (const MachineOperand &MO : MI.explicit_operands()) {
108 if (!MO.isReg())
109 continue;
110
111 MCRegister Reg = MO.getReg().asMCReg();
112 assert(!X86II::isZMMReg(Reg) &&
113 "ZMM instructions should not be in the EVEX->VEX tables");
114 if (isHiRegIdx(Reg))
115 return true;
116 }
117
118 return false;
119}
120
121// Return true if the EVEX form of \p MI can encode its memory displacement as
122// a compressed disp8*N (1 byte) while the VEX/legacy twin would be forced to
123// spend a full disp32 (4 bytes). In that window the EVEX encoding is strictly
124// shorter overall, despite its 1-2 byte larger prefix, so compressing it to
125// VEX would grow code size.
126static bool hasShorterEVEXViaCDisp8(const MachineInstr &MI) {
127 int MemOpIdx = X86::getFirstAddrOperandIdx(MI);
128 if (MemOpIdx < 0)
129 return false;
130
131 const MachineOperand &Disp = MI.getOperand(i: MemOpIdx + X86::AddrDisp);
132 // Only a constant displacement can be range-checked here; symbolic ones
133 // (globals, constant pool, jump tables, ...) are resolved later.
134 if (!Disp.isImm())
135 return false;
136
137 int64_t Val = Disp.getImm();
138 return !isInt<8>(x: Val) && X86II::isDispOrCDisp8(TSFlags: MI.getDesc().TSFlags, Value: Val);
139}
140
141// Do any custom cleanup needed to finalize the conversion.
142static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) {
143 (void)NewOpc;
144 unsigned Opc = MI.getOpcode();
145 switch (Opc) {
146 case X86::VALIGNDZ128rri:
147 case X86::VALIGNDZ128rmi:
148 case X86::VALIGNQZ128rri:
149 case X86::VALIGNQZ128rmi: {
150 assert((NewOpc == X86::VPALIGNRrri || NewOpc == X86::VPALIGNRrmi) &&
151 "Unexpected new opcode!");
152 unsigned Scale =
153 (Opc == X86::VALIGNQZ128rri || Opc == X86::VALIGNQZ128rmi) ? 8 : 4;
154 MachineOperand &Imm = MI.getOperand(i: MI.getNumExplicitOperands() - 1);
155 Imm.setImm(Imm.getImm() * Scale);
156 break;
157 }
158 case X86::VSHUFF32X4Z256rmi:
159 case X86::VSHUFF32X4Z256rri:
160 case X86::VSHUFF64X2Z256rmi:
161 case X86::VSHUFF64X2Z256rri:
162 case X86::VSHUFI32X4Z256rmi:
163 case X86::VSHUFI32X4Z256rri:
164 case X86::VSHUFI64X2Z256rmi:
165 case X86::VSHUFI64X2Z256rri: {
166 assert((NewOpc == X86::VPERM2F128rri || NewOpc == X86::VPERM2I128rri ||
167 NewOpc == X86::VPERM2F128rmi || NewOpc == X86::VPERM2I128rmi) &&
168 "Unexpected new opcode!");
169 MachineOperand &Imm = MI.getOperand(i: MI.getNumExplicitOperands() - 1);
170 int64_t ImmVal = Imm.getImm();
171 // Set bit 5, move bit 1 to bit 4, copy bit 0.
172 Imm.setImm(0x20 | ((ImmVal & 2) << 3) | (ImmVal & 1));
173 break;
174 }
175 case X86::VRNDSCALEPDZ128rri:
176 case X86::VRNDSCALEPDZ128rmi:
177 case X86::VRNDSCALEPSZ128rri:
178 case X86::VRNDSCALEPSZ128rmi:
179 case X86::VRNDSCALEPDZ256rri:
180 case X86::VRNDSCALEPDZ256rmi:
181 case X86::VRNDSCALEPSZ256rri:
182 case X86::VRNDSCALEPSZ256rmi:
183 case X86::VRNDSCALESDZrri:
184 case X86::VRNDSCALESDZrmi:
185 case X86::VRNDSCALESSZrri:
186 case X86::VRNDSCALESSZrmi:
187 case X86::VRNDSCALESDZrri_Int:
188 case X86::VRNDSCALESDZrmi_Int:
189 case X86::VRNDSCALESSZrri_Int:
190 case X86::VRNDSCALESSZrmi_Int:
191 const MachineOperand &Imm = MI.getOperand(i: MI.getNumExplicitOperands() - 1);
192 int64_t ImmVal = Imm.getImm();
193 // Ensure that only bits 3:0 of the immediate are used.
194 if ((ImmVal & 0xf) != ImmVal)
195 return false;
196 break;
197 }
198
199 return true;
200}
201
202static unsigned getMovMskBits(unsigned Opc) {
203 switch (Opc) {
204 case X86::VPMOVQ2MZ128kr:
205 case X86::VPCMPQZ128rri:
206 return 2;
207 case X86::VPMOVQ2MZ256kr:
208 case X86::VPMOVD2MZ128kr:
209 case X86::VPCMPQZ256rri:
210 case X86::VPCMPDZ128rri:
211 return 4;
212 case X86::VPMOVD2MZ256kr:
213 case X86::VPCMPDZ256rri:
214 return 8;
215 case X86::VPMOVB2MZ128kr:
216 case X86::VPCMPBZ128rri:
217 return 16;
218 case X86::VPMOVB2MZ256kr:
219 case X86::VPCMPBZ256rri:
220 return 32;
221 default:
222 llvm_unreachable("Unknown opcode");
223 }
224}
225
226static bool isKMovNarrowing(unsigned MaskBits, unsigned KMOVOpc) {
227 unsigned KMOVSize = 0;
228 switch (KMOVOpc) {
229 case X86::KMOVBrk:
230 KMOVSize = 8;
231 break;
232 case X86::KMOVWrk:
233 KMOVSize = 16;
234 break;
235 case X86::KMOVDrk:
236 KMOVSize = 32;
237 break;
238 default:
239 llvm_unreachable("Unknown KMOV opcode");
240 }
241
242 return KMOVSize < MaskBits;
243}
244
245static bool isZeroVector(const MachineInstr &MI) {
246 switch (MI.getOpcode()) {
247 case X86::VPXORrr:
248 case X86::VPXORYrr:
249 case X86::VXORPSrr:
250 case X86::VXORPSYrr:
251 return MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
252 default:
253 return false;
254 }
255}
256
257static bool isAllOnesVector(const MachineInstr &MI, bool Is256Bit) {
258 switch (MI.getOpcode()) {
259 case X86::VPCMPEQDrr:
260 return !Is256Bit && MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
261 case X86::VPCMPEQDYrr:
262 return MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
263 default:
264 return false;
265 }
266}
267
268static MachineInstr *getSignMaskConstantDef(MachineInstr &MI, Register Reg,
269 bool IsZero, bool Is256Bit,
270 const TargetRegisterInfo *TRI) {
271 for (MachineInstr &DefMI : llvm::reverse(C: llvm::make_range(
272 x: MI.getParent()->begin(), y: MachineBasicBlock::iterator(MI)))) {
273 if (!DefMI.modifiesRegister(Reg, TRI))
274 continue;
275 // Stop at the nearest def/clobber; an older matching constant may no
276 // longer be the reaching definition.
277 if (IsZero ? isZeroVector(MI: DefMI) : isAllOnesVector(MI: DefMI, Is256Bit))
278 return &DefMI;
279 break;
280 }
281 return nullptr;
282}
283
284static bool isCompressibleBlendVUse(unsigned BlendOpc, unsigned UseOpc) {
285 switch (BlendOpc) {
286 case X86::VBLENDVPSrrr:
287 switch (UseOpc) {
288 case X86::VMOVAPSZ128rrk:
289 case X86::VMOVUPSZ128rrk:
290 case X86::VMOVDQA32Z128rrk:
291 case X86::VMOVDQU32Z128rrk:
292 return true;
293 default:
294 return false;
295 }
296 case X86::VBLENDVPSYrrr:
297 switch (UseOpc) {
298 case X86::VMOVAPSZ256rrk:
299 case X86::VMOVUPSZ256rrk:
300 case X86::VMOVDQA32Z256rrk:
301 case X86::VMOVDQU32Z256rrk:
302 return true;
303 default:
304 return false;
305 }
306 case X86::VBLENDVPDrrr:
307 switch (UseOpc) {
308 case X86::VMOVAPDZ128rrk:
309 case X86::VMOVUPDZ128rrk:
310 case X86::VMOVDQA64Z128rrk:
311 case X86::VMOVDQU64Z128rrk:
312 return true;
313 default:
314 return false;
315 }
316 case X86::VBLENDVPDYrrr:
317 switch (UseOpc) {
318 case X86::VMOVAPDZ256rrk:
319 case X86::VMOVUPDZ256rrk:
320 case X86::VMOVDQA64Z256rrk:
321 case X86::VMOVDQU64Z256rrk:
322 return true;
323 default:
324 return false;
325 }
326 case X86::VPBLENDVBrrr:
327 return UseOpc == X86::VMOVDQU8Z128rrk;
328 case X86::VPBLENDVBYrrr:
329 return UseOpc == X86::VMOVDQU8Z256rrk;
330 default:
331 return false;
332 }
333}
334
335static bool isCompressibleMaskedBlendUse(unsigned BlendOpc, unsigned UseOpc) {
336 switch (BlendOpc) {
337 case X86::VBLENDVPSrrr:
338 return UseOpc == X86::VPBLENDMDZ128rrk || UseOpc == X86::VBLENDMPSZ128rrk;
339 case X86::VBLENDVPSYrrr:
340 return UseOpc == X86::VPBLENDMDZ256rrk || UseOpc == X86::VBLENDMPSZ256rrk;
341 case X86::VBLENDVPDrrr:
342 return UseOpc == X86::VPBLENDMQZ128rrk || UseOpc == X86::VBLENDMPDZ128rrk;
343 case X86::VBLENDVPDYrrr:
344 return UseOpc == X86::VPBLENDMQZ256rrk || UseOpc == X86::VBLENDMPDZ256rrk;
345 default:
346 return false;
347 }
348}
349
350// Try to compress mask producer chains:
351// vpmov*2m %xmm0, %k0 -> (erase this)
352// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
353//
354// vpcmpge* $0, %xmm0, %k0 -> (erase this) (X >= 0)
355// vpcmpgt* $-1, %xmm0, %k0 -> (erase this) (X > -1)
356// kmov* %k0, %eax -> vmovmskp* %xmm0, %eax
357// bounded complement of %eax
358//
359// vpmov*2m %xmm0, %k1 -> (erase this)
360// vmov* %xmm1, %xmm2 {%k1} -> vblendv* %xmm0, %xmm2, %xmm1, %xmm2
361static bool tryCompressMaskProducer(MachineInstr &MI, MachineBasicBlock &MBB,
362 const X86Subtarget &ST,
363 SmallVectorImpl<MachineInstr *> &ToErase) {
364 const X86InstrInfo *TII = ST.getInstrInfo();
365 const TargetRegisterInfo *TRI = ST.getRegisterInfo();
366 MachineRegisterInfo *MRI = &MBB.getParent()->getRegInfo();
367
368 unsigned Opc = MI.getOpcode();
369 bool IsSignMaskCmp = Opc == X86::VPCMPBZ128rri || Opc == X86::VPCMPBZ256rri ||
370 Opc == X86::VPCMPDZ128rri || Opc == X86::VPCMPDZ256rri ||
371 Opc == X86::VPCMPQZ128rri || Opc == X86::VPCMPQZ256rri;
372 if (!IsSignMaskCmp && Opc != X86::VPMOVD2MZ128kr &&
373 Opc != X86::VPMOVD2MZ256kr && Opc != X86::VPMOVQ2MZ128kr &&
374 Opc != X86::VPMOVQ2MZ256kr && Opc != X86::VPMOVB2MZ128kr &&
375 Opc != X86::VPMOVB2MZ256kr)
376 return false;
377
378 if (usesExtendedRegister(MI))
379 return false;
380
381 Register MaskReg = MI.getOperand(i: 0).getReg();
382 Register SrcVecReg = MI.getOperand(i: 1).getReg();
383 MachineInstr *ConstantDef = nullptr;
384 bool ConstantDefOnlyFeedsCmp = false;
385
386 if (IsSignMaskCmp) {
387 int64_t Pred = MI.getOperand(i: 3).getImm();
388 // VPCMP signed predicates: nlt (5) folds X >= 0, nle (6) folds X > -1.
389 if (Pred != 5 && Pred != 6)
390 return false;
391 Register ConstantReg = MI.getOperand(i: 2).getReg();
392 bool Is256Bit = Opc == X86::VPCMPBZ256rri || Opc == X86::VPCMPDZ256rri ||
393 Opc == X86::VPCMPQZ256rri;
394 // The sign-mask fold is valid only for compares against the reaching
395 // zero/all-ones vector definition.
396 ConstantDef =
397 getSignMaskConstantDef(MI, Reg: ConstantReg, IsZero: Pred == 5, Is256Bit, TRI);
398 if (!ConstantDef)
399 return false;
400 // If the constant feeds only this compare, erase it with the compare.
401 ConstantDefOnlyFeedsCmp = !TRI->regsOverlap(RegA: ConstantReg, RegB: SrcVecReg);
402 for (MachineInstr &UseMI :
403 llvm::make_range(x: std::next(x: MachineBasicBlock::iterator(*ConstantDef)),
404 y: MachineBasicBlock::iterator(MI)))
405 if (UseMI.readsRegister(Reg: ConstantReg, TRI)) {
406 ConstantDefOnlyFeedsCmp = false;
407 break;
408 }
409 }
410
411 unsigned MovMskOpc = 0;
412 unsigned BlendOpc = 0;
413 switch (Opc) {
414 case X86::VPCMPDZ128rri:
415 case X86::VPMOVD2MZ128kr:
416 MovMskOpc = X86::VMOVMSKPSrr;
417 BlendOpc = X86::VBLENDVPSrrr;
418 break;
419 case X86::VPCMPDZ256rri:
420 case X86::VPMOVD2MZ256kr:
421 MovMskOpc = X86::VMOVMSKPSYrr;
422 BlendOpc = X86::VBLENDVPSYrrr;
423 break;
424 case X86::VPCMPQZ128rri:
425 case X86::VPMOVQ2MZ128kr:
426 MovMskOpc = X86::VMOVMSKPDrr;
427 BlendOpc = X86::VBLENDVPDrrr;
428 break;
429 case X86::VPCMPQZ256rri:
430 case X86::VPMOVQ2MZ256kr:
431 MovMskOpc = X86::VMOVMSKPDYrr;
432 BlendOpc = X86::VBLENDVPDYrrr;
433 break;
434 case X86::VPCMPBZ128rri:
435 case X86::VPMOVB2MZ128kr:
436 MovMskOpc = X86::VPMOVMSKBrr;
437 BlendOpc = X86::VPBLENDVBrrr;
438 break;
439 case X86::VPCMPBZ256rri:
440 case X86::VPMOVB2MZ256kr:
441 MovMskOpc = X86::VPMOVMSKBYrr;
442 BlendOpc = X86::VPBLENDVBYrrr;
443 break;
444 default:
445 llvm_unreachable("Unknown VPMOV opcode");
446 }
447
448 MachineInstr *KMovMI = nullptr;
449 MachineInstr *BlendMI = nullptr;
450 bool BlendIsMaskedBlend = false;
451
452 for (MachineInstr &CurMI : llvm::make_range(
453 x: std::next(x: MachineBasicBlock::iterator(MI)), y: MBB.end())) {
454 if (CurMI.readsRegister(Reg: MaskReg, TRI)) {
455 if (KMovMI || BlendMI)
456 return false; // Fail: Mask has MULTIPLE uses
457
458 unsigned UseOpc = CurMI.getOpcode();
459 bool IsKMOV = UseOpc == X86::KMOVBrk || UseOpc == X86::KMOVWrk ||
460 UseOpc == X86::KMOVDrk;
461 // Only allow non-narrowing KMOV uses of the mask.
462 if (IsKMOV && CurMI.getOperand(i: 1).getReg() == MaskReg &&
463 !usesExtendedRegister(MI: CurMI) &&
464 !isKMovNarrowing(MaskBits: getMovMskBits(Opc), KMOVOpc: UseOpc)) {
465 KMovMI = &CurMI;
466 // continue scanning to ensure
467 // there are no *other* uses of the mask later in the block.
468 } else {
469 bool IsMaskedMove =
470 !IsSignMaskCmp && isCompressibleBlendVUse(BlendOpc, UseOpc);
471 bool IsMaskedBlend =
472 !IsSignMaskCmp && isCompressibleMaskedBlendUse(BlendOpc, UseOpc);
473
474 if (!IsMaskedMove && !IsMaskedBlend)
475 return false;
476
477 unsigned MaskOpIdx = IsMaskedBlend ? 1 : 2;
478 if (CurMI.getOperand(i: MaskOpIdx).getReg() == MaskReg &&
479 !usesExtendedRegister(MI: CurMI) && checkPredicate(Opc: BlendOpc, Subtarget: &ST)) {
480 BlendMI = &CurMI;
481 BlendIsMaskedBlend = IsMaskedBlend;
482 } else {
483 return false;
484 }
485 }
486 }
487
488 if (CurMI.modifiesRegister(Reg: MaskReg, TRI)) {
489 if (!KMovMI && !BlendMI)
490 return false; // Mask clobbered before use
491 break;
492 }
493
494 if (!KMovMI && !BlendMI && CurMI.modifiesRegister(Reg: SrcVecReg, TRI)) {
495 return false; // SrcVecReg modified before it could be reused
496 }
497 }
498
499 if (!KMovMI && !BlendMI)
500 return false;
501
502 unsigned MovMskBits = getMovMskBits(Opc);
503 // Bounded complements define EFLAGS, unlike VPCMP + KMOV. A 32-bit
504 // complement uses NOT, which does not modify EFLAGS.
505 if (IsSignMaskCmp && KMovMI) {
506 if (KMovMI->getOperand(i: 0).isDead() ||
507 (MovMskBits != 32 &&
508 MBB.computeRegisterLiveness(
509 TRI, Reg: X86::EFLAGS,
510 Before: std::next(x: MachineBasicBlock::const_iterator(*KMovMI)),
511 Neighborhood: MBB.size()) != MachineBasicBlock::LQR_Dead))
512 return false;
513 }
514
515 // Check if MaskReg is used in any other basic blocks
516 for (const MachineInstr &UseMI : MRI->use_instructions(Reg: MaskReg))
517 if (UseMI.getParent() != &MBB)
518 return false;
519
520 // Apply the transformation
521 MachineInstr *NewMI = nullptr;
522 if (KMovMI) {
523 MachineOperand OldDst = KMovMI->getOperand(i: 0);
524 KMovMI->setDesc(TII->get(Opcode: MovMskOpc));
525 MachineOperand &NewSrc = KMovMI->getOperand(i: 1);
526 NewSrc.setReg(SrcVecReg);
527 // setReg() keeps the mask operand's kill flag; take the source's kill
528 // state from the VPMOV instead.
529 NewSrc.setIsKill(MI.getOperand(i: 1).isKill());
530 NewMI = KMovMI;
531 if (IsSignMaskCmp) {
532 Register DstReg = OldDst.getReg();
533 int64_t ComplementMask =
534 APInt::getLowBitsSet(numBits: 32, loBitsSet: MovMskBits).getSExtValue();
535 unsigned ComplementOpc =
536 MovMskBits == 32
537 ? X86::NOT32r
538 : (isInt<8>(x: ComplementMask) ? X86::XOR32ri8 : X86::XOR32ri);
539 auto MIB = BuildMI(BB&: MBB, I: std::next(x: MachineBasicBlock::iterator(*KMovMI)),
540 MIMD: KMovMI->getDebugLoc(), MCID: TII->get(Opcode: ComplementOpc), DestReg: DstReg)
541 .addReg(RegNo: DstReg, Flags: RegState::Kill);
542 if (MovMskBits != 32) {
543 MIB.addImm(Val: ComplementMask);
544 MIB->findRegisterDefOperand(Reg: X86::EFLAGS, TRI)->setIsDead();
545 }
546 MIB->getOperand(i: 0).setIsRenamable(OldDst.isRenamable());
547 }
548 } else if (BlendMI) {
549 const MachineOperand &MaskVec = MI.getOperand(i: 1);
550 const MachineOperand &Dst = BlendMI->getOperand(i: 0);
551 const MachineOperand &Passthru =
552 BlendMI->getOperand(i: BlendIsMaskedBlend ? 2 : 1);
553 const MachineOperand &Src = BlendMI->getOperand(i: 3);
554
555 // Build a replacement instead of changing BlendMI in place because
556 // masked VMOV and VPBLENDM have different operand layouts from VBLENDV.
557 auto MIB =
558 BuildMI(BB&: MBB, I&: *BlendMI, MIMD: BlendMI->getDebugLoc(), MCID: TII->get(Opcode: BlendOpc))
559 .addReg(RegNo: Dst.getReg(), Flags: getRegState(RegOp: Dst))
560 .addReg(RegNo: Passthru.getReg(), Flags: getRegState(RegOp: Passthru))
561 .addReg(RegNo: Src.getReg(), Flags: getRegState(RegOp: Src))
562 .addReg(RegNo: MaskVec.getReg(), Flags: getRegState(RegOp: MaskVec));
563 NewMI = MIB;
564 ToErase.push_back(Elt: BlendMI);
565 }
566 assert(NewMI && "Expected a compressed instruction");
567 NewMI->setAsmPrinterFlag(X86::AC_EVEX_2_VEX);
568 ToErase.push_back(Elt: &MI);
569 if (ConstantDefOnlyFeedsCmp && MI.getOperand(i: 2).isKill())
570 ToErase.push_back(Elt: ConstantDef);
571 return true;
572}
573
574static bool CompressEVEXImpl(MachineInstr &MI, MachineBasicBlock &MBB,
575 const X86Subtarget &ST,
576 SmallVectorImpl<MachineInstr *> &ToErase) {
577 uint64_t TSFlags = MI.getDesc().TSFlags;
578
579 // Check for EVEX instructions only.
580 if ((TSFlags & X86II::EncodingMask) != X86II::EVEX)
581 return false;
582
583 // Instructions with mask or 512-bit vector can't be converted to VEX.
584 if (TSFlags & (X86II::EVEX_K | X86II::EVEX_L2))
585 return false;
586
587 // Keep the EVEX encoding when there's 1-byte compressed disp8*N.
588 if (hasShorterEVEXViaCDisp8(MI))
589 return false;
590
591 // Specialized mask-producing folds to MOVMSK/VBLENDV first.
592 if (tryCompressMaskProducer(MI, MBB, ST, ToErase))
593 return true;
594
595 auto IsRedundantNewDataDest = [&](unsigned &Opc) {
596 // $rbx = ADD64rr_ND $rbx, $rax / $rbx = ADD64rr_ND $rax, $rbx
597 // ->
598 // $rbx = ADD64rr $rbx, $rax
599 const MCInstrDesc &Desc = MI.getDesc();
600 Register Reg0 = MI.getOperand(i: 0).getReg();
601 const MachineOperand &Op1 = MI.getOperand(i: 1);
602 if (!Op1.isReg() || X86::getFirstAddrOperandIdx(MI) == 1 ||
603 X86::isCFCMOVCC(Opcode: MI.getOpcode()))
604 return false;
605 Register Reg1 = Op1.getReg();
606 if (Reg1 == Reg0)
607 return true;
608
609 // Op1 and Op2 may be commutable for ND instructions.
610 if (!Desc.isCommutable() || Desc.getNumOperands() < 3 ||
611 !MI.getOperand(i: 2).isReg() || MI.getOperand(i: 2).getReg() != Reg0)
612 return false;
613 // Opcode may change after commute, e.g. SHRD -> SHLD
614 ST.getInstrInfo()->commuteInstruction(MI, NewMI: false, OpIdx1: 1, OpIdx2: 2);
615 Opc = MI.getOpcode();
616 return true;
617 };
618
619 // EVEX_B has several meanings.
620 // AVX512:
621 // register form: rounding control or SAE
622 // memory form: broadcast
623 //
624 // APX:
625 // MAP4: NDD, ZU
626 //
627 // For AVX512 cases, EVEX prefix is needed in order to carry this information
628 // thus preventing the transformation to VEX encoding.
629 bool IsND = X86II::hasNewDataDest(TSFlags);
630 unsigned Opc = MI.getOpcode();
631 bool IsSetZUCCm = Opc == X86::SETZUCCm;
632 if (TSFlags & X86II::EVEX_B && !IsND && !IsSetZUCCm)
633 return false;
634 // MOVBE*rr is special because it has semantic of NDD but not set EVEX_B.
635 bool IsNDLike = IsND || Opc == X86::MOVBE32rr || Opc == X86::MOVBE64rr;
636 bool IsRedundantNDD = IsNDLike ? IsRedundantNewDataDest(Opc) : false;
637
638 auto GetCompressedOpc = [&](unsigned Opc) -> unsigned {
639 ArrayRef<X86TableEntry> Table = ArrayRef(X86CompressEVEXTable);
640 const auto I = llvm::lower_bound(Range&: Table, Value&: Opc);
641 if (I == Table.end() || I->OldOpc != Opc)
642 return 0;
643
644 if (usesExtendedRegister(MI) || !checkPredicate(Opc: I->NewOpc, Subtarget: &ST) ||
645 !performCustomAdjustments(MI, NewOpc: I->NewOpc))
646 return 0;
647 return I->NewOpc;
648 };
649
650 Register Dst = MI.getOperand(i: 0).getReg();
651 if (IsRedundantNDD) {
652 // Redundant NDD ops cannot be safely compressed if either:
653 // - the legacy op would introduce a partial write that BreakFalseDeps
654 // identified as a potential stall, or
655 // - the op is writing to a subregister of a live register, i.e. the
656 // full (zeroed) result is used.
657 // Both cases are indicated by an implicit def of the superregister.
658 if (Dst &&
659 (X86::GR16RegClass.contains(Reg: Dst) || X86::GR8RegClass.contains(Reg: Dst))) {
660 Register Super = getX86SubSuperRegister(Reg: Dst, Size: 64);
661 if (MI.definesRegister(Reg: Super, /*TRI=*/nullptr))
662 IsRedundantNDD = false;
663 }
664
665 // ADDrm/mr instructions with NDD + relocation had been transformed to the
666 // instructions without NDD in X86SuppressAPXForRelocation pass. That is to
667 // keep backward compatibility with linkers without APX support.
668 if (!ST.getCLOpts().enable_apx_for_relocation)
669 assert(!isAddMemInstrWithRelocation(MI) &&
670 "Unexpected NDD instruction with relocation!");
671 } else if (Opc == X86::ADD32ri_ND || Opc == X86::ADD64ri32_ND ||
672 Opc == X86::ADD32rr_ND || Opc == X86::ADD64rr_ND) {
673 // Non-redundant NDD ADD can be compressed to LEA when:
674 // - No EGPR register used and
675 // - EFLAGS is dead.
676 if (!usesExtendedRegister(MI) &&
677 MI.registerDefIsDead(Reg: X86::EFLAGS, /*TRI=*/nullptr)) {
678 Register Src1 = MI.getOperand(i: 1).getReg();
679 const MachineOperand &Src2 = MI.getOperand(i: 2);
680 bool Is32BitReg = Opc == X86::ADD32ri_ND || Opc == X86::ADD32rr_ND;
681 const MCInstrDesc &NewDesc =
682 ST.getInstrInfo()->get(Opcode: Is32BitReg ? X86::LEA64_32r : X86::LEA64r);
683 if (Is32BitReg)
684 Src1 = getX86SubSuperRegister(Reg: Src1, Size: 64);
685 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: NewDesc, DestReg: Dst)
686 .addReg(RegNo: Src1)
687 .addImm(Val: 1);
688 if (Opc == X86::ADD32ri_ND || Opc == X86::ADD64ri32_ND)
689 MIB.addReg(RegNo: 0).add(MO: Src2);
690 else if (Is32BitReg)
691 MIB.addReg(RegNo: getX86SubSuperRegister(Reg: Src2.getReg(), Size: 64)).addImm(Val: 0);
692 else
693 MIB.add(MO: Src2).addImm(Val: 0);
694 MIB.addReg(RegNo: 0);
695 MI.removeFromParent();
696 return true;
697 }
698 }
699
700 // NonNF -> NF only if it's not a compressible NDD instruction and eflags is
701 // dead.
702 unsigned NewOpc = IsRedundantNDD
703 ? X86::getNonNDVariant(Opc)
704 : ((IsNDLike && ST.hasNF() &&
705 MI.registerDefIsDead(Reg: X86::EFLAGS, /*TRI=*/nullptr))
706 ? X86::getNFVariant(Opc)
707 : GetCompressedOpc(Opc));
708
709 if (!NewOpc)
710 return false;
711 // NF (No Flags) instructions cannot compress to VEX/legacy encoding.
712 // NF_ND can still compress to NF (both remain EVEX).
713 assert((IsND || !(TSFlags & X86II::EVEX_NF)) &&
714 "Unexpected to compress NF instructions without ND.");
715
716 const MCInstrDesc &NewDesc = ST.getInstrInfo()->get(Opcode: NewOpc);
717 MI.setDesc(NewDesc);
718 unsigned AsmComment;
719 switch (NewDesc.TSFlags & X86II::EncodingMask) {
720 case X86II::LEGACY:
721 AsmComment = X86::AC_EVEX_2_LEGACY;
722 break;
723 case X86II::VEX:
724 AsmComment = X86::AC_EVEX_2_VEX;
725 break;
726 case X86II::EVEX:
727 AsmComment = X86::AC_EVEX_2_EVEX;
728 assert(IsND && (NewDesc.TSFlags & X86II::EVEX_NF) &&
729 "Unknown EVEX2EVEX compression");
730 break;
731 default:
732 llvm_unreachable("Unknown EVEX compression");
733 }
734 MI.setAsmPrinterFlag(AsmComment);
735 if (IsRedundantNDD)
736 MI.tieOperands(DefIdx: 0, UseIdx: 1);
737
738 return true;
739}
740
741static bool runOnMF(MachineFunction &MF) {
742 LLVM_DEBUG(dbgs() << "Start X86CompressEVEXPass\n";);
743#ifndef NDEBUG
744 // Make sure the tables are sorted.
745 static std::atomic<bool> TableChecked(false);
746 if (!TableChecked.load(std::memory_order_relaxed)) {
747 assert(llvm::is_sorted(X86CompressEVEXTable) &&
748 "X86CompressEVEXTable is not sorted!");
749 TableChecked.store(true, std::memory_order_relaxed);
750 }
751#endif
752 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
753 if (!ST.hasAVX512() && !ST.hasEGPR() && !ST.hasNDD() && !ST.hasZU())
754 return false;
755
756 bool Changed = false;
757
758 for (MachineBasicBlock &MBB : MF) {
759 SmallVector<MachineInstr *, 4> ToErase;
760
761 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
762 Changed |= CompressEVEXImpl(MI, MBB, ST, ToErase);
763 }
764
765 for (MachineInstr *MI : ToErase) {
766 MI->eraseFromParent();
767 }
768 }
769 LLVM_DEBUG(dbgs() << "End X86CompressEVEXPass\n";);
770 return Changed;
771}
772
773INITIALIZE_PASS(CompressEVEXLegacy, COMP_EVEX_NAME, COMP_EVEX_DESC, false,
774 false)
775
776FunctionPass *llvm::createX86CompressEVEXLegacyPass() {
777 return new CompressEVEXLegacy();
778}
779
780bool CompressEVEXLegacy::runOnMachineFunction(MachineFunction &MF) {
781 return runOnMF(MF);
782}
783
784PreservedAnalyses
785X86CompressEVEXPass::run(MachineFunction &MF,
786 MachineFunctionAnalysisManager &MFAM) {
787 bool Changed = runOnMF(MF);
788 if (!Changed)
789 return PreservedAnalyses::all();
790 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
791 PA.preserveSet<CFGAnalyses>();
792 return PA;
793}
794