1//===-- X86FixupInstTuning.cpp - replace instructions -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file does a tuning pass replacing slower machine instructions
10// with faster ones. We do this here, as opposed to during normal ISel, as
11// attempting to get the "right" instruction can break patterns. This pass
12// is not meant search for special cases where an instruction can be transformed
13// to another, it is only meant to do transformations where the old instruction
14// is always replacable with the new instructions. For example:
15//
16// `vpermq ymm` -> `vshufd ymm`
17// -- BAD, not always valid (lane cross/non-repeated mask)
18//
19// `vpermilps ymm` -> `vshufd ymm`
20// -- GOOD, always replaceable
21//
22//===----------------------------------------------------------------------===//
23
24#include "X86.h"
25#include "X86InstrInfo.h"
26#include "X86RegisterInfo.h"
27#include "X86Subtarget.h"
28#include "llvm/ADT/Statistic.h"
29#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
30#include "llvm/CodeGen/MachineFunctionPass.h"
31#include "llvm/CodeGen/MachineInstrBuilder.h"
32#include "llvm/CodeGen/MachinePassManager.h"
33#include "llvm/IR/Analysis.h"
34
35using namespace llvm;
36
37#define DEBUG_TYPE "x86-fixup-inst-tuning"
38
39STATISTIC(NumInstChanges, "Number of instructions changes");
40
41namespace {
42class X86FixupInstTuningImpl {
43public:
44 bool runOnMachineFunction(MachineFunction &MF);
45
46private:
47 bool processInstruction(MachineFunction &MF, MachineBasicBlock &MBB,
48 MachineBasicBlock::iterator &I);
49
50 const X86InstrInfo *TII = nullptr;
51 const X86Subtarget *ST = nullptr;
52 const MCSchedModel *SM = nullptr;
53 const X86RegisterInfo *TRI = nullptr;
54};
55
56class X86FixupInstTuningLegacy : public MachineFunctionPass {
57public:
58 static char ID;
59
60 X86FixupInstTuningLegacy() : MachineFunctionPass(ID) {}
61
62 StringRef getPassName() const override { return "X86 Fixup Inst Tuning"; }
63
64 bool runOnMachineFunction(MachineFunction &MF) override;
65
66 // This pass runs after regalloc and doesn't support VReg operands.
67 MachineFunctionProperties getRequiredProperties() const override {
68 return MachineFunctionProperties().setNoVRegs();
69 }
70};
71} // end anonymous namespace
72
73char X86FixupInstTuningLegacy ::ID = 0;
74
75INITIALIZE_PASS(X86FixupInstTuningLegacy, DEBUG_TYPE, DEBUG_TYPE, false, false)
76
77FunctionPass *llvm::createX86FixupInstTuningLegacyPass() {
78 return new X86FixupInstTuningLegacy();
79}
80
81template <typename T>
82static std::optional<bool> CmpOptionals(T NewVal, T CurVal) {
83 if (NewVal.has_value() && CurVal.has_value() && *NewVal != *CurVal)
84 return *NewVal < *CurVal;
85
86 return std::nullopt;
87}
88
89bool X86FixupInstTuningImpl::processInstruction(
90 MachineFunction &MF, MachineBasicBlock &MBB,
91 MachineBasicBlock::iterator &I) {
92 MachineInstr &MI = *I;
93 unsigned Opc = MI.getOpcode();
94 unsigned NumOperands = MI.getDesc().getNumOperands();
95 bool OptSize = MF.getFunction().hasOptSize();
96
97 auto GetInstTput = [&](unsigned Opcode) -> std::optional<double> {
98 // We already checked that SchedModel exists in `NewOpcPreferable`.
99 return MCSchedModel::getReciprocalThroughput(
100 STI: *ST, SCDesc: *(SM->getSchedClassDesc(SchedClassIdx: TII->get(Opcode).getSchedClass())));
101 };
102
103 auto GetInstLat = [&](unsigned Opcode) -> std::optional<double> {
104 // We already checked that SchedModel exists in `NewOpcPreferable`.
105 return MCSchedModel::computeInstrLatency(
106 STI: *ST, SCDesc: *(SM->getSchedClassDesc(SchedClassIdx: TII->get(Opcode).getSchedClass())));
107 };
108
109 auto GetInstSize = [&](unsigned Opcode) -> std::optional<unsigned> {
110 if (unsigned Size = TII->get(Opcode).getSize())
111 return Size;
112 // Zero size means we where unable to compute it.
113 return std::nullopt;
114 };
115
116 auto NewOpcPreferable = [&](unsigned NewOpc,
117 bool ReplaceInTie = true) -> bool {
118 std::optional<bool> Res;
119 if (SM->hasInstrSchedModel()) {
120 // Compare tput -> lat -> code size.
121 Res = CmpOptionals(NewVal: GetInstTput(NewOpc), CurVal: GetInstTput(Opc));
122 if (Res.has_value())
123 return *Res;
124
125 Res = CmpOptionals(NewVal: GetInstLat(NewOpc), CurVal: GetInstLat(Opc));
126 if (Res.has_value())
127 return *Res;
128 }
129
130 Res = CmpOptionals(NewVal: GetInstSize(Opc), CurVal: GetInstSize(NewOpc));
131 if (Res.has_value())
132 return *Res;
133
134 // We either have either were unable to get tput/lat/codesize or all values
135 // were equal. Return specified option for a tie.
136 return ReplaceInTie;
137 };
138
139 // `vpermilpd r, i` -> `vshufpd r, r, i`
140 // `vpermilpd r, i, k` -> `vshufpd r, r, i, k`
141 // `vshufpd` is always as fast or faster than `vpermilpd` and takes
142 // 1 less byte of code size for VEX and EVEX encoding.
143 auto ProcessVPERMILPDri = [&](unsigned NewOpc) -> bool {
144 if (!NewOpcPreferable(NewOpc))
145 return false;
146 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
147 {
148 unsigned MaskImm = MI.getOperand(i: NumOperands - 1).getImm();
149 MI.removeOperand(OpNo: NumOperands - 1);
150 MI.addOperand(Op: MI.getOperand(i: NumOperands - 2));
151 MI.setDesc(TII->get(Opcode: NewOpc));
152 MI.addOperand(Op: MachineOperand::CreateImm(Val: MaskImm));
153 }
154 LLVM_DEBUG(dbgs() << " With: " << MI);
155 return true;
156 };
157
158 // `vpermilps r, i` -> `vshufps r, r, i`
159 // `vpermilps r, i, k` -> `vshufps r, r, i, k`
160 // `vshufps` is always as fast or faster than `vpermilps` and takes
161 // 1 less byte of code size for VEX and EVEX encoding.
162 auto ProcessVPERMILPSri = [&](unsigned NewOpc) -> bool {
163 if (!NewOpcPreferable(NewOpc))
164 return false;
165 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
166 {
167 unsigned MaskImm = MI.getOperand(i: NumOperands - 1).getImm();
168 MI.removeOperand(OpNo: NumOperands - 1);
169 MI.addOperand(Op: MI.getOperand(i: NumOperands - 2));
170 MI.setDesc(TII->get(Opcode: NewOpc));
171 MI.addOperand(Op: MachineOperand::CreateImm(Val: MaskImm));
172 }
173 LLVM_DEBUG(dbgs() << " With: " << MI);
174 return true;
175 };
176
177 // `vpermilps m, i` -> `vpshufd m, i` iff no domain delay penalty on shuffles.
178 // `vpshufd` is always as fast or faster than `vpermilps` and takes 1 less
179 // byte of code size.
180 auto ProcessVPERMILPSmi = [&](unsigned NewOpc) -> bool {
181 // TODO: Might be work adding bypass delay if -Os/-Oz is enabled as
182 // `vpshufd` saves a byte of code size.
183 if (!ST->hasNoDomainDelayShuffle() ||
184 !NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
185 return false;
186 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
187 {
188 MI.setDesc(TII->get(Opcode: NewOpc));
189 }
190 LLVM_DEBUG(dbgs() << " With: " << MI);
191 return true;
192 };
193
194 // `vunpcklpd/vmovlhps r, r` -> `vunpcklqdq r, r`/`vshufpd r, r, 0x00`
195 // `vunpckhpd/vmovlhps r, r` -> `vunpckhqdq r, r`/`vshufpd r, r, 0xff`
196 // `vunpcklpd r, r, k` -> `vunpcklqdq r, r, k`/`vshufpd r, r, k, 0x00`
197 // `vunpckhpd r, r, k` -> `vunpckhqdq r, r, k`/`vshufpd r, r, k, 0xff`
198 // `vunpcklpd r, m` -> `vunpcklqdq r, m, k`
199 // `vunpckhpd r, m` -> `vunpckhqdq r, m, k`
200 // `vunpcklpd r, m, k` -> `vunpcklqdq r, m, k`
201 // `vunpckhpd r, m, k` -> `vunpckhqdq r, m, k`
202 // 1) If no bypass delay and `vunpck{l|h}qdq` faster than `vunpck{l|h}pd`
203 // -> `vunpck{l|h}qdq`
204 // 2) If `vshufpd` faster than `vunpck{l|h}pd`
205 // -> `vshufpd`
206 //
207 // `vunpcklps` -> `vunpckldq` (for all operand types if no bypass delay)
208 auto ProcessUNPCK = [&](unsigned NewOpc, unsigned MaskImm) -> bool {
209 if (!NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
210 return false;
211 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
212 {
213 MI.setDesc(TII->get(Opcode: NewOpc));
214 MI.addOperand(Op: MachineOperand::CreateImm(Val: MaskImm));
215 }
216 LLVM_DEBUG(dbgs() << " With: " << MI);
217 return true;
218 };
219
220 auto ProcessUNPCKToIntDomain = [&](unsigned NewOpc) -> bool {
221 // TODO it may be worth it to set ReplaceInTie to `true` as there is no real
222 // downside to the integer unpck, but if someone doesn't specify exact
223 // target we won't find it faster.
224 if (!ST->hasNoDomainDelayShuffle() ||
225 !NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
226 return false;
227 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
228 {
229 MI.setDesc(TII->get(Opcode: NewOpc));
230 }
231 LLVM_DEBUG(dbgs() << " With: " << MI);
232 return true;
233 };
234
235 auto ProcessUNPCKLPDrr = [&](unsigned NewOpcIntDomain,
236 unsigned NewOpc) -> bool {
237 if (ProcessUNPCKToIntDomain(NewOpcIntDomain))
238 return true;
239 return ProcessUNPCK(NewOpc, 0x00);
240 };
241 auto ProcessUNPCKHPDrr = [&](unsigned NewOpcIntDomain,
242 unsigned NewOpc) -> bool {
243 if (ProcessUNPCKToIntDomain(NewOpcIntDomain))
244 return true;
245 return ProcessUNPCK(NewOpc, 0xff);
246 };
247
248 auto ProcessUNPCKPDrm = [&](unsigned NewOpcIntDomain) -> bool {
249 return ProcessUNPCKToIntDomain(NewOpcIntDomain);
250 };
251
252 auto ProcessUNPCKPS = [&](unsigned NewOpc) -> bool {
253 return ProcessUNPCKToIntDomain(NewOpc);
254 };
255
256 // MOVUPS/MOVAPS takes 1 less byte of code size. Only replace when
257 // there is no move domain-delay penalty on the target, or -Oz is set.
258 auto ProcessMOVPDToMOVPS = [&](unsigned NewOpc) -> bool {
259 assert(NewOpcPreferable(NewOpc) &&
260 "MOVUPS/MOVAPS should be preferred over MOVUPD/MOVAPD");
261 if (!ST->hasNoDomainDelayMov() && !MF.getFunction().hasMinSize())
262 return false;
263 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
264 MI.setDesc(TII->get(Opcode: NewOpc));
265 LLVM_DEBUG(dbgs() << " With: " << MI);
266 return true;
267 };
268
269 // `extractps $0, x, r32` -> `movd x, r32`
270 // `extractps $0, x, m32` -> `movss x, m32`
271 // Extracting element 0 only needs the low 32 bits, and MOVD/MOVSS are
272 // smaller and usually faster than EXTRACTPS.
273 auto ProcessEXTRACTPS = [&](unsigned NewOpc) -> bool {
274 if (MI.getOperand(i: NumOperands - 1).getImm() != 0 ||
275 !NewOpcPreferable(NewOpc))
276 return false;
277 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
278 {
279 MI.removeOperand(OpNo: NumOperands - 1);
280 MI.setDesc(TII->get(Opcode: NewOpc));
281 }
282 LLVM_DEBUG(dbgs() << " With: " << MI);
283 return true;
284 };
285
286 // If we're permuting the lower halves of the 256-bit registers, use a
287 // subvector insertion instead.
288 auto ProcessVPERM2x128ToVINSERT128 = [&](unsigned InsertOpc) -> bool {
289 unsigned PermImm = MI.getOperand(i: NumOperands - 1).getImm();
290 // TODO: Handle 0x00/0x02/0x22 when we have test coverage.
291 if (PermImm != 0x20 || !NewOpcPreferable(InsertOpc))
292 return false;
293 Register RHSRegYMM = MI.getOperand(i: NumOperands - 2).getReg();
294 Register RHSRegXMM = TRI->getSubReg(Reg: RHSRegYMM, Idx: X86::sub_xmm);
295 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
296 {
297 MI.setDesc(TII->get(Opcode: InsertOpc));
298 MI.removeOperand(OpNo: NumOperands - 1);
299 MI.removeOperand(OpNo: NumOperands - 2);
300 // Add the XMM subregister operand.
301 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RHSRegXMM, /*isDef=*/false,
302 /*isImp=*/false,
303 /*isKill=*/false));
304 // Add the immediate (1 = insert into high 128-bits).
305 MI.addOperand(Op: MachineOperand::CreateImm(Val: 1));
306 }
307 LLVM_DEBUG(dbgs() << " With: " << MI);
308 return true;
309 };
310
311 auto ProcessBLENDWToBLENDD = [&](unsigned MovOpc, unsigned NumElts) -> bool {
312 if (!ST->hasAVX2() || !NewOpcPreferable(MovOpc))
313 return false;
314 // Convert to VPBLENDD if scaling the VPBLENDW mask down/up loses no bits.
315 APInt MaskW =
316 APInt(8, MI.getOperand(i: NumOperands - 1).getImm(), /*IsSigned=*/false,
317 /*implicitTrunc=*/true);
318 APInt MaskD = APIntOps::ScaleBitMask(A: MaskW, NewBitWidth: 4, /*MatchAllBits=*/true);
319 if (MaskW != APIntOps::ScaleBitMask(A: MaskD, NewBitWidth: 8, /*MatchAllBits=*/true))
320 return false;
321 APInt NewMaskD = APInt::getSplat(NewLen: NumElts, V: MaskD);
322 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
323 {
324 MI.setDesc(TII->get(Opcode: MovOpc));
325 MI.removeOperand(OpNo: NumOperands - 1);
326 MI.addOperand(Op: MachineOperand::CreateImm(Val: NewMaskD.getZExtValue()));
327 }
328 LLVM_DEBUG(dbgs() << " With: " << MI);
329 return true;
330 };
331
332 auto ProcessBLENDToMOV = [&](unsigned MovOpc, unsigned Mask,
333 unsigned MovImm) -> bool {
334 if ((MI.getOperand(i: NumOperands - 1).getImm() & Mask) != MovImm)
335 return false;
336 if (!OptSize && !NewOpcPreferable(MovOpc))
337 return false;
338 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
339 {
340 MI.setDesc(TII->get(Opcode: MovOpc));
341 MI.removeOperand(OpNo: NumOperands - 1);
342 }
343 LLVM_DEBUG(dbgs() << " With: " << MI);
344 return true;
345 };
346
347 // Is ADD(X,X) more efficient than SHL(X,1)?
348 auto ProcessShiftLeftToAdd = [&](unsigned AddOpc) -> bool {
349 if (MI.getOperand(i: NumOperands - 1).getImm() != 1)
350 return false;
351 if (!NewOpcPreferable(AddOpc, /*ReplaceInTie*/ true))
352 return false;
353 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
354 {
355 MI.setDesc(TII->get(Opcode: AddOpc));
356 MI.removeOperand(OpNo: NumOperands - 1);
357 MI.addOperand(Op: MI.getOperand(i: NumOperands - 2));
358 }
359 LLVM_DEBUG(dbgs() << " With: " << MI);
360 return true;
361 };
362
363 // `vpermq ymm, ymm, 0x44` -> `vinserti128 ymm, ymm, xmm, 1`
364 // `vpermpd ymm, ymm, 0x44` -> `vinsertf128 ymm, ymm, xmm, 1`
365 // When the immediate is 0x44, VPERMQ/VPERMPD duplicates the lower 128-bit
366 // lane to both lanes. 0x44 = 0b01_00_01_00 means qwords[3:0] = {src[1],
367 // src[0], src[1], src[0]} This is equivalent to inserting the lower 128-bits
368 // into the upper 128-bit position.
369 auto ProcessVPERMQToVINSERT128 = [&](unsigned NewOpc) -> bool {
370 if (MI.getOperand(i: NumOperands - 1).getImm() != 0x44)
371 return false;
372 if (!NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
373 return false;
374
375 // Get the XMM subregister of the source YMM register.
376 Register SrcReg = MI.getOperand(i: 1).getReg();
377 Register XmmReg = TRI->getSubReg(Reg: SrcReg, Idx: X86::sub_xmm);
378
379 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
380 {
381 // Transform: VPERMQ $dst, $src, $0x44
382 // Into: VINSERTI128 $dst, $src, $xmm_src, $1
383 MI.setDesc(TII->get(Opcode: NewOpc));
384 // Remove the immediate operand.
385 MI.removeOperand(OpNo: NumOperands - 1);
386 // Add the XMM subregister operand.
387 MI.addOperand(Op: MachineOperand::CreateReg(Reg: XmmReg, /*isDef=*/false,
388 /*isImp=*/false,
389 /*isKill=*/false));
390 // Add the immediate (1 = insert into high 128-bits).
391 MI.addOperand(Op: MachineOperand::CreateImm(Val: 1));
392 }
393 LLVM_DEBUG(dbgs() << " With: " << MI);
394 return true;
395 };
396
397 switch (Opc) {
398 case X86::BLENDPDrri:
399 return ProcessBLENDToMOV(X86::MOVSDrr, 0x3, 0x1);
400 case X86::VBLENDPDrri:
401 return ProcessBLENDToMOV(X86::VMOVSDrr, 0x3, 0x1);
402
403 case X86::BLENDPSrri:
404 return ProcessBLENDToMOV(X86::MOVSSrr, 0xF, 0x1) ||
405 ProcessBLENDToMOV(X86::MOVSDrr, 0xF, 0x3);
406 case X86::VBLENDPSrri:
407 return ProcessBLENDToMOV(X86::VMOVSSrr, 0xF, 0x1) ||
408 ProcessBLENDToMOV(X86::VMOVSDrr, 0xF, 0x3);
409
410 case X86::VPBLENDWrri:
411 // TODO: Add X86::VPBLENDWrmi handling
412 // TODO: Add X86::VPBLENDWYrri handling
413 // TODO: Add X86::VPBLENDWYrmi handling
414 return ProcessBLENDWToBLENDD(X86::VPBLENDDrri, 4);
415
416 case X86::VPERM2F128rri:
417 return ProcessVPERM2x128ToVINSERT128(X86::VINSERTF128rri);
418 case X86::VPERM2I128rri:
419 return ProcessVPERM2x128ToVINSERT128(X86::VINSERTI128rri);
420
421 case X86::VPERMILPDri:
422 return ProcessVPERMILPDri(X86::VSHUFPDrri);
423 case X86::VPERMILPDYri:
424 return ProcessVPERMILPDri(X86::VSHUFPDYrri);
425 case X86::VPERMILPDZ128ri:
426 return ProcessVPERMILPDri(X86::VSHUFPDZ128rri);
427 case X86::VPERMILPDZ256ri:
428 return ProcessVPERMILPDri(X86::VSHUFPDZ256rri);
429 case X86::VPERMILPDZri:
430 return ProcessVPERMILPDri(X86::VSHUFPDZrri);
431 case X86::VPERMILPDZ128rikz:
432 return ProcessVPERMILPDri(X86::VSHUFPDZ128rrikz);
433 case X86::VPERMILPDZ256rikz:
434 return ProcessVPERMILPDri(X86::VSHUFPDZ256rrikz);
435 case X86::VPERMILPDZrikz:
436 return ProcessVPERMILPDri(X86::VSHUFPDZrrikz);
437 case X86::VPERMILPDZ128rik:
438 return ProcessVPERMILPDri(X86::VSHUFPDZ128rrik);
439 case X86::VPERMILPDZ256rik:
440 return ProcessVPERMILPDri(X86::VSHUFPDZ256rrik);
441 case X86::VPERMILPDZrik:
442 return ProcessVPERMILPDri(X86::VSHUFPDZrrik);
443
444 case X86::VPERMILPSri:
445 return ProcessVPERMILPSri(X86::VSHUFPSrri);
446 case X86::VPERMILPSYri:
447 return ProcessVPERMILPSri(X86::VSHUFPSYrri);
448 case X86::VPERMILPSZ128ri:
449 return ProcessVPERMILPSri(X86::VSHUFPSZ128rri);
450 case X86::VPERMILPSZ256ri:
451 return ProcessVPERMILPSri(X86::VSHUFPSZ256rri);
452 case X86::VPERMILPSZri:
453 return ProcessVPERMILPSri(X86::VSHUFPSZrri);
454 case X86::VPERMILPSZ128rikz:
455 return ProcessVPERMILPSri(X86::VSHUFPSZ128rrikz);
456 case X86::VPERMILPSZ256rikz:
457 return ProcessVPERMILPSri(X86::VSHUFPSZ256rrikz);
458 case X86::VPERMILPSZrikz:
459 return ProcessVPERMILPSri(X86::VSHUFPSZrrikz);
460 case X86::VPERMILPSZ128rik:
461 return ProcessVPERMILPSri(X86::VSHUFPSZ128rrik);
462 case X86::VPERMILPSZ256rik:
463 return ProcessVPERMILPSri(X86::VSHUFPSZ256rrik);
464 case X86::VPERMILPSZrik:
465 return ProcessVPERMILPSri(X86::VSHUFPSZrrik);
466 case X86::VPERMILPSmi:
467 return ProcessVPERMILPSmi(X86::VPSHUFDmi);
468 case X86::VPERMILPSYmi:
469 // TODO: See if there is a more generic way we can test if the replacement
470 // instruction is supported.
471 return ST->hasAVX2() ? ProcessVPERMILPSmi(X86::VPSHUFDYmi) : false;
472 case X86::VPERMILPSZ128mi:
473 return ProcessVPERMILPSmi(X86::VPSHUFDZ128mi);
474 case X86::VPERMILPSZ256mi:
475 return ProcessVPERMILPSmi(X86::VPSHUFDZ256mi);
476 case X86::VPERMILPSZmi:
477 return ProcessVPERMILPSmi(X86::VPSHUFDZmi);
478 case X86::VPERMILPSZ128mikz:
479 return ProcessVPERMILPSmi(X86::VPSHUFDZ128mikz);
480 case X86::VPERMILPSZ256mikz:
481 return ProcessVPERMILPSmi(X86::VPSHUFDZ256mikz);
482 case X86::VPERMILPSZmikz:
483 return ProcessVPERMILPSmi(X86::VPSHUFDZmikz);
484 case X86::VPERMILPSZ128mik:
485 return ProcessVPERMILPSmi(X86::VPSHUFDZ128mik);
486 case X86::VPERMILPSZ256mik:
487 return ProcessVPERMILPSmi(X86::VPSHUFDZ256mik);
488 case X86::VPERMILPSZmik:
489 return ProcessVPERMILPSmi(X86::VPSHUFDZmik);
490 case X86::VPERMQYri:
491 return ProcessVPERMQToVINSERT128(X86::VINSERTI128rri);
492 case X86::VPERMPDYri:
493 return ProcessVPERMQToVINSERT128(X86::VINSERTF128rri);
494 case X86::MOVLHPSrr:
495 case X86::UNPCKLPDrr:
496 return ProcessUNPCKLPDrr(X86::PUNPCKLQDQrr, X86::SHUFPDrri);
497 case X86::VMOVLHPSrr:
498 case X86::VUNPCKLPDrr:
499 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQrr, X86::VSHUFPDrri);
500 case X86::VUNPCKLPDYrr:
501 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQYrr, X86::VSHUFPDYrri);
502 // VMOVLHPS is always 128 bits.
503 case X86::VMOVLHPSZrr:
504 case X86::VUNPCKLPDZ128rr:
505 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ128rr, X86::VSHUFPDZ128rri);
506 case X86::VUNPCKLPDZ256rr:
507 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ256rr, X86::VSHUFPDZ256rri);
508 case X86::VUNPCKLPDZrr:
509 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZrr, X86::VSHUFPDZrri);
510 case X86::VUNPCKLPDZ128rrk:
511 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ128rrk, X86::VSHUFPDZ128rrik);
512 case X86::VUNPCKLPDZ256rrk:
513 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ256rrk, X86::VSHUFPDZ256rrik);
514 case X86::VUNPCKLPDZrrk:
515 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZrrk, X86::VSHUFPDZrrik);
516 case X86::VUNPCKLPDZ128rrkz:
517 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ128rrkz, X86::VSHUFPDZ128rrikz);
518 case X86::VUNPCKLPDZ256rrkz:
519 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ256rrkz, X86::VSHUFPDZ256rrikz);
520 case X86::VUNPCKLPDZrrkz:
521 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZrrkz, X86::VSHUFPDZrrikz);
522 case X86::UNPCKHPDrr:
523 return ProcessUNPCKHPDrr(X86::PUNPCKHQDQrr, X86::SHUFPDrri);
524 case X86::VUNPCKHPDrr:
525 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQrr, X86::VSHUFPDrri);
526 case X86::VUNPCKHPDYrr:
527 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQYrr, X86::VSHUFPDYrri);
528 case X86::VUNPCKHPDZ128rr:
529 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ128rr, X86::VSHUFPDZ128rri);
530 case X86::VUNPCKHPDZ256rr:
531 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ256rr, X86::VSHUFPDZ256rri);
532 case X86::VUNPCKHPDZrr:
533 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZrr, X86::VSHUFPDZrri);
534 case X86::VUNPCKHPDZ128rrk:
535 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ128rrk, X86::VSHUFPDZ128rrik);
536 case X86::VUNPCKHPDZ256rrk:
537 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ256rrk, X86::VSHUFPDZ256rrik);
538 case X86::VUNPCKHPDZrrk:
539 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZrrk, X86::VSHUFPDZrrik);
540 case X86::VUNPCKHPDZ128rrkz:
541 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ128rrkz, X86::VSHUFPDZ128rrikz);
542 case X86::VUNPCKHPDZ256rrkz:
543 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ256rrkz, X86::VSHUFPDZ256rrikz);
544 case X86::VUNPCKHPDZrrkz:
545 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZrrkz, X86::VSHUFPDZrrikz);
546 case X86::UNPCKLPDrm:
547 return ProcessUNPCKPDrm(X86::PUNPCKLQDQrm);
548 case X86::VUNPCKLPDrm:
549 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQrm);
550 case X86::VUNPCKLPDYrm:
551 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQYrm);
552 case X86::VUNPCKLPDZ128rm:
553 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ128rm);
554 case X86::VUNPCKLPDZ256rm:
555 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ256rm);
556 case X86::VUNPCKLPDZrm:
557 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZrm);
558 case X86::VUNPCKLPDZ128rmk:
559 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ128rmk);
560 case X86::VUNPCKLPDZ256rmk:
561 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ256rmk);
562 case X86::VUNPCKLPDZrmk:
563 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZrmk);
564 case X86::VUNPCKLPDZ128rmkz:
565 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ128rmkz);
566 case X86::VUNPCKLPDZ256rmkz:
567 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ256rmkz);
568 case X86::VUNPCKLPDZrmkz:
569 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZrmkz);
570 case X86::UNPCKHPDrm:
571 return ProcessUNPCKPDrm(X86::PUNPCKHQDQrm);
572 case X86::VUNPCKHPDrm:
573 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQrm);
574 case X86::VUNPCKHPDYrm:
575 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQYrm);
576 case X86::VUNPCKHPDZ128rm:
577 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ128rm);
578 case X86::VUNPCKHPDZ256rm:
579 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ256rm);
580 case X86::VUNPCKHPDZrm:
581 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZrm);
582 case X86::VUNPCKHPDZ128rmk:
583 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ128rmk);
584 case X86::VUNPCKHPDZ256rmk:
585 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ256rmk);
586 case X86::VUNPCKHPDZrmk:
587 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZrmk);
588 case X86::VUNPCKHPDZ128rmkz:
589 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ128rmkz);
590 case X86::VUNPCKHPDZ256rmkz:
591 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ256rmkz);
592 case X86::VUNPCKHPDZrmkz:
593 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZrmkz);
594
595 case X86::UNPCKLPSrr:
596 return ProcessUNPCKPS(X86::PUNPCKLDQrr);
597 case X86::VUNPCKLPSrr:
598 return ProcessUNPCKPS(X86::VPUNPCKLDQrr);
599 case X86::VUNPCKLPSYrr:
600 return ProcessUNPCKPS(X86::VPUNPCKLDQYrr);
601 case X86::VUNPCKLPSZ128rr:
602 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rr);
603 case X86::VUNPCKLPSZ256rr:
604 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rr);
605 case X86::VUNPCKLPSZrr:
606 return ProcessUNPCKPS(X86::VPUNPCKLDQZrr);
607 case X86::VUNPCKLPSZ128rrk:
608 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rrk);
609 case X86::VUNPCKLPSZ256rrk:
610 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rrk);
611 case X86::VUNPCKLPSZrrk:
612 return ProcessUNPCKPS(X86::VPUNPCKLDQZrrk);
613 case X86::VUNPCKLPSZ128rrkz:
614 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rrkz);
615 case X86::VUNPCKLPSZ256rrkz:
616 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rrkz);
617 case X86::VUNPCKLPSZrrkz:
618 return ProcessUNPCKPS(X86::VPUNPCKLDQZrrkz);
619 case X86::UNPCKHPSrr:
620 return ProcessUNPCKPS(X86::PUNPCKHDQrr);
621 case X86::VUNPCKHPSrr:
622 return ProcessUNPCKPS(X86::VPUNPCKHDQrr);
623 case X86::VUNPCKHPSYrr:
624 return ProcessUNPCKPS(X86::VPUNPCKHDQYrr);
625 case X86::VUNPCKHPSZ128rr:
626 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rr);
627 case X86::VUNPCKHPSZ256rr:
628 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rr);
629 case X86::VUNPCKHPSZrr:
630 return ProcessUNPCKPS(X86::VPUNPCKHDQZrr);
631 case X86::VUNPCKHPSZ128rrk:
632 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rrk);
633 case X86::VUNPCKHPSZ256rrk:
634 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rrk);
635 case X86::VUNPCKHPSZrrk:
636 return ProcessUNPCKPS(X86::VPUNPCKHDQZrrk);
637 case X86::VUNPCKHPSZ128rrkz:
638 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rrkz);
639 case X86::VUNPCKHPSZ256rrkz:
640 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rrkz);
641 case X86::VUNPCKHPSZrrkz:
642 return ProcessUNPCKPS(X86::VPUNPCKHDQZrrkz);
643 case X86::UNPCKLPSrm:
644 return ProcessUNPCKPS(X86::PUNPCKLDQrm);
645 case X86::VUNPCKLPSrm:
646 return ProcessUNPCKPS(X86::VPUNPCKLDQrm);
647 case X86::VUNPCKLPSYrm:
648 return ProcessUNPCKPS(X86::VPUNPCKLDQYrm);
649 case X86::VUNPCKLPSZ128rm:
650 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rm);
651 case X86::VUNPCKLPSZ256rm:
652 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rm);
653 case X86::VUNPCKLPSZrm:
654 return ProcessUNPCKPS(X86::VPUNPCKLDQZrm);
655 case X86::VUNPCKLPSZ128rmk:
656 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rmk);
657 case X86::VUNPCKLPSZ256rmk:
658 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rmk);
659 case X86::VUNPCKLPSZrmk:
660 return ProcessUNPCKPS(X86::VPUNPCKLDQZrmk);
661 case X86::VUNPCKLPSZ128rmkz:
662 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rmkz);
663 case X86::VUNPCKLPSZ256rmkz:
664 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rmkz);
665 case X86::VUNPCKLPSZrmkz:
666 return ProcessUNPCKPS(X86::VPUNPCKLDQZrmkz);
667 case X86::UNPCKHPSrm:
668 return ProcessUNPCKPS(X86::PUNPCKHDQrm);
669 case X86::VUNPCKHPSrm:
670 return ProcessUNPCKPS(X86::VPUNPCKHDQrm);
671 case X86::VUNPCKHPSYrm:
672 return ProcessUNPCKPS(X86::VPUNPCKHDQYrm);
673 case X86::VUNPCKHPSZ128rm:
674 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rm);
675 case X86::VUNPCKHPSZ256rm:
676 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rm);
677 case X86::VUNPCKHPSZrm:
678 return ProcessUNPCKPS(X86::VPUNPCKHDQZrm);
679 case X86::VUNPCKHPSZ128rmk:
680 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rmk);
681 case X86::VUNPCKHPSZ256rmk:
682 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rmk);
683 case X86::VUNPCKHPSZrmk:
684 return ProcessUNPCKPS(X86::VPUNPCKHDQZrmk);
685 case X86::VUNPCKHPSZ128rmkz:
686 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rmkz);
687 case X86::VUNPCKHPSZ256rmkz:
688 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rmkz);
689 case X86::VUNPCKHPSZrmkz:
690 return ProcessUNPCKPS(X86::VPUNPCKHDQZrmkz);
691
692 case X86::PSLLWri:
693 return ProcessShiftLeftToAdd(X86::PADDWrr);
694 case X86::VPSLLWri:
695 return ProcessShiftLeftToAdd(X86::VPADDWrr);
696 case X86::VPSLLWYri:
697 return ProcessShiftLeftToAdd(X86::VPADDWYrr);
698 case X86::VPSLLWZ128ri:
699 return ProcessShiftLeftToAdd(X86::VPADDWZ128rr);
700 case X86::VPSLLWZ256ri:
701 return ProcessShiftLeftToAdd(X86::VPADDWZ256rr);
702 case X86::VPSLLWZri:
703 return ProcessShiftLeftToAdd(X86::VPADDWZrr);
704 case X86::PSLLDri:
705 return ProcessShiftLeftToAdd(X86::PADDDrr);
706 case X86::VPSLLDri:
707 return ProcessShiftLeftToAdd(X86::VPADDDrr);
708 case X86::VPSLLDYri:
709 return ProcessShiftLeftToAdd(X86::VPADDDYrr);
710 case X86::VPSLLDZ128ri:
711 return ProcessShiftLeftToAdd(X86::VPADDDZ128rr);
712 case X86::VPSLLDZ256ri:
713 return ProcessShiftLeftToAdd(X86::VPADDDZ256rr);
714 case X86::VPSLLDZri:
715 return ProcessShiftLeftToAdd(X86::VPADDDZrr);
716 case X86::PSLLQri:
717 return ProcessShiftLeftToAdd(X86::PADDQrr);
718 case X86::VPSLLQri:
719 return ProcessShiftLeftToAdd(X86::VPADDQrr);
720 case X86::VPSLLQYri:
721 return ProcessShiftLeftToAdd(X86::VPADDQYrr);
722 case X86::VPSLLQZ128ri:
723 return ProcessShiftLeftToAdd(X86::VPADDQZ128rr);
724 case X86::VPSLLQZ256ri:
725 return ProcessShiftLeftToAdd(X86::VPADDQZ256rr);
726 case X86::VPSLLQZri:
727 return ProcessShiftLeftToAdd(X86::VPADDQZrr);
728 case X86::MOVUPDrr:
729 return ProcessMOVPDToMOVPS(X86::MOVUPSrr);
730 case X86::MOVUPDrm:
731 return ProcessMOVPDToMOVPS(X86::MOVUPSrm);
732 case X86::MOVUPDmr:
733 return ProcessMOVPDToMOVPS(X86::MOVUPSmr);
734 case X86::MOVAPDrr:
735 return ProcessMOVPDToMOVPS(X86::MOVAPSrr);
736 case X86::MOVAPDrm:
737 return ProcessMOVPDToMOVPS(X86::MOVAPSrm);
738 case X86::MOVAPDmr:
739 return ProcessMOVPDToMOVPS(X86::MOVAPSmr);
740
741 case X86::EXTRACTPSrri:
742 return ProcessEXTRACTPS(X86::MOVPDI2DIrr);
743 case X86::VEXTRACTPSrri:
744 return ProcessEXTRACTPS(X86::VMOVPDI2DIrr);
745 case X86::VEXTRACTPSZrri:
746 return ProcessEXTRACTPS(X86::VMOVPDI2DIZrr);
747 case X86::EXTRACTPSmri:
748 return ProcessEXTRACTPS(X86::MOVSSmr);
749 case X86::VEXTRACTPSmri:
750 return ProcessEXTRACTPS(X86::VMOVSSmr);
751 case X86::VEXTRACTPSZmri:
752 return ProcessEXTRACTPS(X86::VMOVSSZmr);
753
754 default:
755 return false;
756 }
757}
758
759bool X86FixupInstTuningImpl::runOnMachineFunction(MachineFunction &MF) {
760 LLVM_DEBUG(dbgs() << "Start X86FixupInstTuning\n";);
761 bool Changed = false;
762 ST = &MF.getSubtarget<X86Subtarget>();
763 TII = ST->getInstrInfo();
764 TRI = ST->getRegisterInfo();
765 SM = &ST->getSchedModel();
766
767 for (MachineBasicBlock &MBB : MF) {
768 for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) {
769 if (processInstruction(MF, MBB, I)) {
770 ++NumInstChanges;
771 Changed = true;
772 }
773 }
774 }
775 LLVM_DEBUG(dbgs() << "End X86FixupInstTuning\n";);
776 return Changed;
777}
778
779bool X86FixupInstTuningLegacy::runOnMachineFunction(MachineFunction &MF) {
780 X86FixupInstTuningImpl Impl;
781 return Impl.runOnMachineFunction(MF);
782}
783
784PreservedAnalyses
785X86FixupInstTuningPass::run(MachineFunction &MF,
786 MachineFunctionAnalysisManager &MFAM) {
787 X86FixupInstTuningImpl Impl;
788 return Impl.runOnMachineFunction(MF)
789 ? getMachineFunctionPassPreservedAnalyses()
790 .preserveSet<CFGAnalyses>()
791 : PreservedAnalyses::all();
792}
793