1//===-- X86FixupInstTuning.cpp - replace instructions -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file does a tuning pass replacing slower machine instructions
10// with faster ones. We do this here, as opposed to during normal ISel, as
11// attempting to get the "right" instruction can break patterns. This pass
12// is not meant search for special cases where an instruction can be transformed
13// to another, it is only meant to do transformations where the old instruction
14// is always replacable with the new instructions. For example:
15//
16// `vpermq ymm` -> `vshufd ymm`
17// -- BAD, not always valid (lane cross/non-repeated mask)
18//
19// `vpermilps ymm` -> `vshufd ymm`
20// -- GOOD, always replaceable
21//
22//===----------------------------------------------------------------------===//
23
24#include "X86.h"
25#include "X86InstrInfo.h"
26#include "X86RegisterInfo.h"
27#include "X86Subtarget.h"
28#include "llvm/ADT/Statistic.h"
29#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
30#include "llvm/CodeGen/MachineFunctionPass.h"
31#include "llvm/CodeGen/MachineInstrBuilder.h"
32#include "llvm/CodeGen/MachinePassManager.h"
33#include "llvm/IR/Analysis.h"
34
35using namespace llvm;
36
37#define DEBUG_TYPE "x86-fixup-inst-tuning"
38
39STATISTIC(NumInstChanges, "Number of instructions changes");
40
41namespace {
42class X86FixupInstTuningImpl {
43public:
44 bool runOnMachineFunction(MachineFunction &MF);
45
46private:
47 bool processInstruction(MachineFunction &MF, MachineBasicBlock &MBB,
48 MachineBasicBlock::iterator &I);
49
50 const X86InstrInfo *TII = nullptr;
51 const X86Subtarget *ST = nullptr;
52 const MCSchedModel *SM = nullptr;
53 const X86RegisterInfo *TRI = nullptr;
54};
55
56class X86FixupInstTuningLegacy : public MachineFunctionPass {
57public:
58 static char ID;
59
60 X86FixupInstTuningLegacy() : MachineFunctionPass(ID) {}
61
62 StringRef getPassName() const override { return "X86 Fixup Inst Tuning"; }
63
64 bool runOnMachineFunction(MachineFunction &MF) override;
65 bool processInstruction(MachineFunction &MF, MachineBasicBlock &MBB,
66 MachineBasicBlock::iterator &I);
67
68 // This pass runs after regalloc and doesn't support VReg operands.
69 MachineFunctionProperties getRequiredProperties() const override {
70 return MachineFunctionProperties().setNoVRegs();
71 }
72};
73} // end anonymous namespace
74
75char X86FixupInstTuningLegacy ::ID = 0;
76
77INITIALIZE_PASS(X86FixupInstTuningLegacy, DEBUG_TYPE, DEBUG_TYPE, false, false)
78
79FunctionPass *llvm::createX86FixupInstTuningLegacyPass() {
80 return new X86FixupInstTuningLegacy();
81}
82
83template <typename T>
84static std::optional<bool> CmpOptionals(T NewVal, T CurVal) {
85 if (NewVal.has_value() && CurVal.has_value() && *NewVal != *CurVal)
86 return *NewVal < *CurVal;
87
88 return std::nullopt;
89}
90
91bool X86FixupInstTuningImpl::processInstruction(
92 MachineFunction &MF, MachineBasicBlock &MBB,
93 MachineBasicBlock::iterator &I) {
94 MachineInstr &MI = *I;
95 unsigned Opc = MI.getOpcode();
96 unsigned NumOperands = MI.getDesc().getNumOperands();
97 bool OptSize = MF.getFunction().hasOptSize();
98
99 auto GetInstTput = [&](unsigned Opcode) -> std::optional<double> {
100 // We already checked that SchedModel exists in `NewOpcPreferable`.
101 return MCSchedModel::getReciprocalThroughput(
102 STI: *ST, SCDesc: *(SM->getSchedClassDesc(SchedClassIdx: TII->get(Opcode).getSchedClass())));
103 };
104
105 auto GetInstLat = [&](unsigned Opcode) -> std::optional<double> {
106 // We already checked that SchedModel exists in `NewOpcPreferable`.
107 return MCSchedModel::computeInstrLatency(
108 STI: *ST, SCDesc: *(SM->getSchedClassDesc(SchedClassIdx: TII->get(Opcode).getSchedClass())));
109 };
110
111 auto GetInstSize = [&](unsigned Opcode) -> std::optional<unsigned> {
112 if (unsigned Size = TII->get(Opcode).getSize())
113 return Size;
114 // Zero size means we where unable to compute it.
115 return std::nullopt;
116 };
117
118 auto NewOpcPreferable = [&](unsigned NewOpc,
119 bool ReplaceInTie = true) -> bool {
120 std::optional<bool> Res;
121 if (SM->hasInstrSchedModel()) {
122 // Compare tput -> lat -> code size.
123 Res = CmpOptionals(NewVal: GetInstTput(NewOpc), CurVal: GetInstTput(Opc));
124 if (Res.has_value())
125 return *Res;
126
127 Res = CmpOptionals(NewVal: GetInstLat(NewOpc), CurVal: GetInstLat(Opc));
128 if (Res.has_value())
129 return *Res;
130 }
131
132 Res = CmpOptionals(NewVal: GetInstSize(Opc), CurVal: GetInstSize(NewOpc));
133 if (Res.has_value())
134 return *Res;
135
136 // We either have either were unable to get tput/lat/codesize or all values
137 // were equal. Return specified option for a tie.
138 return ReplaceInTie;
139 };
140
141 // `vpermilpd r, i` -> `vshufpd r, r, i`
142 // `vpermilpd r, i, k` -> `vshufpd r, r, i, k`
143 // `vshufpd` is always as fast or faster than `vpermilpd` and takes
144 // 1 less byte of code size for VEX and EVEX encoding.
145 auto ProcessVPERMILPDri = [&](unsigned NewOpc) -> bool {
146 if (!NewOpcPreferable(NewOpc))
147 return false;
148 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
149 {
150 unsigned MaskImm = MI.getOperand(i: NumOperands - 1).getImm();
151 MI.removeOperand(OpNo: NumOperands - 1);
152 MI.addOperand(Op: MI.getOperand(i: NumOperands - 2));
153 MI.setDesc(TII->get(Opcode: NewOpc));
154 MI.addOperand(Op: MachineOperand::CreateImm(Val: MaskImm));
155 }
156 LLVM_DEBUG(dbgs() << " With: " << MI);
157 return true;
158 };
159
160 // `vpermilps r, i` -> `vshufps r, r, i`
161 // `vpermilps r, i, k` -> `vshufps r, r, i, k`
162 // `vshufps` is always as fast or faster than `vpermilps` and takes
163 // 1 less byte of code size for VEX and EVEX encoding.
164 auto ProcessVPERMILPSri = [&](unsigned NewOpc) -> bool {
165 if (!NewOpcPreferable(NewOpc))
166 return false;
167 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
168 {
169 unsigned MaskImm = MI.getOperand(i: NumOperands - 1).getImm();
170 MI.removeOperand(OpNo: NumOperands - 1);
171 MI.addOperand(Op: MI.getOperand(i: NumOperands - 2));
172 MI.setDesc(TII->get(Opcode: NewOpc));
173 MI.addOperand(Op: MachineOperand::CreateImm(Val: MaskImm));
174 }
175 LLVM_DEBUG(dbgs() << " With: " << MI);
176 return true;
177 };
178
179 // `vpermilps m, i` -> `vpshufd m, i` iff no domain delay penalty on shuffles.
180 // `vpshufd` is always as fast or faster than `vpermilps` and takes 1 less
181 // byte of code size.
182 auto ProcessVPERMILPSmi = [&](unsigned NewOpc) -> bool {
183 // TODO: Might be work adding bypass delay if -Os/-Oz is enabled as
184 // `vpshufd` saves a byte of code size.
185 if (!ST->hasNoDomainDelayShuffle() ||
186 !NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
187 return false;
188 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
189 {
190 MI.setDesc(TII->get(Opcode: NewOpc));
191 }
192 LLVM_DEBUG(dbgs() << " With: " << MI);
193 return true;
194 };
195
196 // `vunpcklpd/vmovlhps r, r` -> `vunpcklqdq r, r`/`vshufpd r, r, 0x00`
197 // `vunpckhpd/vmovlhps r, r` -> `vunpckhqdq r, r`/`vshufpd r, r, 0xff`
198 // `vunpcklpd r, r, k` -> `vunpcklqdq r, r, k`/`vshufpd r, r, k, 0x00`
199 // `vunpckhpd r, r, k` -> `vunpckhqdq r, r, k`/`vshufpd r, r, k, 0xff`
200 // `vunpcklpd r, m` -> `vunpcklqdq r, m, k`
201 // `vunpckhpd r, m` -> `vunpckhqdq r, m, k`
202 // `vunpcklpd r, m, k` -> `vunpcklqdq r, m, k`
203 // `vunpckhpd r, m, k` -> `vunpckhqdq r, m, k`
204 // 1) If no bypass delay and `vunpck{l|h}qdq` faster than `vunpck{l|h}pd`
205 // -> `vunpck{l|h}qdq`
206 // 2) If `vshufpd` faster than `vunpck{l|h}pd`
207 // -> `vshufpd`
208 //
209 // `vunpcklps` -> `vunpckldq` (for all operand types if no bypass delay)
210 auto ProcessUNPCK = [&](unsigned NewOpc, unsigned MaskImm) -> bool {
211 if (!NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
212 return false;
213 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
214 {
215 MI.setDesc(TII->get(Opcode: NewOpc));
216 MI.addOperand(Op: MachineOperand::CreateImm(Val: MaskImm));
217 }
218 LLVM_DEBUG(dbgs() << " With: " << MI);
219 return true;
220 };
221
222 auto ProcessUNPCKToIntDomain = [&](unsigned NewOpc) -> bool {
223 // TODO it may be worth it to set ReplaceInTie to `true` as there is no real
224 // downside to the integer unpck, but if someone doesn't specify exact
225 // target we won't find it faster.
226 if (!ST->hasNoDomainDelayShuffle() ||
227 !NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
228 return false;
229 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
230 {
231 MI.setDesc(TII->get(Opcode: NewOpc));
232 }
233 LLVM_DEBUG(dbgs() << " With: " << MI);
234 return true;
235 };
236
237 auto ProcessUNPCKLPDrr = [&](unsigned NewOpcIntDomain,
238 unsigned NewOpc) -> bool {
239 if (ProcessUNPCKToIntDomain(NewOpcIntDomain))
240 return true;
241 return ProcessUNPCK(NewOpc, 0x00);
242 };
243 auto ProcessUNPCKHPDrr = [&](unsigned NewOpcIntDomain,
244 unsigned NewOpc) -> bool {
245 if (ProcessUNPCKToIntDomain(NewOpcIntDomain))
246 return true;
247 return ProcessUNPCK(NewOpc, 0xff);
248 };
249
250 auto ProcessUNPCKPDrm = [&](unsigned NewOpcIntDomain) -> bool {
251 return ProcessUNPCKToIntDomain(NewOpcIntDomain);
252 };
253
254 auto ProcessUNPCKPS = [&](unsigned NewOpc) -> bool {
255 return ProcessUNPCKToIntDomain(NewOpc);
256 };
257
258 // MOVUPS/MOVAPS takes 1 less byte of code size. Only replace when
259 // there is no move domain-delay penalty on the target, or -Oz is set.
260 auto ProcessMOVPDToMOVPS = [&](unsigned NewOpc) -> bool {
261 assert(NewOpcPreferable(NewOpc) &&
262 "MOVUPS/MOVAPS should be preferred over MOVUPD/MOVAPD");
263 if (!ST->hasNoDomainDelayMov() && !MF.getFunction().hasMinSize())
264 return false;
265 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
266 MI.setDesc(TII->get(Opcode: NewOpc));
267 LLVM_DEBUG(dbgs() << " With: " << MI);
268 return true;
269 };
270
271 // If we're permuting the lower halves of the 256-bit registers, use a
272 // subvector insertion instead.
273 auto ProcessVPERM2x128ToVINSERT128 = [&](unsigned InsertOpc) -> bool {
274 unsigned PermImm = MI.getOperand(i: NumOperands - 1).getImm();
275 // TODO: Handle 0x00/0x02/0x22 when we have test coverage.
276 if (PermImm != 0x20 || !NewOpcPreferable(InsertOpc))
277 return false;
278 Register RHSRegYMM = MI.getOperand(i: NumOperands - 2).getReg();
279 Register RHSRegXMM = TRI->getSubReg(Reg: RHSRegYMM, Idx: X86::sub_xmm);
280 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
281 {
282 MI.setDesc(TII->get(Opcode: InsertOpc));
283 MI.removeOperand(OpNo: NumOperands - 1);
284 MI.removeOperand(OpNo: NumOperands - 2);
285 // Add the XMM subregister operand.
286 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RHSRegXMM, /*isDef=*/false,
287 /*isImp=*/false,
288 /*isKill=*/false));
289 // Add the immediate (1 = insert into high 128-bits).
290 MI.addOperand(Op: MachineOperand::CreateImm(Val: 1));
291 }
292 LLVM_DEBUG(dbgs() << " With: " << MI);
293 return true;
294 };
295
296 auto ProcessBLENDWToBLENDD = [&](unsigned MovOpc, unsigned NumElts) -> bool {
297 if (!ST->hasAVX2() || !NewOpcPreferable(MovOpc))
298 return false;
299 // Convert to VPBLENDD if scaling the VPBLENDW mask down/up loses no bits.
300 APInt MaskW =
301 APInt(8, MI.getOperand(i: NumOperands - 1).getImm(), /*IsSigned=*/false,
302 /*implicitTrunc=*/true);
303 APInt MaskD = APIntOps::ScaleBitMask(A: MaskW, NewBitWidth: 4, /*MatchAllBits=*/true);
304 if (MaskW != APIntOps::ScaleBitMask(A: MaskD, NewBitWidth: 8, /*MatchAllBits=*/true))
305 return false;
306 APInt NewMaskD = APInt::getSplat(NewLen: NumElts, V: MaskD);
307 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
308 {
309 MI.setDesc(TII->get(Opcode: MovOpc));
310 MI.removeOperand(OpNo: NumOperands - 1);
311 MI.addOperand(Op: MachineOperand::CreateImm(Val: NewMaskD.getZExtValue()));
312 }
313 LLVM_DEBUG(dbgs() << " With: " << MI);
314 return true;
315 };
316
317 auto ProcessBLENDToMOV = [&](unsigned MovOpc, unsigned Mask,
318 unsigned MovImm) -> bool {
319 if ((MI.getOperand(i: NumOperands - 1).getImm() & Mask) != MovImm)
320 return false;
321 if (!OptSize && !NewOpcPreferable(MovOpc))
322 return false;
323 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
324 {
325 MI.setDesc(TII->get(Opcode: MovOpc));
326 MI.removeOperand(OpNo: NumOperands - 1);
327 }
328 LLVM_DEBUG(dbgs() << " With: " << MI);
329 return true;
330 };
331
332 // Is ADD(X,X) more efficient than SHL(X,1)?
333 auto ProcessShiftLeftToAdd = [&](unsigned AddOpc) -> bool {
334 if (MI.getOperand(i: NumOperands - 1).getImm() != 1)
335 return false;
336 if (!NewOpcPreferable(AddOpc, /*ReplaceInTie*/ true))
337 return false;
338 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
339 {
340 MI.setDesc(TII->get(Opcode: AddOpc));
341 MI.removeOperand(OpNo: NumOperands - 1);
342 MI.addOperand(Op: MI.getOperand(i: NumOperands - 2));
343 }
344 LLVM_DEBUG(dbgs() << " With: " << MI);
345 return true;
346 };
347
348 // `vpermq ymm, ymm, 0x44` -> `vinserti128 ymm, ymm, xmm, 1`
349 // `vpermpd ymm, ymm, 0x44` -> `vinsertf128 ymm, ymm, xmm, 1`
350 // When the immediate is 0x44, VPERMQ/VPERMPD duplicates the lower 128-bit
351 // lane to both lanes. 0x44 = 0b01_00_01_00 means qwords[3:0] = {src[1],
352 // src[0], src[1], src[0]} This is equivalent to inserting the lower 128-bits
353 // into the upper 128-bit position.
354 auto ProcessVPERMQToVINSERT128 = [&](unsigned NewOpc) -> bool {
355 if (MI.getOperand(i: NumOperands - 1).getImm() != 0x44)
356 return false;
357 if (!NewOpcPreferable(NewOpc, /*ReplaceInTie*/ false))
358 return false;
359
360 // Get the XMM subregister of the source YMM register.
361 Register SrcReg = MI.getOperand(i: 1).getReg();
362 Register XmmReg = TRI->getSubReg(Reg: SrcReg, Idx: X86::sub_xmm);
363
364 LLVM_DEBUG(dbgs() << "Replacing: " << MI);
365 {
366 // Transform: VPERMQ $dst, $src, $0x44
367 // Into: VINSERTI128 $dst, $src, $xmm_src, $1
368 MI.setDesc(TII->get(Opcode: NewOpc));
369 // Remove the immediate operand.
370 MI.removeOperand(OpNo: NumOperands - 1);
371 // Add the XMM subregister operand.
372 MI.addOperand(Op: MachineOperand::CreateReg(Reg: XmmReg, /*isDef=*/false,
373 /*isImp=*/false,
374 /*isKill=*/false));
375 // Add the immediate (1 = insert into high 128-bits).
376 MI.addOperand(Op: MachineOperand::CreateImm(Val: 1));
377 }
378 LLVM_DEBUG(dbgs() << " With: " << MI);
379 return true;
380 };
381
382 switch (Opc) {
383 case X86::BLENDPDrri:
384 return ProcessBLENDToMOV(X86::MOVSDrr, 0x3, 0x1);
385 case X86::VBLENDPDrri:
386 return ProcessBLENDToMOV(X86::VMOVSDrr, 0x3, 0x1);
387
388 case X86::BLENDPSrri:
389 return ProcessBLENDToMOV(X86::MOVSSrr, 0xF, 0x1) ||
390 ProcessBLENDToMOV(X86::MOVSDrr, 0xF, 0x3);
391 case X86::VBLENDPSrri:
392 return ProcessBLENDToMOV(X86::VMOVSSrr, 0xF, 0x1) ||
393 ProcessBLENDToMOV(X86::VMOVSDrr, 0xF, 0x3);
394
395 case X86::VPBLENDWrri:
396 // TODO: Add X86::VPBLENDWrmi handling
397 // TODO: Add X86::VPBLENDWYrri handling
398 // TODO: Add X86::VPBLENDWYrmi handling
399 return ProcessBLENDWToBLENDD(X86::VPBLENDDrri, 4);
400
401 case X86::VPERM2F128rri:
402 return ProcessVPERM2x128ToVINSERT128(X86::VINSERTF128rri);
403 case X86::VPERM2I128rri:
404 return ProcessVPERM2x128ToVINSERT128(X86::VINSERTI128rri);
405
406 case X86::VPERMILPDri:
407 return ProcessVPERMILPDri(X86::VSHUFPDrri);
408 case X86::VPERMILPDYri:
409 return ProcessVPERMILPDri(X86::VSHUFPDYrri);
410 case X86::VPERMILPDZ128ri:
411 return ProcessVPERMILPDri(X86::VSHUFPDZ128rri);
412 case X86::VPERMILPDZ256ri:
413 return ProcessVPERMILPDri(X86::VSHUFPDZ256rri);
414 case X86::VPERMILPDZri:
415 return ProcessVPERMILPDri(X86::VSHUFPDZrri);
416 case X86::VPERMILPDZ128rikz:
417 return ProcessVPERMILPDri(X86::VSHUFPDZ128rrikz);
418 case X86::VPERMILPDZ256rikz:
419 return ProcessVPERMILPDri(X86::VSHUFPDZ256rrikz);
420 case X86::VPERMILPDZrikz:
421 return ProcessVPERMILPDri(X86::VSHUFPDZrrikz);
422 case X86::VPERMILPDZ128rik:
423 return ProcessVPERMILPDri(X86::VSHUFPDZ128rrik);
424 case X86::VPERMILPDZ256rik:
425 return ProcessVPERMILPDri(X86::VSHUFPDZ256rrik);
426 case X86::VPERMILPDZrik:
427 return ProcessVPERMILPDri(X86::VSHUFPDZrrik);
428
429 case X86::VPERMILPSri:
430 return ProcessVPERMILPSri(X86::VSHUFPSrri);
431 case X86::VPERMILPSYri:
432 return ProcessVPERMILPSri(X86::VSHUFPSYrri);
433 case X86::VPERMILPSZ128ri:
434 return ProcessVPERMILPSri(X86::VSHUFPSZ128rri);
435 case X86::VPERMILPSZ256ri:
436 return ProcessVPERMILPSri(X86::VSHUFPSZ256rri);
437 case X86::VPERMILPSZri:
438 return ProcessVPERMILPSri(X86::VSHUFPSZrri);
439 case X86::VPERMILPSZ128rikz:
440 return ProcessVPERMILPSri(X86::VSHUFPSZ128rrikz);
441 case X86::VPERMILPSZ256rikz:
442 return ProcessVPERMILPSri(X86::VSHUFPSZ256rrikz);
443 case X86::VPERMILPSZrikz:
444 return ProcessVPERMILPSri(X86::VSHUFPSZrrikz);
445 case X86::VPERMILPSZ128rik:
446 return ProcessVPERMILPSri(X86::VSHUFPSZ128rrik);
447 case X86::VPERMILPSZ256rik:
448 return ProcessVPERMILPSri(X86::VSHUFPSZ256rrik);
449 case X86::VPERMILPSZrik:
450 return ProcessVPERMILPSri(X86::VSHUFPSZrrik);
451 case X86::VPERMILPSmi:
452 return ProcessVPERMILPSmi(X86::VPSHUFDmi);
453 case X86::VPERMILPSYmi:
454 // TODO: See if there is a more generic way we can test if the replacement
455 // instruction is supported.
456 return ST->hasAVX2() ? ProcessVPERMILPSmi(X86::VPSHUFDYmi) : false;
457 case X86::VPERMILPSZ128mi:
458 return ProcessVPERMILPSmi(X86::VPSHUFDZ128mi);
459 case X86::VPERMILPSZ256mi:
460 return ProcessVPERMILPSmi(X86::VPSHUFDZ256mi);
461 case X86::VPERMILPSZmi:
462 return ProcessVPERMILPSmi(X86::VPSHUFDZmi);
463 case X86::VPERMILPSZ128mikz:
464 return ProcessVPERMILPSmi(X86::VPSHUFDZ128mikz);
465 case X86::VPERMILPSZ256mikz:
466 return ProcessVPERMILPSmi(X86::VPSHUFDZ256mikz);
467 case X86::VPERMILPSZmikz:
468 return ProcessVPERMILPSmi(X86::VPSHUFDZmikz);
469 case X86::VPERMILPSZ128mik:
470 return ProcessVPERMILPSmi(X86::VPSHUFDZ128mik);
471 case X86::VPERMILPSZ256mik:
472 return ProcessVPERMILPSmi(X86::VPSHUFDZ256mik);
473 case X86::VPERMILPSZmik:
474 return ProcessVPERMILPSmi(X86::VPSHUFDZmik);
475 case X86::VPERMQYri:
476 return ProcessVPERMQToVINSERT128(X86::VINSERTI128rri);
477 case X86::VPERMPDYri:
478 return ProcessVPERMQToVINSERT128(X86::VINSERTF128rri);
479 case X86::MOVLHPSrr:
480 case X86::UNPCKLPDrr:
481 return ProcessUNPCKLPDrr(X86::PUNPCKLQDQrr, X86::SHUFPDrri);
482 case X86::VMOVLHPSrr:
483 case X86::VUNPCKLPDrr:
484 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQrr, X86::VSHUFPDrri);
485 case X86::VUNPCKLPDYrr:
486 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQYrr, X86::VSHUFPDYrri);
487 // VMOVLHPS is always 128 bits.
488 case X86::VMOVLHPSZrr:
489 case X86::VUNPCKLPDZ128rr:
490 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ128rr, X86::VSHUFPDZ128rri);
491 case X86::VUNPCKLPDZ256rr:
492 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ256rr, X86::VSHUFPDZ256rri);
493 case X86::VUNPCKLPDZrr:
494 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZrr, X86::VSHUFPDZrri);
495 case X86::VUNPCKLPDZ128rrk:
496 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ128rrk, X86::VSHUFPDZ128rrik);
497 case X86::VUNPCKLPDZ256rrk:
498 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ256rrk, X86::VSHUFPDZ256rrik);
499 case X86::VUNPCKLPDZrrk:
500 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZrrk, X86::VSHUFPDZrrik);
501 case X86::VUNPCKLPDZ128rrkz:
502 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ128rrkz, X86::VSHUFPDZ128rrikz);
503 case X86::VUNPCKLPDZ256rrkz:
504 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZ256rrkz, X86::VSHUFPDZ256rrikz);
505 case X86::VUNPCKLPDZrrkz:
506 return ProcessUNPCKLPDrr(X86::VPUNPCKLQDQZrrkz, X86::VSHUFPDZrrikz);
507 case X86::UNPCKHPDrr:
508 return ProcessUNPCKHPDrr(X86::PUNPCKHQDQrr, X86::SHUFPDrri);
509 case X86::VUNPCKHPDrr:
510 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQrr, X86::VSHUFPDrri);
511 case X86::VUNPCKHPDYrr:
512 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQYrr, X86::VSHUFPDYrri);
513 case X86::VUNPCKHPDZ128rr:
514 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ128rr, X86::VSHUFPDZ128rri);
515 case X86::VUNPCKHPDZ256rr:
516 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ256rr, X86::VSHUFPDZ256rri);
517 case X86::VUNPCKHPDZrr:
518 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZrr, X86::VSHUFPDZrri);
519 case X86::VUNPCKHPDZ128rrk:
520 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ128rrk, X86::VSHUFPDZ128rrik);
521 case X86::VUNPCKHPDZ256rrk:
522 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ256rrk, X86::VSHUFPDZ256rrik);
523 case X86::VUNPCKHPDZrrk:
524 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZrrk, X86::VSHUFPDZrrik);
525 case X86::VUNPCKHPDZ128rrkz:
526 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ128rrkz, X86::VSHUFPDZ128rrikz);
527 case X86::VUNPCKHPDZ256rrkz:
528 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZ256rrkz, X86::VSHUFPDZ256rrikz);
529 case X86::VUNPCKHPDZrrkz:
530 return ProcessUNPCKHPDrr(X86::VPUNPCKHQDQZrrkz, X86::VSHUFPDZrrikz);
531 case X86::UNPCKLPDrm:
532 return ProcessUNPCKPDrm(X86::PUNPCKLQDQrm);
533 case X86::VUNPCKLPDrm:
534 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQrm);
535 case X86::VUNPCKLPDYrm:
536 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQYrm);
537 case X86::VUNPCKLPDZ128rm:
538 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ128rm);
539 case X86::VUNPCKLPDZ256rm:
540 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ256rm);
541 case X86::VUNPCKLPDZrm:
542 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZrm);
543 case X86::VUNPCKLPDZ128rmk:
544 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ128rmk);
545 case X86::VUNPCKLPDZ256rmk:
546 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ256rmk);
547 case X86::VUNPCKLPDZrmk:
548 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZrmk);
549 case X86::VUNPCKLPDZ128rmkz:
550 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ128rmkz);
551 case X86::VUNPCKLPDZ256rmkz:
552 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZ256rmkz);
553 case X86::VUNPCKLPDZrmkz:
554 return ProcessUNPCKPDrm(X86::VPUNPCKLQDQZrmkz);
555 case X86::UNPCKHPDrm:
556 return ProcessUNPCKPDrm(X86::PUNPCKHQDQrm);
557 case X86::VUNPCKHPDrm:
558 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQrm);
559 case X86::VUNPCKHPDYrm:
560 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQYrm);
561 case X86::VUNPCKHPDZ128rm:
562 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ128rm);
563 case X86::VUNPCKHPDZ256rm:
564 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ256rm);
565 case X86::VUNPCKHPDZrm:
566 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZrm);
567 case X86::VUNPCKHPDZ128rmk:
568 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ128rmk);
569 case X86::VUNPCKHPDZ256rmk:
570 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ256rmk);
571 case X86::VUNPCKHPDZrmk:
572 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZrmk);
573 case X86::VUNPCKHPDZ128rmkz:
574 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ128rmkz);
575 case X86::VUNPCKHPDZ256rmkz:
576 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZ256rmkz);
577 case X86::VUNPCKHPDZrmkz:
578 return ProcessUNPCKPDrm(X86::VPUNPCKHQDQZrmkz);
579
580 case X86::UNPCKLPSrr:
581 return ProcessUNPCKPS(X86::PUNPCKLDQrr);
582 case X86::VUNPCKLPSrr:
583 return ProcessUNPCKPS(X86::VPUNPCKLDQrr);
584 case X86::VUNPCKLPSYrr:
585 return ProcessUNPCKPS(X86::VPUNPCKLDQYrr);
586 case X86::VUNPCKLPSZ128rr:
587 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rr);
588 case X86::VUNPCKLPSZ256rr:
589 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rr);
590 case X86::VUNPCKLPSZrr:
591 return ProcessUNPCKPS(X86::VPUNPCKLDQZrr);
592 case X86::VUNPCKLPSZ128rrk:
593 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rrk);
594 case X86::VUNPCKLPSZ256rrk:
595 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rrk);
596 case X86::VUNPCKLPSZrrk:
597 return ProcessUNPCKPS(X86::VPUNPCKLDQZrrk);
598 case X86::VUNPCKLPSZ128rrkz:
599 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rrkz);
600 case X86::VUNPCKLPSZ256rrkz:
601 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rrkz);
602 case X86::VUNPCKLPSZrrkz:
603 return ProcessUNPCKPS(X86::VPUNPCKLDQZrrkz);
604 case X86::UNPCKHPSrr:
605 return ProcessUNPCKPS(X86::PUNPCKHDQrr);
606 case X86::VUNPCKHPSrr:
607 return ProcessUNPCKPS(X86::VPUNPCKHDQrr);
608 case X86::VUNPCKHPSYrr:
609 return ProcessUNPCKPS(X86::VPUNPCKHDQYrr);
610 case X86::VUNPCKHPSZ128rr:
611 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rr);
612 case X86::VUNPCKHPSZ256rr:
613 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rr);
614 case X86::VUNPCKHPSZrr:
615 return ProcessUNPCKPS(X86::VPUNPCKHDQZrr);
616 case X86::VUNPCKHPSZ128rrk:
617 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rrk);
618 case X86::VUNPCKHPSZ256rrk:
619 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rrk);
620 case X86::VUNPCKHPSZrrk:
621 return ProcessUNPCKPS(X86::VPUNPCKHDQZrrk);
622 case X86::VUNPCKHPSZ128rrkz:
623 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rrkz);
624 case X86::VUNPCKHPSZ256rrkz:
625 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rrkz);
626 case X86::VUNPCKHPSZrrkz:
627 return ProcessUNPCKPS(X86::VPUNPCKHDQZrrkz);
628 case X86::UNPCKLPSrm:
629 return ProcessUNPCKPS(X86::PUNPCKLDQrm);
630 case X86::VUNPCKLPSrm:
631 return ProcessUNPCKPS(X86::VPUNPCKLDQrm);
632 case X86::VUNPCKLPSYrm:
633 return ProcessUNPCKPS(X86::VPUNPCKLDQYrm);
634 case X86::VUNPCKLPSZ128rm:
635 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rm);
636 case X86::VUNPCKLPSZ256rm:
637 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rm);
638 case X86::VUNPCKLPSZrm:
639 return ProcessUNPCKPS(X86::VPUNPCKLDQZrm);
640 case X86::VUNPCKLPSZ128rmk:
641 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rmk);
642 case X86::VUNPCKLPSZ256rmk:
643 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rmk);
644 case X86::VUNPCKLPSZrmk:
645 return ProcessUNPCKPS(X86::VPUNPCKLDQZrmk);
646 case X86::VUNPCKLPSZ128rmkz:
647 return ProcessUNPCKPS(X86::VPUNPCKLDQZ128rmkz);
648 case X86::VUNPCKLPSZ256rmkz:
649 return ProcessUNPCKPS(X86::VPUNPCKLDQZ256rmkz);
650 case X86::VUNPCKLPSZrmkz:
651 return ProcessUNPCKPS(X86::VPUNPCKLDQZrmkz);
652 case X86::UNPCKHPSrm:
653 return ProcessUNPCKPS(X86::PUNPCKHDQrm);
654 case X86::VUNPCKHPSrm:
655 return ProcessUNPCKPS(X86::VPUNPCKHDQrm);
656 case X86::VUNPCKHPSYrm:
657 return ProcessUNPCKPS(X86::VPUNPCKHDQYrm);
658 case X86::VUNPCKHPSZ128rm:
659 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rm);
660 case X86::VUNPCKHPSZ256rm:
661 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rm);
662 case X86::VUNPCKHPSZrm:
663 return ProcessUNPCKPS(X86::VPUNPCKHDQZrm);
664 case X86::VUNPCKHPSZ128rmk:
665 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rmk);
666 case X86::VUNPCKHPSZ256rmk:
667 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rmk);
668 case X86::VUNPCKHPSZrmk:
669 return ProcessUNPCKPS(X86::VPUNPCKHDQZrmk);
670 case X86::VUNPCKHPSZ128rmkz:
671 return ProcessUNPCKPS(X86::VPUNPCKHDQZ128rmkz);
672 case X86::VUNPCKHPSZ256rmkz:
673 return ProcessUNPCKPS(X86::VPUNPCKHDQZ256rmkz);
674 case X86::VUNPCKHPSZrmkz:
675 return ProcessUNPCKPS(X86::VPUNPCKHDQZrmkz);
676
677 case X86::PSLLWri:
678 return ProcessShiftLeftToAdd(X86::PADDWrr);
679 case X86::VPSLLWri:
680 return ProcessShiftLeftToAdd(X86::VPADDWrr);
681 case X86::VPSLLWYri:
682 return ProcessShiftLeftToAdd(X86::VPADDWYrr);
683 case X86::VPSLLWZ128ri:
684 return ProcessShiftLeftToAdd(X86::VPADDWZ128rr);
685 case X86::VPSLLWZ256ri:
686 return ProcessShiftLeftToAdd(X86::VPADDWZ256rr);
687 case X86::VPSLLWZri:
688 return ProcessShiftLeftToAdd(X86::VPADDWZrr);
689 case X86::PSLLDri:
690 return ProcessShiftLeftToAdd(X86::PADDDrr);
691 case X86::VPSLLDri:
692 return ProcessShiftLeftToAdd(X86::VPADDDrr);
693 case X86::VPSLLDYri:
694 return ProcessShiftLeftToAdd(X86::VPADDDYrr);
695 case X86::VPSLLDZ128ri:
696 return ProcessShiftLeftToAdd(X86::VPADDDZ128rr);
697 case X86::VPSLLDZ256ri:
698 return ProcessShiftLeftToAdd(X86::VPADDDZ256rr);
699 case X86::VPSLLDZri:
700 return ProcessShiftLeftToAdd(X86::VPADDDZrr);
701 case X86::PSLLQri:
702 return ProcessShiftLeftToAdd(X86::PADDQrr);
703 case X86::VPSLLQri:
704 return ProcessShiftLeftToAdd(X86::VPADDQrr);
705 case X86::VPSLLQYri:
706 return ProcessShiftLeftToAdd(X86::VPADDQYrr);
707 case X86::VPSLLQZ128ri:
708 return ProcessShiftLeftToAdd(X86::VPADDQZ128rr);
709 case X86::VPSLLQZ256ri:
710 return ProcessShiftLeftToAdd(X86::VPADDQZ256rr);
711 case X86::VPSLLQZri:
712 return ProcessShiftLeftToAdd(X86::VPADDQZrr);
713 case X86::MOVUPDrr:
714 return ProcessMOVPDToMOVPS(X86::MOVUPSrr);
715 case X86::MOVUPDrm:
716 return ProcessMOVPDToMOVPS(X86::MOVUPSrm);
717 case X86::MOVUPDmr:
718 return ProcessMOVPDToMOVPS(X86::MOVUPSmr);
719 case X86::MOVAPDrr:
720 return ProcessMOVPDToMOVPS(X86::MOVAPSrr);
721 case X86::MOVAPDrm:
722 return ProcessMOVPDToMOVPS(X86::MOVAPSrm);
723 case X86::MOVAPDmr:
724 return ProcessMOVPDToMOVPS(X86::MOVAPSmr);
725
726 default:
727 return false;
728 }
729}
730
731bool X86FixupInstTuningImpl::runOnMachineFunction(MachineFunction &MF) {
732 LLVM_DEBUG(dbgs() << "Start X86FixupInstTuning\n";);
733 bool Changed = false;
734 ST = &MF.getSubtarget<X86Subtarget>();
735 TII = ST->getInstrInfo();
736 TRI = ST->getRegisterInfo();
737 SM = &ST->getSchedModel();
738
739 for (MachineBasicBlock &MBB : MF) {
740 for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); ++I) {
741 if (processInstruction(MF, MBB, I)) {
742 ++NumInstChanges;
743 Changed = true;
744 }
745 }
746 }
747 LLVM_DEBUG(dbgs() << "End X86FixupInstTuning\n";);
748 return Changed;
749}
750
751bool X86FixupInstTuningLegacy::runOnMachineFunction(MachineFunction &MF) {
752 X86FixupInstTuningImpl Impl;
753 return Impl.runOnMachineFunction(MF);
754}
755
756PreservedAnalyses
757X86FixupInstTuningPass::run(MachineFunction &MF,
758 MachineFunctionAnalysisManager &MFAM) {
759 X86FixupInstTuningImpl Impl;
760 return Impl.runOnMachineFunction(MF)
761 ? getMachineFunctionPassPreservedAnalyses()
762 .preserveSet<CFGAnalyses>()
763 : PreservedAnalyses::all();
764}
765