1//===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the X86 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86InstrInfo.h"
14#include "X86.h"
15#include "X86InstrBuilder.h"
16#include "X86InstrFoldTables.h"
17#include "X86MachineFunctionInfo.h"
18#include "X86Subtarget.h"
19#include "X86TargetMachine.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/Sequence.h"
22#include "llvm/CodeGen/LiveIntervals.h"
23#include "llvm/CodeGen/LivePhysRegs.h"
24#include "llvm/CodeGen/LiveVariables.h"
25#include "llvm/CodeGen/MachineConstantPool.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineInstr.h"
28#include "llvm/CodeGen/MachineInstrBuilder.h"
29#include "llvm/CodeGen/MachineModuleInfo.h"
30#include "llvm/CodeGen/MachineOperand.h"
31#include "llvm/CodeGen/MachineRegisterInfo.h"
32#include "llvm/CodeGen/StackMaps.h"
33#include "llvm/IR/DebugInfoMetadata.h"
34#include "llvm/IR/DerivedTypes.h"
35#include "llvm/IR/Function.h"
36#include "llvm/IR/InstrTypes.h"
37#include "llvm/IR/Module.h"
38#include "llvm/MC/MCAsmInfo.h"
39#include "llvm/MC/MCExpr.h"
40#include "llvm/MC/MCInst.h"
41#include "llvm/Support/CommandLine.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/Support/raw_ostream.h"
46#include "llvm/Target/TargetOptions.h"
47#include <atomic>
48#include <optional>
49
50using namespace llvm;
51
52#define DEBUG_TYPE "x86-instr-info"
53
54#define GET_INSTRINFO_CTOR_DTOR
55#include "X86GenInstrInfo.inc"
56
57extern cl::opt<bool> X86EnableAPXForRelocation;
58
59static cl::opt<bool>
60 NoFusing("disable-spill-fusing",
61 cl::desc("Disable fusing of spill code into instructions"),
62 cl::Hidden);
63static cl::opt<bool>
64 PrintFailedFusing("print-failed-fuse-candidates",
65 cl::desc("Print instructions that the allocator wants to"
66 " fuse, but the X86 backend currently can't"),
67 cl::Hidden);
68static cl::opt<bool>
69 ReMatPICStubLoad("remat-pic-stub-load",
70 cl::desc("Re-materialize load from stub in PIC mode"),
71 cl::init(Val: false), cl::Hidden);
72static cl::opt<unsigned>
73 PartialRegUpdateClearance("partial-reg-update-clearance",
74 cl::desc("Clearance between two register writes "
75 "for inserting XOR to avoid partial "
76 "register update"),
77 cl::init(Val: 64), cl::Hidden);
78static cl::opt<unsigned> UndefRegClearance(
79 "undef-reg-clearance",
80 cl::desc("How many idle instructions we would like before "
81 "certain undef register reads"),
82 cl::init(Val: 128), cl::Hidden);
83
84static cl::opt<unsigned> MaxNFConversions(
85 "x86-max-nf-conversions-for-cmp-reuse",
86 cl::desc("Maximum number of NF conversions allowed to reuse EFLAGS from a "
87 "producer dominating a multi-predecessor block"),
88 cl::init(Val: 6), cl::Hidden);
89
90// Pin the vtable to this file.
91void X86InstrInfo::anchor() {}
92
93X86InstrInfo::X86InstrInfo(const X86Subtarget &STI)
94 : X86GenInstrInfo(STI, RI,
95 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64
96 : X86::ADJCALLSTACKDOWN32),
97 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64
98 : X86::ADJCALLSTACKUP32),
99 X86::CATCHRET, (STI.is64Bit() ? X86::RET64 : X86::RET32)),
100 Subtarget(STI), RI(STI.getTargetTriple()) {}
101
102const TargetRegisterClass *X86InstrInfo::getRegClass(const MCInstrDesc &MCID,
103 unsigned OpNum) const {
104 auto *RC = TargetInstrInfo::getRegClass(MCID, OpNum);
105 // If the target does not have egpr, then r16-r31 will be resereved for all
106 // instructions.
107 if (!RC || !Subtarget.hasEGPR())
108 return RC;
109
110 if (X86II::canUseApxExtendedReg(Desc: MCID))
111 return RC;
112
113 const X86RegisterInfo *RI = Subtarget.getRegisterInfo();
114 return RI->constrainRegClassToNonRex2(RC);
115}
116
117bool X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
118 Register &SrcReg, Register &DstReg,
119 unsigned &SubIdx) const {
120 switch (MI.getOpcode()) {
121 default:
122 break;
123 case X86::MOVSX16rr8:
124 case X86::MOVZX16rr8:
125 case X86::MOVSX32rr8:
126 case X86::MOVZX32rr8:
127 case X86::MOVSX64rr8:
128 if (!Subtarget.is64Bit())
129 // It's not always legal to reference the low 8-bit of the larger
130 // register in 32-bit mode.
131 return false;
132 [[fallthrough]];
133 case X86::MOVSX32rr16:
134 case X86::MOVZX32rr16:
135 case X86::MOVSX64rr16:
136 case X86::MOVSX64rr32: {
137 if (MI.getOperand(i: 0).getSubReg() || MI.getOperand(i: 1).getSubReg())
138 // Be conservative.
139 return false;
140 SrcReg = MI.getOperand(i: 1).getReg();
141 DstReg = MI.getOperand(i: 0).getReg();
142 switch (MI.getOpcode()) {
143 default:
144 llvm_unreachable("Unreachable!");
145 case X86::MOVSX16rr8:
146 case X86::MOVZX16rr8:
147 case X86::MOVSX32rr8:
148 case X86::MOVZX32rr8:
149 case X86::MOVSX64rr8:
150 SubIdx = X86::sub_8bit;
151 break;
152 case X86::MOVSX32rr16:
153 case X86::MOVZX32rr16:
154 case X86::MOVSX64rr16:
155 SubIdx = X86::sub_16bit;
156 break;
157 case X86::MOVSX64rr32:
158 SubIdx = X86::sub_32bit;
159 break;
160 }
161 return true;
162 }
163 }
164 return false;
165}
166
167bool X86InstrInfo::isDataInvariant(MachineInstr &MI) {
168 if (MI.mayLoad() || MI.mayStore())
169 return false;
170
171 // Some target-independent operations that trivially lower to data-invariant
172 // instructions.
173 if (MI.isCopyLike() || MI.isInsertSubreg())
174 return true;
175
176 unsigned Opcode = MI.getOpcode();
177 using namespace X86;
178 // On x86 it is believed that imul is constant time w.r.t. the loaded data.
179 // However, they set flags and are perhaps the most surprisingly constant
180 // time operations so we call them out here separately.
181 if (isIMUL(Opcode))
182 return true;
183 // Bit scanning and counting instructions that are somewhat surprisingly
184 // constant time as they scan across bits and do other fairly complex
185 // operations like popcnt, but are believed to be constant time on x86.
186 // However, these set flags.
187 if (isBSF(Opcode) || isBSR(Opcode) || isLZCNT(Opcode) || isPOPCNT(Opcode) ||
188 isTZCNT(Opcode))
189 return true;
190 // Bit manipulation instructions are effectively combinations of basic
191 // arithmetic ops, and should still execute in constant time. These also
192 // set flags.
193 if (isBLCFILL(Opcode) || isBLCI(Opcode) || isBLCIC(Opcode) ||
194 isBLCMSK(Opcode) || isBLCS(Opcode) || isBLSFILL(Opcode) ||
195 isBLSI(Opcode) || isBLSIC(Opcode) || isBLSMSK(Opcode) || isBLSR(Opcode) ||
196 isTZMSK(Opcode))
197 return true;
198 // Bit extracting and clearing instructions should execute in constant time,
199 // and set flags.
200 if (isBEXTR(Opcode) || isBZHI(Opcode))
201 return true;
202 // Shift and rotate.
203 if (isROL(Opcode) || isROR(Opcode) || isSAR(Opcode) || isSHL(Opcode) ||
204 isSHR(Opcode) || isSHLD(Opcode) || isSHRD(Opcode))
205 return true;
206 // Basic arithmetic is constant time on the input but does set flags.
207 if (isADC(Opcode) || isADD(Opcode) || isAND(Opcode) || isOR(Opcode) ||
208 isSBB(Opcode) || isSUB(Opcode) || isXOR(Opcode))
209 return true;
210 // Arithmetic with just 32-bit and 64-bit variants and no immediates.
211 if (isANDN(Opcode))
212 return true;
213 // Unary arithmetic operations.
214 if (isDEC(Opcode) || isINC(Opcode) || isNEG(Opcode))
215 return true;
216 // Unlike other arithmetic, NOT doesn't set EFLAGS.
217 if (isNOT(Opcode))
218 return true;
219 // Various move instructions used to zero or sign extend things. Note that we
220 // intentionally don't support the _NOREX variants as we can't handle that
221 // register constraint anyways.
222 if (isMOVSX(Opcode) || isMOVZX(Opcode) || isMOVSXD(Opcode) || isMOV(Opcode))
223 return true;
224 // Arithmetic instructions that are both constant time and don't set flags.
225 if (isRORX(Opcode) || isSARX(Opcode) || isSHLX(Opcode) || isSHRX(Opcode))
226 return true;
227 // LEA doesn't actually access memory, and its arithmetic is constant time.
228 if (isLEA(Opcode))
229 return true;
230 // By default, assume that the instruction is not data invariant.
231 return false;
232}
233
234bool X86InstrInfo::isDataInvariantLoad(MachineInstr &MI) {
235 switch (MI.getOpcode()) {
236 default:
237 // By default, assume that the load will immediately leak.
238 return false;
239
240 // On x86 it is believed that imul is constant time w.r.t. the loaded data.
241 // However, they set flags and are perhaps the most surprisingly constant
242 // time operations so we call them out here separately.
243 case X86::IMUL16rm:
244 case X86::IMUL16rmi:
245 case X86::IMUL32rm:
246 case X86::IMUL32rmi:
247 case X86::IMUL64rm:
248 case X86::IMUL64rmi32:
249
250 // Bit scanning and counting instructions that are somewhat surprisingly
251 // constant time as they scan across bits and do other fairly complex
252 // operations like popcnt, but are believed to be constant time on x86.
253 // However, these set flags.
254 case X86::BSF16rm:
255 case X86::BSF32rm:
256 case X86::BSF64rm:
257 case X86::BSR16rm:
258 case X86::BSR32rm:
259 case X86::BSR64rm:
260 case X86::LZCNT16rm:
261 case X86::LZCNT32rm:
262 case X86::LZCNT64rm:
263 case X86::POPCNT16rm:
264 case X86::POPCNT32rm:
265 case X86::POPCNT64rm:
266 case X86::TZCNT16rm:
267 case X86::TZCNT32rm:
268 case X86::TZCNT64rm:
269
270 // Bit manipulation instructions are effectively combinations of basic
271 // arithmetic ops, and should still execute in constant time. These also
272 // set flags.
273 case X86::BLCFILL32rm:
274 case X86::BLCFILL64rm:
275 case X86::BLCI32rm:
276 case X86::BLCI64rm:
277 case X86::BLCIC32rm:
278 case X86::BLCIC64rm:
279 case X86::BLCMSK32rm:
280 case X86::BLCMSK64rm:
281 case X86::BLCS32rm:
282 case X86::BLCS64rm:
283 case X86::BLSFILL32rm:
284 case X86::BLSFILL64rm:
285 case X86::BLSI32rm:
286 case X86::BLSI64rm:
287 case X86::BLSIC32rm:
288 case X86::BLSIC64rm:
289 case X86::BLSMSK32rm:
290 case X86::BLSMSK64rm:
291 case X86::BLSR32rm:
292 case X86::BLSR64rm:
293 case X86::TZMSK32rm:
294 case X86::TZMSK64rm:
295
296 // Bit extracting and clearing instructions should execute in constant time,
297 // and set flags.
298 case X86::BEXTR32rm:
299 case X86::BEXTR64rm:
300 case X86::BEXTRI32mi:
301 case X86::BEXTRI64mi:
302 case X86::BZHI32rm:
303 case X86::BZHI64rm:
304
305 // Basic arithmetic is constant time on the input but does set flags.
306 case X86::ADC8rm:
307 case X86::ADC16rm:
308 case X86::ADC32rm:
309 case X86::ADC64rm:
310 case X86::ADD8rm:
311 case X86::ADD16rm:
312 case X86::ADD32rm:
313 case X86::ADD64rm:
314 case X86::AND8rm:
315 case X86::AND16rm:
316 case X86::AND32rm:
317 case X86::AND64rm:
318 case X86::ANDN32rm:
319 case X86::ANDN64rm:
320 case X86::OR8rm:
321 case X86::OR16rm:
322 case X86::OR32rm:
323 case X86::OR64rm:
324 case X86::SBB8rm:
325 case X86::SBB16rm:
326 case X86::SBB32rm:
327 case X86::SBB64rm:
328 case X86::SUB8rm:
329 case X86::SUB16rm:
330 case X86::SUB32rm:
331 case X86::SUB64rm:
332 case X86::XOR8rm:
333 case X86::XOR16rm:
334 case X86::XOR32rm:
335 case X86::XOR64rm:
336
337 // Integer multiply w/o affecting flags is still believed to be constant
338 // time on x86. Called out separately as this is among the most surprising
339 // instructions to exhibit that behavior.
340 case X86::MULX32rm:
341 case X86::MULX64rm:
342
343 // Arithmetic instructions that are both constant time and don't set flags.
344 case X86::RORX32mi:
345 case X86::RORX64mi:
346 case X86::SARX32rm:
347 case X86::SARX64rm:
348 case X86::SHLX32rm:
349 case X86::SHLX64rm:
350 case X86::SHRX32rm:
351 case X86::SHRX64rm:
352
353 // Conversions are believed to be constant time and don't set flags.
354 case X86::CVTTSD2SI64rm:
355 case X86::VCVTTSD2SI64rm:
356 case X86::VCVTTSD2SI64Zrm:
357 case X86::CVTTSD2SIrm:
358 case X86::VCVTTSD2SIrm:
359 case X86::VCVTTSD2SIZrm:
360 case X86::CVTTSS2SI64rm:
361 case X86::VCVTTSS2SI64rm:
362 case X86::VCVTTSS2SI64Zrm:
363 case X86::CVTTSS2SIrm:
364 case X86::VCVTTSS2SIrm:
365 case X86::VCVTTSS2SIZrm:
366 case X86::CVTSI2SDrm:
367 case X86::VCVTSI2SDrm:
368 case X86::VCVTSI2SDZrm:
369 case X86::CVTSI2SSrm:
370 case X86::VCVTSI2SSrm:
371 case X86::VCVTSI2SSZrm:
372 case X86::CVTSI642SDrm:
373 case X86::VCVTSI642SDrm:
374 case X86::VCVTSI642SDZrm:
375 case X86::CVTSI642SSrm:
376 case X86::VCVTSI642SSrm:
377 case X86::VCVTSI642SSZrm:
378 case X86::CVTSS2SDrm:
379 case X86::VCVTSS2SDrm:
380 case X86::VCVTSS2SDZrm:
381 case X86::CVTSD2SSrm:
382 case X86::VCVTSD2SSrm:
383 case X86::VCVTSD2SSZrm:
384 // AVX512 added unsigned integer conversions.
385 case X86::VCVTTSD2USI64Zrm:
386 case X86::VCVTTSD2USIZrm:
387 case X86::VCVTTSS2USI64Zrm:
388 case X86::VCVTTSS2USIZrm:
389 case X86::VCVTUSI2SDZrm:
390 case X86::VCVTUSI642SDZrm:
391 case X86::VCVTUSI2SSZrm:
392 case X86::VCVTUSI642SSZrm:
393
394 // Loads to register don't set flags.
395 case X86::MOV8rm:
396 case X86::MOV8rm_NOREX:
397 case X86::MOV16rm:
398 case X86::MOV32rm:
399 case X86::MOV64rm:
400 case X86::MOVSX16rm8:
401 case X86::MOVSX32rm16:
402 case X86::MOVSX32rm8:
403 case X86::MOVSX32rm8_NOREX:
404 case X86::MOVSX64rm16:
405 case X86::MOVSX64rm32:
406 case X86::MOVSX64rm8:
407 case X86::MOVZX16rm8:
408 case X86::MOVZX32rm16:
409 case X86::MOVZX32rm8:
410 case X86::MOVZX32rm8_NOREX:
411 case X86::MOVZX64rm16:
412 case X86::MOVZX64rm8:
413 return true;
414 }
415}
416
417int X86InstrInfo::getSPAdjust(const MachineInstr &MI) const {
418 const MachineFunction *MF = MI.getParent()->getParent();
419 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
420
421 if (isFrameInstr(I: MI)) {
422 int SPAdj = alignTo(Size: getFrameSize(I: MI), A: TFI->getStackAlign());
423 SPAdj -= getFrameAdjustment(I: MI);
424 if (!isFrameSetup(I: MI))
425 SPAdj = -SPAdj;
426 return SPAdj;
427 }
428
429 // To know whether a call adjusts the stack, we need information
430 // that is bound to the following ADJCALLSTACKUP pseudo.
431 // Look for the next ADJCALLSTACKUP that follows the call.
432 if (MI.isCall()) {
433 const MachineBasicBlock *MBB = MI.getParent();
434 auto I = ++MachineBasicBlock::const_iterator(MI);
435 for (auto E = MBB->end(); I != E; ++I) {
436 if (I->getOpcode() == getCallFrameDestroyOpcode() || I->isCall())
437 break;
438 }
439
440 // If we could not find a frame destroy opcode, then it has already
441 // been simplified, so we don't care.
442 if (I->getOpcode() != getCallFrameDestroyOpcode())
443 return 0;
444
445 return -(I->getOperand(i: 1).getImm());
446 }
447
448 // Currently handle only PUSHes we can reasonably expect to see
449 // in call sequences
450 switch (MI.getOpcode()) {
451 default:
452 return 0;
453 case X86::PUSH32r:
454 case X86::PUSH32rmm:
455 case X86::PUSH32rmr:
456 case X86::PUSH32i:
457 return 4;
458 case X86::PUSH64r:
459 case X86::PUSH64rmm:
460 case X86::PUSH64rmr:
461 case X86::PUSH64i32:
462 return 8;
463 }
464}
465
466/// Return true and the FrameIndex if the specified
467/// operand and follow operands form a reference to the stack frame.
468bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op,
469 int &FrameIndex) const {
470 if (MI.getOperand(i: Op + X86::AddrBaseReg).isFI() &&
471 MI.getOperand(i: Op + X86::AddrScaleAmt).isImm() &&
472 MI.getOperand(i: Op + X86::AddrIndexReg).isReg() &&
473 MI.getOperand(i: Op + X86::AddrDisp).isImm() &&
474 MI.getOperand(i: Op + X86::AddrScaleAmt).getImm() == 1 &&
475 MI.getOperand(i: Op + X86::AddrIndexReg).getReg() == 0 &&
476 MI.getOperand(i: Op + X86::AddrDisp).getImm() == 0) {
477 FrameIndex = MI.getOperand(i: Op + X86::AddrBaseReg).getIndex();
478 return true;
479 }
480 return false;
481}
482
483static bool isFrameLoadOpcode(int Opcode, TypeSize &MemBytes) {
484 switch (Opcode) {
485 default:
486 return false;
487 case X86::MOV8rm:
488 case X86::KMOVBkm:
489 case X86::KMOVBkm_EVEX:
490 MemBytes = TypeSize::getFixed(ExactSize: 1);
491 return true;
492 case X86::MOV16rm:
493 case X86::KMOVWkm:
494 case X86::KMOVWkm_EVEX:
495 case X86::VMOVSHZrm:
496 case X86::VMOVSHZrm_alt:
497 MemBytes = TypeSize::getFixed(ExactSize: 2);
498 return true;
499 case X86::MOV32rm:
500 case X86::MOVSSrm:
501 case X86::MOVSSrm_alt:
502 case X86::VMOVSSrm:
503 case X86::VMOVSSrm_alt:
504 case X86::VMOVSSZrm:
505 case X86::VMOVSSZrm_alt:
506 case X86::KMOVDkm:
507 case X86::KMOVDkm_EVEX:
508 MemBytes = TypeSize::getFixed(ExactSize: 4);
509 return true;
510 case X86::MOV64rm:
511 case X86::LD_Fp64m:
512 case X86::MOVSDrm:
513 case X86::MOVSDrm_alt:
514 case X86::VMOVSDrm:
515 case X86::VMOVSDrm_alt:
516 case X86::VMOVSDZrm:
517 case X86::VMOVSDZrm_alt:
518 case X86::MMX_MOVD64rm:
519 case X86::MMX_MOVQ64rm:
520 case X86::KMOVQkm:
521 case X86::KMOVQkm_EVEX:
522 MemBytes = TypeSize::getFixed(ExactSize: 8);
523 return true;
524 case X86::MOVAPSrm:
525 case X86::MOVUPSrm:
526 case X86::MOVAPDrm:
527 case X86::MOVUPDrm:
528 case X86::MOVDQArm:
529 case X86::MOVDQUrm:
530 case X86::VMOVAPSrm:
531 case X86::VMOVUPSrm:
532 case X86::VMOVAPDrm:
533 case X86::VMOVUPDrm:
534 case X86::VMOVDQArm:
535 case X86::VMOVDQUrm:
536 case X86::VMOVAPSZ128rm:
537 case X86::VMOVUPSZ128rm:
538 case X86::VMOVAPSZ128rm_NOVLX:
539 case X86::VMOVUPSZ128rm_NOVLX:
540 case X86::VMOVAPDZ128rm:
541 case X86::VMOVUPDZ128rm:
542 case X86::VMOVDQU8Z128rm:
543 case X86::VMOVDQU16Z128rm:
544 case X86::VMOVDQA32Z128rm:
545 case X86::VMOVDQU32Z128rm:
546 case X86::VMOVDQA64Z128rm:
547 case X86::VMOVDQU64Z128rm:
548 MemBytes = TypeSize::getFixed(ExactSize: 16);
549 return true;
550 case X86::VMOVAPSYrm:
551 case X86::VMOVUPSYrm:
552 case X86::VMOVAPDYrm:
553 case X86::VMOVUPDYrm:
554 case X86::VMOVDQAYrm:
555 case X86::VMOVDQUYrm:
556 case X86::VMOVAPSZ256rm:
557 case X86::VMOVUPSZ256rm:
558 case X86::VMOVAPSZ256rm_NOVLX:
559 case X86::VMOVUPSZ256rm_NOVLX:
560 case X86::VMOVAPDZ256rm:
561 case X86::VMOVUPDZ256rm:
562 case X86::VMOVDQU8Z256rm:
563 case X86::VMOVDQU16Z256rm:
564 case X86::VMOVDQA32Z256rm:
565 case X86::VMOVDQU32Z256rm:
566 case X86::VMOVDQA64Z256rm:
567 case X86::VMOVDQU64Z256rm:
568 MemBytes = TypeSize::getFixed(ExactSize: 32);
569 return true;
570 case X86::VMOVAPSZrm:
571 case X86::VMOVUPSZrm:
572 case X86::VMOVAPDZrm:
573 case X86::VMOVUPDZrm:
574 case X86::VMOVDQU8Zrm:
575 case X86::VMOVDQU16Zrm:
576 case X86::VMOVDQA32Zrm:
577 case X86::VMOVDQU32Zrm:
578 case X86::VMOVDQA64Zrm:
579 case X86::VMOVDQU64Zrm:
580 MemBytes = TypeSize::getFixed(ExactSize: 64);
581 return true;
582 }
583}
584
585static bool isFrameStoreOpcode(int Opcode, TypeSize &MemBytes) {
586 switch (Opcode) {
587 default:
588 return false;
589 case X86::MOV8mr:
590 case X86::KMOVBmk:
591 case X86::KMOVBmk_EVEX:
592 MemBytes = TypeSize::getFixed(ExactSize: 1);
593 return true;
594 case X86::MOV16mr:
595 case X86::KMOVWmk:
596 case X86::KMOVWmk_EVEX:
597 case X86::VMOVSHZmr:
598 MemBytes = TypeSize::getFixed(ExactSize: 2);
599 return true;
600 case X86::MOV32mr:
601 case X86::MOVSSmr:
602 case X86::VMOVSSmr:
603 case X86::VMOVSSZmr:
604 case X86::KMOVDmk:
605 case X86::KMOVDmk_EVEX:
606 MemBytes = TypeSize::getFixed(ExactSize: 4);
607 return true;
608 case X86::MOV64mr:
609 case X86::ST_FpP64m:
610 case X86::MOVSDmr:
611 case X86::VMOVSDmr:
612 case X86::VMOVSDZmr:
613 case X86::MMX_MOVD64mr:
614 case X86::MMX_MOVQ64mr:
615 case X86::MMX_MOVNTQmr:
616 case X86::KMOVQmk:
617 case X86::KMOVQmk_EVEX:
618 MemBytes = TypeSize::getFixed(ExactSize: 8);
619 return true;
620 case X86::MOVAPSmr:
621 case X86::MOVUPSmr:
622 case X86::MOVAPDmr:
623 case X86::MOVUPDmr:
624 case X86::MOVDQAmr:
625 case X86::MOVDQUmr:
626 case X86::VMOVAPSmr:
627 case X86::VMOVUPSmr:
628 case X86::VMOVAPDmr:
629 case X86::VMOVUPDmr:
630 case X86::VMOVDQAmr:
631 case X86::VMOVDQUmr:
632 case X86::VMOVUPSZ128mr:
633 case X86::VMOVAPSZ128mr:
634 case X86::VMOVUPSZ128mr_NOVLX:
635 case X86::VMOVAPSZ128mr_NOVLX:
636 case X86::VMOVUPDZ128mr:
637 case X86::VMOVAPDZ128mr:
638 case X86::VMOVDQA32Z128mr:
639 case X86::VMOVDQU32Z128mr:
640 case X86::VMOVDQA64Z128mr:
641 case X86::VMOVDQU64Z128mr:
642 case X86::VMOVDQU8Z128mr:
643 case X86::VMOVDQU16Z128mr:
644 MemBytes = TypeSize::getFixed(ExactSize: 16);
645 return true;
646 case X86::VMOVUPSYmr:
647 case X86::VMOVAPSYmr:
648 case X86::VMOVUPDYmr:
649 case X86::VMOVAPDYmr:
650 case X86::VMOVDQUYmr:
651 case X86::VMOVDQAYmr:
652 case X86::VMOVUPSZ256mr:
653 case X86::VMOVAPSZ256mr:
654 case X86::VMOVUPSZ256mr_NOVLX:
655 case X86::VMOVAPSZ256mr_NOVLX:
656 case X86::VMOVUPDZ256mr:
657 case X86::VMOVAPDZ256mr:
658 case X86::VMOVDQU8Z256mr:
659 case X86::VMOVDQU16Z256mr:
660 case X86::VMOVDQA32Z256mr:
661 case X86::VMOVDQU32Z256mr:
662 case X86::VMOVDQA64Z256mr:
663 case X86::VMOVDQU64Z256mr:
664 MemBytes = TypeSize::getFixed(ExactSize: 32);
665 return true;
666 case X86::VMOVUPSZmr:
667 case X86::VMOVAPSZmr:
668 case X86::VMOVUPDZmr:
669 case X86::VMOVAPDZmr:
670 case X86::VMOVDQU8Zmr:
671 case X86::VMOVDQU16Zmr:
672 case X86::VMOVDQA32Zmr:
673 case X86::VMOVDQU32Zmr:
674 case X86::VMOVDQA64Zmr:
675 case X86::VMOVDQU64Zmr:
676 MemBytes = TypeSize::getFixed(ExactSize: 64);
677 return true;
678 }
679 return false;
680}
681
682Register X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
683 int &FrameIndex) const {
684 TypeSize Dummy = TypeSize::getZero();
685 return X86InstrInfo::isLoadFromStackSlot(MI, FrameIndex, MemBytes&: Dummy);
686}
687
688Register X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
689 int &FrameIndex,
690 TypeSize &MemBytes) const {
691 if (isFrameLoadOpcode(Opcode: MI.getOpcode(), MemBytes))
692 if (MI.getOperand(i: 0).getSubReg() == 0 && isFrameOperand(MI, Op: 1, FrameIndex))
693 return MI.getOperand(i: 0).getReg();
694 return Register();
695}
696
697Register X86InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
698 int &FrameIndex) const {
699 TypeSize Dummy = TypeSize::getZero();
700 if (isFrameLoadOpcode(Opcode: MI.getOpcode(), MemBytes&: Dummy)) {
701 if (Register Reg = isLoadFromStackSlot(MI, FrameIndex))
702 return Reg;
703 // Check for post-frame index elimination operations
704 SmallVector<const MachineMemOperand *, 1> Accesses;
705 if (hasLoadFromStackSlot(MI, Accesses)) {
706 FrameIndex =
707 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
708 ->getFrameIndex();
709 return MI.getOperand(i: 0).getReg();
710 }
711 }
712 return Register();
713}
714
715Register X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
716 int &FrameIndex) const {
717 TypeSize Dummy = TypeSize::getZero();
718 return X86InstrInfo::isStoreToStackSlot(MI, FrameIndex, MemBytes&: Dummy);
719}
720
721Register X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
722 int &FrameIndex,
723 TypeSize &MemBytes) const {
724 if (isFrameStoreOpcode(Opcode: MI.getOpcode(), MemBytes))
725 if (MI.getOperand(i: X86::AddrNumOperands).getSubReg() == 0 &&
726 isFrameOperand(MI, Op: 0, FrameIndex))
727 return MI.getOperand(i: X86::AddrNumOperands).getReg();
728 return Register();
729}
730
731Register X86InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
732 int &FrameIndex) const {
733 TypeSize Dummy = TypeSize::getZero();
734 if (isFrameStoreOpcode(Opcode: MI.getOpcode(), MemBytes&: Dummy)) {
735 if (Register Reg = isStoreToStackSlot(MI, FrameIndex))
736 return Reg;
737 // Check for post-frame index elimination operations
738 SmallVector<const MachineMemOperand *, 1> Accesses;
739 if (hasStoreToStackSlot(MI, Accesses)) {
740 FrameIndex =
741 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
742 ->getFrameIndex();
743 return MI.getOperand(i: X86::AddrNumOperands).getReg();
744 }
745 }
746 return Register();
747}
748
749/// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r.
750static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI) {
751 // Don't waste compile time scanning use-def chains of physregs.
752 if (!BaseReg.isVirtual())
753 return false;
754 bool isPICBase = false;
755 for (const MachineInstr &DefMI : MRI.def_instructions(Reg: BaseReg)) {
756 if (DefMI.getOpcode() != X86::MOVPC32r)
757 return false;
758 assert(!isPICBase && "More than one PIC base?");
759 isPICBase = true;
760 }
761 return isPICBase;
762}
763
764bool X86InstrInfo::isReMaterializableImpl(
765 const MachineInstr &MI) const {
766 switch (MI.getOpcode()) {
767 default:
768 // This function should only be called for opcodes with the ReMaterializable
769 // flag set.
770 llvm_unreachable("Unknown rematerializable operation!");
771 break;
772 case X86::IMPLICIT_DEF:
773 // Defer to generic logic.
774 break;
775 case X86::LOAD_STACK_GUARD:
776 case X86::LD_Fp032:
777 case X86::LD_Fp064:
778 case X86::LD_Fp080:
779 case X86::LD_Fp132:
780 case X86::LD_Fp164:
781 case X86::LD_Fp180:
782 case X86::AVX1_SETALLONES:
783 case X86::AVX2_SETALLONES:
784 case X86::AVX512_128_SET0:
785 case X86::AVX512_256_SET0:
786 case X86::AVX512_512_SET0:
787 case X86::AVX512_128_SETALLONES:
788 case X86::AVX512_256_SETALLONES:
789 case X86::AVX512_512_SETALLONES:
790 case X86::AVX512_FsFLD0SD:
791 case X86::AVX512_FsFLD0SH:
792 case X86::AVX512_FsFLD0SS:
793 case X86::AVX512_FsFLD0F128:
794 case X86::AVX_SET0:
795 case X86::FsFLD0SD:
796 case X86::FsFLD0SS:
797 case X86::FsFLD0SH:
798 case X86::FsFLD0F128:
799 case X86::KSET0B:
800 case X86::KSET0D:
801 case X86::KSET0Q:
802 case X86::KSET0W:
803 case X86::KSET1B:
804 case X86::KSET1D:
805 case X86::KSET1Q:
806 case X86::KSET1W:
807 case X86::MMX_SET0:
808 case X86::MOV32ImmSExti8:
809 case X86::MOV32r0:
810 case X86::MOV32r1:
811 case X86::MOV32r_1:
812 case X86::MOV32ri64:
813 case X86::MOV64ImmSExti8:
814 case X86::V_SET0:
815 case X86::V_SETALLONES:
816 case X86::MOV16ri:
817 case X86::MOV32ri:
818 case X86::MOV64ri:
819 case X86::MOV64ri32:
820 case X86::MOV8ri:
821 case X86::PTILEZEROV:
822 return true;
823
824 case X86::MOV8rm:
825 case X86::MOV8rm_NOREX:
826 case X86::MOV16rm:
827 case X86::MOV32rm:
828 case X86::MOV64rm:
829 case X86::MOVSSrm:
830 case X86::MOVSSrm_alt:
831 case X86::MOVSDrm:
832 case X86::MOVSDrm_alt:
833 case X86::MOVAPSrm:
834 case X86::MOVUPSrm:
835 case X86::MOVAPDrm:
836 case X86::MOVUPDrm:
837 case X86::MOVDQArm:
838 case X86::MOVDQUrm:
839 case X86::VMOVSSrm:
840 case X86::VMOVSSrm_alt:
841 case X86::VMOVSDrm:
842 case X86::VMOVSDrm_alt:
843 case X86::VMOVAPSrm:
844 case X86::VMOVUPSrm:
845 case X86::VMOVAPDrm:
846 case X86::VMOVUPDrm:
847 case X86::VMOVDQArm:
848 case X86::VMOVDQUrm:
849 case X86::VMOVAPSYrm:
850 case X86::VMOVUPSYrm:
851 case X86::VMOVAPDYrm:
852 case X86::VMOVUPDYrm:
853 case X86::VMOVDQAYrm:
854 case X86::VMOVDQUYrm:
855 case X86::MMX_MOVD64rm:
856 case X86::MMX_MOVQ64rm:
857 case X86::VBROADCASTSSrm:
858 case X86::VBROADCASTSSYrm:
859 case X86::VBROADCASTSDYrm:
860 // AVX-512
861 case X86::VPBROADCASTBZ128rm:
862 case X86::VPBROADCASTBZ256rm:
863 case X86::VPBROADCASTBZrm:
864 case X86::VBROADCASTF32X2Z256rm:
865 case X86::VBROADCASTF32X2Zrm:
866 case X86::VBROADCASTI32X2Z128rm:
867 case X86::VBROADCASTI32X2Z256rm:
868 case X86::VBROADCASTI32X2Zrm:
869 case X86::VPBROADCASTWZ128rm:
870 case X86::VPBROADCASTWZ256rm:
871 case X86::VPBROADCASTWZrm:
872 case X86::VPBROADCASTDZ128rm:
873 case X86::VPBROADCASTDZ256rm:
874 case X86::VPBROADCASTDZrm:
875 case X86::VBROADCASTSSZ128rm:
876 case X86::VBROADCASTSSZ256rm:
877 case X86::VBROADCASTSSZrm:
878 case X86::VPBROADCASTQZ128rm:
879 case X86::VPBROADCASTQZ256rm:
880 case X86::VPBROADCASTQZrm:
881 case X86::VBROADCASTSDZ256rm:
882 case X86::VBROADCASTSDZrm:
883 case X86::VMOVSSZrm:
884 case X86::VMOVSSZrm_alt:
885 case X86::VMOVSDZrm:
886 case X86::VMOVSDZrm_alt:
887 case X86::VMOVSHZrm:
888 case X86::VMOVSHZrm_alt:
889 case X86::VMOVAPDZ128rm:
890 case X86::VMOVAPDZ256rm:
891 case X86::VMOVAPDZrm:
892 case X86::VMOVAPSZ128rm:
893 case X86::VMOVAPSZ256rm:
894 case X86::VMOVAPSZ128rm_NOVLX:
895 case X86::VMOVAPSZ256rm_NOVLX:
896 case X86::VMOVAPSZrm:
897 case X86::VMOVDQA32Z128rm:
898 case X86::VMOVDQA32Z256rm:
899 case X86::VMOVDQA32Zrm:
900 case X86::VMOVDQA64Z128rm:
901 case X86::VMOVDQA64Z256rm:
902 case X86::VMOVDQA64Zrm:
903 case X86::VMOVDQU16Z128rm:
904 case X86::VMOVDQU16Z256rm:
905 case X86::VMOVDQU16Zrm:
906 case X86::VMOVDQU32Z128rm:
907 case X86::VMOVDQU32Z256rm:
908 case X86::VMOVDQU32Zrm:
909 case X86::VMOVDQU64Z128rm:
910 case X86::VMOVDQU64Z256rm:
911 case X86::VMOVDQU64Zrm:
912 case X86::VMOVDQU8Z128rm:
913 case X86::VMOVDQU8Z256rm:
914 case X86::VMOVDQU8Zrm:
915 case X86::VMOVUPDZ128rm:
916 case X86::VMOVUPDZ256rm:
917 case X86::VMOVUPDZrm:
918 case X86::VMOVUPSZ128rm:
919 case X86::VMOVUPSZ256rm:
920 case X86::VMOVUPSZ128rm_NOVLX:
921 case X86::VMOVUPSZ256rm_NOVLX:
922 case X86::VMOVUPSZrm: {
923 // Loads from constant pools are trivially rematerializable.
924 if (MI.getOperand(i: 1 + X86::AddrBaseReg).isReg() &&
925 MI.getOperand(i: 1 + X86::AddrScaleAmt).isImm() &&
926 MI.getOperand(i: 1 + X86::AddrIndexReg).isReg() &&
927 MI.getOperand(i: 1 + X86::AddrIndexReg).getReg() == 0 &&
928 MI.isDereferenceableInvariantLoad()) {
929 Register BaseReg = MI.getOperand(i: 1 + X86::AddrBaseReg).getReg();
930 if (BaseReg == 0 || BaseReg == X86::RIP)
931 return true;
932 // Allow re-materialization of PIC load.
933 if (!(!ReMatPICStubLoad && MI.getOperand(i: 1 + X86::AddrDisp).isGlobal())) {
934 const MachineFunction &MF = *MI.getParent()->getParent();
935 const MachineRegisterInfo &MRI = MF.getRegInfo();
936 if (regIsPICBase(BaseReg, MRI))
937 return true;
938 }
939 }
940 break;
941 }
942
943 case X86::LEA32r:
944 case X86::LEA64r: {
945 if (MI.getOperand(i: 1 + X86::AddrScaleAmt).isImm() &&
946 MI.getOperand(i: 1 + X86::AddrIndexReg).isReg() &&
947 MI.getOperand(i: 1 + X86::AddrIndexReg).getReg() == 0 &&
948 !MI.getOperand(i: 1 + X86::AddrDisp).isReg()) {
949 // lea fi#, lea GV, etc. are all rematerializable.
950 if (!MI.getOperand(i: 1 + X86::AddrBaseReg).isReg())
951 return true;
952 Register BaseReg = MI.getOperand(i: 1 + X86::AddrBaseReg).getReg();
953 if (BaseReg == 0)
954 return true;
955 // Allow re-materialization of lea PICBase + x.
956 const MachineFunction &MF = *MI.getParent()->getParent();
957 const MachineRegisterInfo &MRI = MF.getRegInfo();
958 if (regIsPICBase(BaseReg, MRI))
959 return true;
960 }
961 break;
962 }
963 }
964 return TargetInstrInfo::isReMaterializableImpl(MI);
965}
966
967void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB,
968 MachineBasicBlock::iterator I,
969 Register DestReg, unsigned SubIdx,
970 const MachineInstr &Orig,
971 LaneBitmask UsedLanes) const {
972 bool ClobbersEFLAGS = Orig.modifiesRegister(Reg: X86::EFLAGS, TRI: &TRI);
973 if (ClobbersEFLAGS && MBB.computeRegisterLiveness(TRI: &TRI, Reg: X86::EFLAGS, Before: I) !=
974 MachineBasicBlock::LQR_Dead) {
975 // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side
976 // effects.
977 int Value;
978 switch (Orig.getOpcode()) {
979 case X86::MOV32r0:
980 Value = 0;
981 break;
982 case X86::MOV32r1:
983 Value = 1;
984 break;
985 case X86::MOV32r_1:
986 Value = -1;
987 break;
988 default:
989 llvm_unreachable("Unexpected instruction!");
990 }
991
992 const DebugLoc &DL = Orig.getDebugLoc();
993 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: X86::MOV32ri))
994 .add(MO: Orig.getOperand(i: 0))
995 .addImm(Val: Value);
996 } else {
997 MachineInstr *MI = MBB.getParent()->CloneMachineInstr(Orig: &Orig);
998 MBB.insert(I, MI);
999 }
1000
1001 MachineInstr &NewMI = *std::prev(x: I);
1002 NewMI.substituteRegister(FromReg: Orig.getOperand(i: 0).getReg(), ToReg: DestReg, SubIdx, RegInfo: TRI);
1003}
1004
1005/// True if MI has a condition code def, e.g. EFLAGS, that is not marked dead.
1006bool X86InstrInfo::hasLiveCondCodeDef(MachineInstr &MI) const {
1007 for (const MachineOperand &MO : MI.operands()) {
1008 if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS &&
1009 !MO.isDead()) {
1010 return true;
1011 }
1012 }
1013 return false;
1014}
1015
1016/// Check whether the shift count for a machine operand is non-zero.
1017inline static unsigned getTruncatedShiftCount(const MachineInstr &MI,
1018 unsigned ShiftAmtOperandIdx) {
1019 // The shift count is six bits with the REX.W prefix and five bits without.
1020 unsigned ShiftCountMask = (MI.getDesc().TSFlags & X86II::REX_W) ? 63 : 31;
1021 unsigned Imm = MI.getOperand(i: ShiftAmtOperandIdx).getImm();
1022 return Imm & ShiftCountMask;
1023}
1024
1025/// Check whether the given shift count is appropriate
1026/// can be represented by a LEA instruction.
1027inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) {
1028 // Left shift instructions can be transformed into load-effective-address
1029 // instructions if we can encode them appropriately.
1030 // A LEA instruction utilizes a SIB byte to encode its scale factor.
1031 // The SIB.scale field is two bits wide which means that we can encode any
1032 // shift amount less than 4.
1033 return ShAmt < 4 && ShAmt > 0;
1034}
1035
1036static bool
1037findRedundantFlagInstr(MachineInstr &CmpInstr, MachineInstr &CmpValDefInstr,
1038 const MachineRegisterInfo *MRI, MachineInstr **AndInstr,
1039 const TargetRegisterInfo *TRI, const X86Subtarget &ST,
1040 bool &NoSignFlag, bool &ClearsOverflowFlag) {
1041 if (!(CmpValDefInstr.getOpcode() == X86::SUBREG_TO_REG &&
1042 CmpInstr.getOpcode() == X86::TEST64rr) &&
1043 !(CmpValDefInstr.getOpcode() == X86::COPY &&
1044 CmpInstr.getOpcode() == X86::TEST16rr))
1045 return false;
1046
1047 // CmpInstr is a TEST16rr/TEST64rr instruction, and
1048 // `X86InstrInfo::analyzeCompare` guarantees that it's analyzable only if two
1049 // registers are identical.
1050 assert((CmpInstr.getOperand(0).getReg() == CmpInstr.getOperand(1).getReg()) &&
1051 "CmpInstr is an analyzable TEST16rr/TEST64rr, and "
1052 "`X86InstrInfo::analyzeCompare` requires two reg operands are the"
1053 "same.");
1054
1055 // Caller (`X86InstrInfo::optimizeCompareInstr`) guarantees that
1056 // `CmpValDefInstr` defines the value that's used by `CmpInstr`; in this case
1057 // if `CmpValDefInstr` sets the EFLAGS, it is likely that `CmpInstr` is
1058 // redundant.
1059 assert(
1060 (MRI->getVRegDef(CmpInstr.getOperand(0).getReg()) == &CmpValDefInstr) &&
1061 "Caller guarantees that TEST64rr is a user of SUBREG_TO_REG or TEST16rr "
1062 "is a user of COPY sub16bit.");
1063 MachineInstr *VregDefInstr = nullptr;
1064 if (CmpInstr.getOpcode() == X86::TEST16rr) {
1065 if (!CmpValDefInstr.getOperand(i: 1).getReg().isVirtual())
1066 return false;
1067 VregDefInstr = MRI->getVRegDef(Reg: CmpValDefInstr.getOperand(i: 1).getReg());
1068 if (!VregDefInstr)
1069 return false;
1070 // We can only remove test when AND32ri or AND64ri32 whose imm can fit 16bit
1071 // size, others 32/64 bit ops would test higher bits which test16rr don't
1072 // want to.
1073 if (!((VregDefInstr->getOpcode() == X86::AND32ri ||
1074 VregDefInstr->getOpcode() == X86::AND64ri32) &&
1075 isUInt<16>(x: VregDefInstr->getOperand(i: 2).getImm())))
1076 return false;
1077 }
1078
1079 if (CmpInstr.getOpcode() == X86::TEST64rr) {
1080 // As seen in X86 td files, CmpValDefInstr.getOperand(3) is typically
1081 // sub_32bit or sub_xmm.
1082 if (CmpValDefInstr.getOperand(i: 2).getImm() != X86::sub_32bit)
1083 return false;
1084
1085 VregDefInstr = MRI->getVRegDef(Reg: CmpValDefInstr.getOperand(i: 1).getReg());
1086 }
1087
1088 assert(VregDefInstr && "Must have a definition (SSA)");
1089
1090 // Requires `CmpValDefInstr` and `VregDefInstr` are from the same MBB
1091 // to simplify the subsequent analysis.
1092 //
1093 // FIXME: If `VregDefInstr->getParent()` is the only predecessor of
1094 // `CmpValDefInstr.getParent()`, this could be handled.
1095 if (VregDefInstr->getParent() != CmpValDefInstr.getParent())
1096 return false;
1097
1098 if (X86::isAND(Opcode: VregDefInstr->getOpcode()) &&
1099 (!ST.hasNF() || VregDefInstr->modifiesRegister(Reg: X86::EFLAGS, TRI))) {
1100 // Get a sequence of instructions like
1101 // %reg = and* ... // Set EFLAGS
1102 // ... // EFLAGS not changed
1103 // %extended_reg = subreg_to_reg %reg, %subreg.sub_32bit
1104 // test64rr %extended_reg, %extended_reg, implicit-def $eflags
1105 // or
1106 // %reg = and32* ...
1107 // ... // EFLAGS not changed.
1108 // %src_reg = copy %reg.sub_16bit:gr32
1109 // test16rr %src_reg, %src_reg, implicit-def $eflags
1110 //
1111 // If subsequent readers use a subset of bits that don't change
1112 // after `and*` instructions, it's likely that the test64rr could
1113 // be optimized away.
1114 for (const MachineInstr &Instr :
1115 make_range(x: std::next(x: MachineBasicBlock::iterator(VregDefInstr)),
1116 y: MachineBasicBlock::iterator(CmpValDefInstr))) {
1117 // There are instructions between 'VregDefInstr' and
1118 // 'CmpValDefInstr' that modifies EFLAGS.
1119 if (Instr.modifiesRegister(Reg: X86::EFLAGS, TRI))
1120 return false;
1121 }
1122
1123 *AndInstr = VregDefInstr;
1124
1125 // AND instruction will essentially update SF and clear OF, so
1126 // NoSignFlag should be false in the sense that SF is modified by `AND`.
1127 //
1128 // However, the implementation artifically sets `NoSignFlag` to true
1129 // to poison the SF bit; that is to say, if SF is looked at later, the
1130 // optimization (to erase TEST64rr) will be disabled.
1131 //
1132 // The reason to poison SF bit is that SF bit value could be different
1133 // in the `AND` and `TEST` operation; signed bit is not known for `AND`,
1134 // and is known to be 0 as a result of `TEST64rr`.
1135 //
1136 // FIXME: As opposed to poisoning the SF bit directly, consider peeking into
1137 // the AND instruction and using the static information to guide peephole
1138 // optimization if possible. For example, it's possible to fold a
1139 // conditional move into a copy if the relevant EFLAG bits could be deduced
1140 // from an immediate operand of and operation.
1141 //
1142 NoSignFlag = true;
1143 // ClearsOverflowFlag is true for AND operation (no surprise).
1144 ClearsOverflowFlag = true;
1145 return true;
1146 }
1147 return false;
1148}
1149
1150bool X86InstrInfo::classifyLEAReg(MachineInstr &MI, const MachineOperand &Src,
1151 unsigned Opc, bool AllowSP, Register &NewSrc,
1152 unsigned &NewSrcSubReg, bool &isKill,
1153 MachineOperand &ImplicitOp, LiveVariables *LV,
1154 LiveIntervals *LIS) const {
1155 MachineFunction &MF = *MI.getParent()->getParent();
1156 const TargetRegisterClass *RC;
1157 if (AllowSP) {
1158 RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass;
1159 } else {
1160 RC = Opc != X86::LEA32r ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass;
1161 }
1162 Register SrcReg = Src.getReg();
1163 unsigned SubReg = Src.getSubReg();
1164 isKill = MI.killsRegister(Reg: SrcReg, /*TRI=*/nullptr);
1165
1166 NewSrcSubReg = X86::NoSubRegister;
1167
1168 // For both LEA64 and LEA32 the register already has essentially the right
1169 // type (32-bit or 64-bit) we may just need to forbid SP.
1170 if (Opc != X86::LEA64_32r) {
1171 NewSrc = SrcReg;
1172 NewSrcSubReg = SubReg;
1173 assert(!Src.isUndef() && "Undef op doesn't need optimization");
1174
1175 if (NewSrc.isVirtual() && !MF.getRegInfo().constrainRegClass(Reg: NewSrc, RC))
1176 return false;
1177
1178 return true;
1179 }
1180
1181 // This is for an LEA64_32r and incoming registers are 32-bit. One way or
1182 // another we need to add 64-bit registers to the final MI.
1183 if (SrcReg.isPhysical()) {
1184 ImplicitOp = Src;
1185 ImplicitOp.setImplicit();
1186
1187 NewSrc = getX86SubSuperRegister(Reg: SrcReg, Size: 64);
1188 assert(!SubReg && "no superregister for source");
1189 assert(NewSrc.isValid() && "Invalid Operand");
1190 assert(!Src.isUndef() && "Undef op doesn't need optimization");
1191 } else {
1192 // Virtual register of the wrong class, we have to create a temporary 64-bit
1193 // vreg to feed into the LEA.
1194 NewSrc = MF.getRegInfo().createVirtualRegister(RegClass: RC);
1195 NewSrcSubReg = X86::NoSubRegister;
1196 MachineInstr *Copy =
1197 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: TargetOpcode::COPY))
1198 .addReg(RegNo: NewSrc, Flags: RegState::Define | RegState::Undef, SubReg: X86::sub_32bit)
1199 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill), SubReg);
1200
1201 // Which is obviously going to be dead after we're done with it.
1202 isKill = true;
1203
1204 if (LV)
1205 LV->replaceKillInstruction(Reg: SrcReg, OldMI&: MI, NewMI&: *Copy);
1206
1207 if (LIS) {
1208 SlotIndex CopyIdx = LIS->InsertMachineInstrInMaps(MI&: *Copy);
1209 SlotIndex Idx = LIS->getInstructionIndex(Instr: MI);
1210 LiveInterval &LI = LIS->getInterval(Reg: SrcReg);
1211 LiveRange::Segment *S = LI.getSegmentContaining(Idx);
1212 if (S->end.getBaseIndex() == Idx)
1213 S->end = CopyIdx.getRegSlot();
1214 }
1215 }
1216
1217 // We've set all the parameters without issue.
1218 return true;
1219}
1220
1221MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc,
1222 MachineInstr &MI,
1223 LiveVariables *LV,
1224 LiveIntervals *LIS,
1225 bool Is8BitOp) const {
1226 // We handle 8-bit adds and various 16-bit opcodes in the switch below.
1227 MachineBasicBlock &MBB = *MI.getParent();
1228 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
1229 assert((Is8BitOp ||
1230 RegInfo.getTargetRegisterInfo()->getRegSizeInBits(
1231 *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) &&
1232 "Unexpected type for LEA transform");
1233
1234 // TODO: For a 32-bit target, we need to adjust the LEA variables with
1235 // something like this:
1236 // Opcode = X86::LEA32r;
1237 // InRegLEA = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1238 // OutRegLEA =
1239 // Is8BitOp ? RegInfo.createVirtualRegister(&X86::GR32ABCD_RegClass)
1240 // : RegInfo.createVirtualRegister(&X86::GR32RegClass);
1241 if (!Subtarget.is64Bit())
1242 return nullptr;
1243
1244 unsigned Opcode = X86::LEA64_32r;
1245 Register InRegLEA = RegInfo.createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
1246 Register OutRegLEA = RegInfo.createVirtualRegister(RegClass: &X86::GR32RegClass);
1247 Register InRegLEA2;
1248
1249 // Build and insert into an implicit UNDEF value. This is OK because
1250 // we will be shifting and then extracting the lower 8/16-bits.
1251 // This has the potential to cause partial register stall. e.g.
1252 // movw (%rbp,%rcx,2), %dx
1253 // leal -65(%rdx), %esi
1254 // But testing has shown this *does* help performance in 64-bit mode (at
1255 // least on modern x86 machines).
1256 MachineBasicBlock::iterator MBBI = MI.getIterator();
1257 Register Dest = MI.getOperand(i: 0).getReg();
1258 Register Src = MI.getOperand(i: 1).getReg();
1259 unsigned SrcSubReg = MI.getOperand(i: 1).getSubReg();
1260 Register Src2;
1261 unsigned Src2SubReg;
1262 bool IsDead = MI.getOperand(i: 0).isDead();
1263 bool IsKill = MI.getOperand(i: 1).isKill();
1264 unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit;
1265 assert(!MI.getOperand(1).isUndef() && "Undef op doesn't need optimization");
1266 MachineInstr *ImpDef =
1267 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: X86::IMPLICIT_DEF), DestReg: InRegLEA);
1268 MachineInstr *InsMI =
1269 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: TargetOpcode::COPY))
1270 .addReg(RegNo: InRegLEA, Flags: RegState::Define, SubReg)
1271 .addReg(RegNo: Src, Flags: getKillRegState(B: IsKill), SubReg: SrcSubReg);
1272 MachineInstr *ImpDef2 = nullptr;
1273 MachineInstr *InsMI2 = nullptr;
1274
1275 MachineInstrBuilder MIB =
1276 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: get(Opcode), DestReg: OutRegLEA);
1277#define CASE_NF(OP) \
1278 case X86::OP: \
1279 case X86::OP##_NF:
1280 switch (MIOpc) {
1281 default:
1282 llvm_unreachable("Unreachable!");
1283 CASE_NF(SHL8ri)
1284 CASE_NF(SHL16ri) {
1285 unsigned ShAmt = MI.getOperand(i: 2).getImm();
1286 MIB.addReg(RegNo: 0)
1287 .addImm(Val: 1LL << ShAmt)
1288 .addReg(RegNo: InRegLEA, Flags: RegState::Kill)
1289 .addImm(Val: 0)
1290 .addReg(RegNo: 0);
1291 break;
1292 }
1293 CASE_NF(INC8r)
1294 CASE_NF(INC16r)
1295 addRegOffset(MIB, Reg: InRegLEA, isKill: true, Offset: 1);
1296 break;
1297 CASE_NF(DEC8r)
1298 CASE_NF(DEC16r)
1299 addRegOffset(MIB, Reg: InRegLEA, isKill: true, Offset: -1);
1300 break;
1301 CASE_NF(ADD8ri)
1302 CASE_NF(ADD16ri)
1303 case X86::ADD8ri_DB:
1304 case X86::ADD16ri_DB:
1305 addRegOffset(MIB, Reg: InRegLEA, isKill: true, Offset: MI.getOperand(i: 2).getImm());
1306 break;
1307 CASE_NF(ADD8rr)
1308 CASE_NF(ADD16rr)
1309 case X86::ADD8rr_DB:
1310 case X86::ADD16rr_DB: {
1311 Src2 = MI.getOperand(i: 2).getReg();
1312 Src2SubReg = MI.getOperand(i: 2).getSubReg();
1313 bool IsKill2 = MI.getOperand(i: 2).isKill();
1314 assert(!MI.getOperand(2).isUndef() && "Undef op doesn't need optimization");
1315 if (Src == Src2) {
1316 // ADD8rr/ADD16rr killed %reg1028, %reg1028
1317 // just a single insert_subreg.
1318 addRegReg(MIB, Reg1: InRegLEA, isKill1: true, SubReg1: X86::NoSubRegister, Reg2: InRegLEA, isKill2: false,
1319 SubReg2: X86::NoSubRegister);
1320 } else {
1321 if (Subtarget.is64Bit())
1322 InRegLEA2 = RegInfo.createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
1323 else
1324 InRegLEA2 = RegInfo.createVirtualRegister(RegClass: &X86::GR32_NOSPRegClass);
1325 // Build and insert into an implicit UNDEF value. This is OK because
1326 // we will be shifting and then extracting the lower 8/16-bits.
1327 ImpDef2 = BuildMI(BB&: MBB, I: &*MIB, MIMD: MI.getDebugLoc(), MCID: get(Opcode: X86::IMPLICIT_DEF),
1328 DestReg: InRegLEA2);
1329 InsMI2 = BuildMI(BB&: MBB, I: &*MIB, MIMD: MI.getDebugLoc(), MCID: get(Opcode: TargetOpcode::COPY))
1330 .addReg(RegNo: InRegLEA2, Flags: RegState::Define, SubReg)
1331 .addReg(RegNo: Src2, Flags: getKillRegState(B: IsKill2), SubReg: Src2SubReg);
1332 addRegReg(MIB, Reg1: InRegLEA, isKill1: true, SubReg1: X86::NoSubRegister, Reg2: InRegLEA2, isKill2: true,
1333 SubReg2: X86::NoSubRegister);
1334 }
1335 if (LV && IsKill2 && InsMI2)
1336 LV->replaceKillInstruction(Reg: Src2, OldMI&: MI, NewMI&: *InsMI2);
1337 break;
1338 }
1339 }
1340
1341 MachineInstr *NewMI = MIB;
1342 MachineInstr *ExtMI =
1343 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: TargetOpcode::COPY))
1344 .addReg(RegNo: Dest, Flags: RegState::Define | getDeadRegState(B: IsDead))
1345 .addReg(RegNo: OutRegLEA, Flags: RegState::Kill, SubReg);
1346
1347 if (LV) {
1348 // Update live variables.
1349 LV->getVarInfo(Reg: InRegLEA).Kills.push_back(x: NewMI);
1350 if (InRegLEA2)
1351 LV->getVarInfo(Reg: InRegLEA2).Kills.push_back(x: NewMI);
1352 LV->getVarInfo(Reg: OutRegLEA).Kills.push_back(x: ExtMI);
1353 if (IsKill)
1354 LV->replaceKillInstruction(Reg: Src, OldMI&: MI, NewMI&: *InsMI);
1355 if (IsDead)
1356 LV->replaceKillInstruction(Reg: Dest, OldMI&: MI, NewMI&: *ExtMI);
1357 }
1358
1359 if (LIS) {
1360 LIS->InsertMachineInstrInMaps(MI&: *ImpDef);
1361 SlotIndex InsIdx = LIS->InsertMachineInstrInMaps(MI&: *InsMI);
1362 if (ImpDef2)
1363 LIS->InsertMachineInstrInMaps(MI&: *ImpDef2);
1364 SlotIndex Ins2Idx;
1365 if (InsMI2)
1366 Ins2Idx = LIS->InsertMachineInstrInMaps(MI&: *InsMI2);
1367 SlotIndex NewIdx = LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *NewMI);
1368 SlotIndex ExtIdx = LIS->InsertMachineInstrInMaps(MI&: *ExtMI);
1369 LIS->getInterval(Reg: InRegLEA);
1370 LIS->getInterval(Reg: OutRegLEA);
1371 if (InRegLEA2)
1372 LIS->getInterval(Reg: InRegLEA2);
1373
1374 // Move the use of Src up to InsMI.
1375 LiveInterval &SrcLI = LIS->getInterval(Reg: Src);
1376 LiveRange::Segment *SrcSeg = SrcLI.getSegmentContaining(Idx: NewIdx);
1377 if (SrcSeg->end == NewIdx.getRegSlot())
1378 SrcSeg->end = InsIdx.getRegSlot();
1379
1380 if (InsMI2) {
1381 // Move the use of Src2 up to InsMI2.
1382 LiveInterval &Src2LI = LIS->getInterval(Reg: Src2);
1383 LiveRange::Segment *Src2Seg = Src2LI.getSegmentContaining(Idx: NewIdx);
1384 if (Src2Seg->end == NewIdx.getRegSlot())
1385 Src2Seg->end = Ins2Idx.getRegSlot();
1386 }
1387
1388 // Move the definition of Dest down to ExtMI.
1389 LiveInterval &DestLI = LIS->getInterval(Reg: Dest);
1390 LiveRange::Segment *DestSeg =
1391 DestLI.getSegmentContaining(Idx: NewIdx.getRegSlot());
1392 assert(DestSeg->start == NewIdx.getRegSlot() &&
1393 DestSeg->valno->def == NewIdx.getRegSlot());
1394 DestSeg->start = ExtIdx.getRegSlot();
1395 DestSeg->valno->def = ExtIdx.getRegSlot();
1396 }
1397
1398 return ExtMI;
1399}
1400
1401/// This method must be implemented by targets that
1402/// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target
1403/// may be able to convert a two-address instruction into a true
1404/// three-address instruction on demand. This allows the X86 target (for
1405/// example) to convert ADD and SHL instructions into LEA instructions if they
1406/// would require register copies due to two-addressness.
1407///
1408/// This method returns a null pointer if the transformation cannot be
1409/// performed, otherwise it returns the new instruction.
1410///
1411MachineInstr *X86InstrInfo::convertToThreeAddress(MachineInstr &MI,
1412 LiveVariables *LV,
1413 LiveIntervals *LIS) const {
1414 // The following opcodes also sets the condition code register(s). Only
1415 // convert them to equivalent lea if the condition code register def's
1416 // are dead!
1417 if (hasLiveCondCodeDef(MI))
1418 return nullptr;
1419
1420 MachineFunction &MF = *MI.getParent()->getParent();
1421 // All instructions input are two-addr instructions. Get the known operands.
1422 const MachineOperand &Dest = MI.getOperand(i: 0);
1423 const MachineOperand &Src = MI.getOperand(i: 1);
1424
1425 // Ideally, operations with undef should be folded before we get here, but we
1426 // can't guarantee it. Bail out because optimizing undefs is a waste of time.
1427 // Without this, we have to forward undef state to new register operands to
1428 // avoid machine verifier errors.
1429 if (Src.isUndef())
1430 return nullptr;
1431 if (MI.getNumOperands() > 2)
1432 if (MI.getOperand(i: 2).isReg() && MI.getOperand(i: 2).isUndef())
1433 return nullptr;
1434
1435 MachineInstr *NewMI = nullptr;
1436 Register SrcReg, SrcReg2;
1437 unsigned SrcSubReg, SrcSubReg2;
1438 bool Is64Bit = Subtarget.is64Bit();
1439
1440 bool Is8BitOp = false;
1441 unsigned NumRegOperands = 2;
1442 unsigned MIOpc = MI.getOpcode();
1443 switch (MIOpc) {
1444 default:
1445 llvm_unreachable("Unreachable!");
1446 CASE_NF(SHL64ri) {
1447 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1448 unsigned ShAmt = getTruncatedShiftCount(MI, ShiftAmtOperandIdx: 2);
1449 if (!isTruncatedShiftCountForLEA(ShAmt))
1450 return nullptr;
1451
1452 // LEA can't handle RSP.
1453 if (Src.getReg().isVirtual() && !MF.getRegInfo().constrainRegClass(
1454 Reg: Src.getReg(), RC: &X86::GR64_NOSPRegClass))
1455 return nullptr;
1456
1457 NewMI = BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: get(Opcode: X86::LEA64r))
1458 .add(MO: Dest)
1459 .addReg(RegNo: 0)
1460 .addImm(Val: 1LL << ShAmt)
1461 .add(MO: Src)
1462 .addImm(Val: 0)
1463 .addReg(RegNo: 0);
1464 break;
1465 }
1466 CASE_NF(SHL32ri) {
1467 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1468 unsigned ShAmt = getTruncatedShiftCount(MI, ShiftAmtOperandIdx: 2);
1469 if (!isTruncatedShiftCountForLEA(ShAmt))
1470 return nullptr;
1471
1472 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1473
1474 // LEA can't handle ESP.
1475 bool isKill;
1476 MachineOperand ImplicitOp = MachineOperand::CreateReg(Reg: 0, isDef: false);
1477 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, NewSrc&: SrcReg, NewSrcSubReg&: SrcSubReg,
1478 isKill, ImplicitOp, LV, LIS))
1479 return nullptr;
1480
1481 MachineInstrBuilder MIB =
1482 BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Opc))
1483 .add(MO: Dest)
1484 .addReg(RegNo: 0)
1485 .addImm(Val: 1LL << ShAmt)
1486 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill), SubReg: SrcSubReg)
1487 .addImm(Val: 0)
1488 .addReg(RegNo: 0);
1489 if (ImplicitOp.getReg() != 0)
1490 MIB.add(MO: ImplicitOp);
1491 NewMI = MIB;
1492
1493 // Add kills if classifyLEAReg created a new register.
1494 if (LV && SrcReg != Src.getReg())
1495 LV->getVarInfo(Reg: SrcReg).Kills.push_back(x: NewMI);
1496 break;
1497 }
1498 CASE_NF(SHL8ri)
1499 Is8BitOp = true;
1500 [[fallthrough]];
1501 CASE_NF(SHL16ri) {
1502 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1503 unsigned ShAmt = getTruncatedShiftCount(MI, ShiftAmtOperandIdx: 2);
1504 if (!isTruncatedShiftCountForLEA(ShAmt))
1505 return nullptr;
1506 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1507 }
1508 CASE_NF(INC64r)
1509 CASE_NF(INC32r) {
1510 assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!");
1511 unsigned Opc = (MIOpc == X86::INC64r || MIOpc == X86::INC64r_NF)
1512 ? X86::LEA64r
1513 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1514 bool isKill;
1515 MachineOperand ImplicitOp = MachineOperand::CreateReg(Reg: 0, isDef: false);
1516 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, NewSrc&: SrcReg, NewSrcSubReg&: SrcSubReg,
1517 isKill, ImplicitOp, LV, LIS))
1518 return nullptr;
1519
1520 MachineInstrBuilder MIB = BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Opc))
1521 .add(MO: Dest)
1522 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill));
1523 if (ImplicitOp.getReg() != 0)
1524 MIB.add(MO: ImplicitOp);
1525
1526 NewMI = addOffset(MIB, Offset: 1);
1527
1528 // Add kills if classifyLEAReg created a new register.
1529 if (LV && SrcReg != Src.getReg())
1530 LV->getVarInfo(Reg: SrcReg).Kills.push_back(x: NewMI);
1531 break;
1532 }
1533 CASE_NF(DEC64r)
1534 CASE_NF(DEC32r) {
1535 assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!");
1536 unsigned Opc = (MIOpc == X86::DEC64r || MIOpc == X86::DEC64r_NF)
1537 ? X86::LEA64r
1538 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1539
1540 bool isKill;
1541 MachineOperand ImplicitOp = MachineOperand::CreateReg(Reg: 0, isDef: false);
1542 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, NewSrc&: SrcReg, NewSrcSubReg&: SrcSubReg,
1543 isKill, ImplicitOp, LV, LIS))
1544 return nullptr;
1545
1546 MachineInstrBuilder MIB = BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Opc))
1547 .add(MO: Dest)
1548 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill));
1549 if (ImplicitOp.getReg() != 0)
1550 MIB.add(MO: ImplicitOp);
1551
1552 NewMI = addOffset(MIB, Offset: -1);
1553
1554 // Add kills if classifyLEAReg created a new register.
1555 if (LV && SrcReg != Src.getReg())
1556 LV->getVarInfo(Reg: SrcReg).Kills.push_back(x: NewMI);
1557 break;
1558 }
1559 CASE_NF(DEC8r)
1560 CASE_NF(INC8r)
1561 Is8BitOp = true;
1562 [[fallthrough]];
1563 CASE_NF(DEC16r)
1564 CASE_NF(INC16r)
1565 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1566 CASE_NF(ADD64rr)
1567 CASE_NF(ADD32rr)
1568 case X86::ADD64rr_DB:
1569 case X86::ADD32rr_DB: {
1570 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1571 unsigned Opc;
1572 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_NF ||
1573 MIOpc == X86::ADD64rr_DB)
1574 Opc = X86::LEA64r;
1575 else
1576 Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1577
1578 const MachineOperand &Src2 = MI.getOperand(i: 2);
1579 bool isKill2;
1580 MachineOperand ImplicitOp2 = MachineOperand::CreateReg(Reg: 0, isDef: false);
1581 if (!classifyLEAReg(MI, Src: Src2, Opc, /*AllowSP=*/false, NewSrc&: SrcReg2, NewSrcSubReg&: SrcSubReg2,
1582 isKill&: isKill2, ImplicitOp&: ImplicitOp2, LV, LIS))
1583 return nullptr;
1584
1585 bool isKill;
1586 MachineOperand ImplicitOp = MachineOperand::CreateReg(Reg: 0, isDef: false);
1587 if (Src.getReg() == Src2.getReg()) {
1588 // Don't call classify LEAReg a second time on the same register, in case
1589 // the first call inserted a COPY from Src2 and marked it as killed.
1590 isKill = isKill2;
1591 SrcReg = SrcReg2;
1592 SrcSubReg = SrcSubReg2;
1593 } else {
1594 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, NewSrc&: SrcReg, NewSrcSubReg&: SrcSubReg,
1595 isKill, ImplicitOp, LV, LIS))
1596 return nullptr;
1597 }
1598
1599 MachineInstrBuilder MIB = BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Opc)).add(MO: Dest);
1600 if (ImplicitOp.getReg() != 0)
1601 MIB.add(MO: ImplicitOp);
1602 if (ImplicitOp2.getReg() != 0)
1603 MIB.add(MO: ImplicitOp2);
1604
1605 NewMI =
1606 addRegReg(MIB, Reg1: SrcReg, isKill1: isKill, SubReg1: SrcSubReg, Reg2: SrcReg2, isKill2, SubReg2: SrcSubReg2);
1607
1608 // Add kills if classifyLEAReg created a new register.
1609 if (LV) {
1610 if (SrcReg2 != Src2.getReg())
1611 LV->getVarInfo(Reg: SrcReg2).Kills.push_back(x: NewMI);
1612 if (SrcReg != SrcReg2 && SrcReg != Src.getReg())
1613 LV->getVarInfo(Reg: SrcReg).Kills.push_back(x: NewMI);
1614 }
1615 NumRegOperands = 3;
1616 break;
1617 }
1618 CASE_NF(ADD8rr)
1619 case X86::ADD8rr_DB:
1620 Is8BitOp = true;
1621 [[fallthrough]];
1622 CASE_NF(ADD16rr)
1623 case X86::ADD16rr_DB:
1624 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1625 CASE_NF(ADD64ri32)
1626 case X86::ADD64ri32_DB:
1627 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1628 NewMI = addOffset(
1629 MIB: BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: get(Opcode: X86::LEA64r)).add(MO: Dest).add(MO: Src),
1630 Offset: MI.getOperand(i: 2));
1631 break;
1632 CASE_NF(ADD32ri)
1633 case X86::ADD32ri_DB: {
1634 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1635 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1636
1637 bool isKill;
1638 MachineOperand ImplicitOp = MachineOperand::CreateReg(Reg: 0, isDef: false);
1639 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, NewSrc&: SrcReg, NewSrcSubReg&: SrcSubReg,
1640 isKill, ImplicitOp, LV, LIS))
1641 return nullptr;
1642
1643 MachineInstrBuilder MIB =
1644 BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Opc))
1645 .add(MO: Dest)
1646 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill), SubReg: SrcSubReg);
1647 if (ImplicitOp.getReg() != 0)
1648 MIB.add(MO: ImplicitOp);
1649
1650 NewMI = addOffset(MIB, Offset: MI.getOperand(i: 2));
1651
1652 // Add kills if classifyLEAReg created a new register.
1653 if (LV && SrcReg != Src.getReg())
1654 LV->getVarInfo(Reg: SrcReg).Kills.push_back(x: NewMI);
1655 break;
1656 }
1657 CASE_NF(ADD8ri)
1658 case X86::ADD8ri_DB:
1659 Is8BitOp = true;
1660 [[fallthrough]];
1661 CASE_NF(ADD16ri)
1662 case X86::ADD16ri_DB:
1663 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1664 CASE_NF(SUB8ri)
1665 CASE_NF(SUB16ri)
1666 /// FIXME: Support these similar to ADD8ri/ADD16ri*.
1667 return nullptr;
1668 CASE_NF(SUB32ri) {
1669 if (!MI.getOperand(i: 2).isImm())
1670 return nullptr;
1671 int64_t Imm = MI.getOperand(i: 2).getImm();
1672 if (!isInt<32>(x: -Imm))
1673 return nullptr;
1674
1675 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1676 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1677
1678 bool isKill;
1679 MachineOperand ImplicitOp = MachineOperand::CreateReg(Reg: 0, isDef: false);
1680 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, NewSrc&: SrcReg, NewSrcSubReg&: SrcSubReg,
1681 isKill, ImplicitOp, LV, LIS))
1682 return nullptr;
1683
1684 MachineInstrBuilder MIB =
1685 BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Opc))
1686 .add(MO: Dest)
1687 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill), SubReg: SrcSubReg);
1688 if (ImplicitOp.getReg() != 0)
1689 MIB.add(MO: ImplicitOp);
1690
1691 NewMI = addOffset(MIB, Offset: -Imm);
1692
1693 // Add kills if classifyLEAReg created a new register.
1694 if (LV && SrcReg != Src.getReg())
1695 LV->getVarInfo(Reg: SrcReg).Kills.push_back(x: NewMI);
1696 break;
1697 }
1698
1699 CASE_NF(SUB64ri32) {
1700 if (!MI.getOperand(i: 2).isImm())
1701 return nullptr;
1702 int64_t Imm = MI.getOperand(i: 2).getImm();
1703 if (!isInt<32>(x: -Imm))
1704 return nullptr;
1705
1706 assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!");
1707
1708 MachineInstrBuilder MIB =
1709 BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: get(Opcode: X86::LEA64r)).add(MO: Dest).add(MO: Src);
1710 NewMI = addOffset(MIB, Offset: -Imm);
1711 break;
1712 }
1713
1714 case X86::VMOVDQU8Z128rmk:
1715 case X86::VMOVDQU8Z256rmk:
1716 case X86::VMOVDQU8Zrmk:
1717 case X86::VMOVDQU16Z128rmk:
1718 case X86::VMOVDQU16Z256rmk:
1719 case X86::VMOVDQU16Zrmk:
1720 case X86::VMOVDQU32Z128rmk:
1721 case X86::VMOVDQA32Z128rmk:
1722 case X86::VMOVDQU32Z256rmk:
1723 case X86::VMOVDQA32Z256rmk:
1724 case X86::VMOVDQU32Zrmk:
1725 case X86::VMOVDQA32Zrmk:
1726 case X86::VMOVDQU64Z128rmk:
1727 case X86::VMOVDQA64Z128rmk:
1728 case X86::VMOVDQU64Z256rmk:
1729 case X86::VMOVDQA64Z256rmk:
1730 case X86::VMOVDQU64Zrmk:
1731 case X86::VMOVDQA64Zrmk:
1732 case X86::VMOVUPDZ128rmk:
1733 case X86::VMOVAPDZ128rmk:
1734 case X86::VMOVUPDZ256rmk:
1735 case X86::VMOVAPDZ256rmk:
1736 case X86::VMOVUPDZrmk:
1737 case X86::VMOVAPDZrmk:
1738 case X86::VMOVUPSZ128rmk:
1739 case X86::VMOVAPSZ128rmk:
1740 case X86::VMOVUPSZ256rmk:
1741 case X86::VMOVAPSZ256rmk:
1742 case X86::VMOVUPSZrmk:
1743 case X86::VMOVAPSZrmk:
1744 case X86::VBROADCASTSDZ256rmk:
1745 case X86::VBROADCASTSDZrmk:
1746 case X86::VBROADCASTSSZ128rmk:
1747 case X86::VBROADCASTSSZ256rmk:
1748 case X86::VBROADCASTSSZrmk:
1749 case X86::VPBROADCASTDZ128rmk:
1750 case X86::VPBROADCASTDZ256rmk:
1751 case X86::VPBROADCASTDZrmk:
1752 case X86::VPBROADCASTQZ128rmk:
1753 case X86::VPBROADCASTQZ256rmk:
1754 case X86::VPBROADCASTQZrmk: {
1755 unsigned Opc;
1756 switch (MIOpc) {
1757 default:
1758 llvm_unreachable("Unreachable!");
1759 case X86::VMOVDQU8Z128rmk:
1760 Opc = X86::VPBLENDMBZ128rmk;
1761 break;
1762 case X86::VMOVDQU8Z256rmk:
1763 Opc = X86::VPBLENDMBZ256rmk;
1764 break;
1765 case X86::VMOVDQU8Zrmk:
1766 Opc = X86::VPBLENDMBZrmk;
1767 break;
1768 case X86::VMOVDQU16Z128rmk:
1769 Opc = X86::VPBLENDMWZ128rmk;
1770 break;
1771 case X86::VMOVDQU16Z256rmk:
1772 Opc = X86::VPBLENDMWZ256rmk;
1773 break;
1774 case X86::VMOVDQU16Zrmk:
1775 Opc = X86::VPBLENDMWZrmk;
1776 break;
1777 case X86::VMOVDQU32Z128rmk:
1778 Opc = X86::VPBLENDMDZ128rmk;
1779 break;
1780 case X86::VMOVDQU32Z256rmk:
1781 Opc = X86::VPBLENDMDZ256rmk;
1782 break;
1783 case X86::VMOVDQU32Zrmk:
1784 Opc = X86::VPBLENDMDZrmk;
1785 break;
1786 case X86::VMOVDQU64Z128rmk:
1787 Opc = X86::VPBLENDMQZ128rmk;
1788 break;
1789 case X86::VMOVDQU64Z256rmk:
1790 Opc = X86::VPBLENDMQZ256rmk;
1791 break;
1792 case X86::VMOVDQU64Zrmk:
1793 Opc = X86::VPBLENDMQZrmk;
1794 break;
1795 case X86::VMOVUPDZ128rmk:
1796 Opc = X86::VBLENDMPDZ128rmk;
1797 break;
1798 case X86::VMOVUPDZ256rmk:
1799 Opc = X86::VBLENDMPDZ256rmk;
1800 break;
1801 case X86::VMOVUPDZrmk:
1802 Opc = X86::VBLENDMPDZrmk;
1803 break;
1804 case X86::VMOVUPSZ128rmk:
1805 Opc = X86::VBLENDMPSZ128rmk;
1806 break;
1807 case X86::VMOVUPSZ256rmk:
1808 Opc = X86::VBLENDMPSZ256rmk;
1809 break;
1810 case X86::VMOVUPSZrmk:
1811 Opc = X86::VBLENDMPSZrmk;
1812 break;
1813 case X86::VMOVDQA32Z128rmk:
1814 Opc = X86::VPBLENDMDZ128rmk;
1815 break;
1816 case X86::VMOVDQA32Z256rmk:
1817 Opc = X86::VPBLENDMDZ256rmk;
1818 break;
1819 case X86::VMOVDQA32Zrmk:
1820 Opc = X86::VPBLENDMDZrmk;
1821 break;
1822 case X86::VMOVDQA64Z128rmk:
1823 Opc = X86::VPBLENDMQZ128rmk;
1824 break;
1825 case X86::VMOVDQA64Z256rmk:
1826 Opc = X86::VPBLENDMQZ256rmk;
1827 break;
1828 case X86::VMOVDQA64Zrmk:
1829 Opc = X86::VPBLENDMQZrmk;
1830 break;
1831 case X86::VMOVAPDZ128rmk:
1832 Opc = X86::VBLENDMPDZ128rmk;
1833 break;
1834 case X86::VMOVAPDZ256rmk:
1835 Opc = X86::VBLENDMPDZ256rmk;
1836 break;
1837 case X86::VMOVAPDZrmk:
1838 Opc = X86::VBLENDMPDZrmk;
1839 break;
1840 case X86::VMOVAPSZ128rmk:
1841 Opc = X86::VBLENDMPSZ128rmk;
1842 break;
1843 case X86::VMOVAPSZ256rmk:
1844 Opc = X86::VBLENDMPSZ256rmk;
1845 break;
1846 case X86::VMOVAPSZrmk:
1847 Opc = X86::VBLENDMPSZrmk;
1848 break;
1849 case X86::VBROADCASTSDZ256rmk:
1850 Opc = X86::VBLENDMPDZ256rmbk;
1851 break;
1852 case X86::VBROADCASTSDZrmk:
1853 Opc = X86::VBLENDMPDZrmbk;
1854 break;
1855 case X86::VBROADCASTSSZ128rmk:
1856 Opc = X86::VBLENDMPSZ128rmbk;
1857 break;
1858 case X86::VBROADCASTSSZ256rmk:
1859 Opc = X86::VBLENDMPSZ256rmbk;
1860 break;
1861 case X86::VBROADCASTSSZrmk:
1862 Opc = X86::VBLENDMPSZrmbk;
1863 break;
1864 case X86::VPBROADCASTDZ128rmk:
1865 Opc = X86::VPBLENDMDZ128rmbk;
1866 break;
1867 case X86::VPBROADCASTDZ256rmk:
1868 Opc = X86::VPBLENDMDZ256rmbk;
1869 break;
1870 case X86::VPBROADCASTDZrmk:
1871 Opc = X86::VPBLENDMDZrmbk;
1872 break;
1873 case X86::VPBROADCASTQZ128rmk:
1874 Opc = X86::VPBLENDMQZ128rmbk;
1875 break;
1876 case X86::VPBROADCASTQZ256rmk:
1877 Opc = X86::VPBLENDMQZ256rmbk;
1878 break;
1879 case X86::VPBROADCASTQZrmk:
1880 Opc = X86::VPBLENDMQZrmbk;
1881 break;
1882 }
1883
1884 NewMI = BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Opc))
1885 .add(MO: Dest)
1886 .add(MO: MI.getOperand(i: 2))
1887 .add(MO: Src)
1888 .add(MO: MI.getOperand(i: 3))
1889 .add(MO: MI.getOperand(i: 4))
1890 .add(MO: MI.getOperand(i: 5))
1891 .add(MO: MI.getOperand(i: 6))
1892 .add(MO: MI.getOperand(i: 7));
1893 NumRegOperands = 4;
1894 break;
1895 }
1896
1897 case X86::VMOVDQU8Z128rrk:
1898 case X86::VMOVDQU8Z256rrk:
1899 case X86::VMOVDQU8Zrrk:
1900 case X86::VMOVDQU16Z128rrk:
1901 case X86::VMOVDQU16Z256rrk:
1902 case X86::VMOVDQU16Zrrk:
1903 case X86::VMOVDQU32Z128rrk:
1904 case X86::VMOVDQA32Z128rrk:
1905 case X86::VMOVDQU32Z256rrk:
1906 case X86::VMOVDQA32Z256rrk:
1907 case X86::VMOVDQU32Zrrk:
1908 case X86::VMOVDQA32Zrrk:
1909 case X86::VMOVDQU64Z128rrk:
1910 case X86::VMOVDQA64Z128rrk:
1911 case X86::VMOVDQU64Z256rrk:
1912 case X86::VMOVDQA64Z256rrk:
1913 case X86::VMOVDQU64Zrrk:
1914 case X86::VMOVDQA64Zrrk:
1915 case X86::VMOVUPDZ128rrk:
1916 case X86::VMOVAPDZ128rrk:
1917 case X86::VMOVUPDZ256rrk:
1918 case X86::VMOVAPDZ256rrk:
1919 case X86::VMOVUPDZrrk:
1920 case X86::VMOVAPDZrrk:
1921 case X86::VMOVUPSZ128rrk:
1922 case X86::VMOVAPSZ128rrk:
1923 case X86::VMOVUPSZ256rrk:
1924 case X86::VMOVAPSZ256rrk:
1925 case X86::VMOVUPSZrrk:
1926 case X86::VMOVAPSZrrk: {
1927 unsigned Opc;
1928 switch (MIOpc) {
1929 default:
1930 llvm_unreachable("Unreachable!");
1931 case X86::VMOVDQU8Z128rrk:
1932 Opc = X86::VPBLENDMBZ128rrk;
1933 break;
1934 case X86::VMOVDQU8Z256rrk:
1935 Opc = X86::VPBLENDMBZ256rrk;
1936 break;
1937 case X86::VMOVDQU8Zrrk:
1938 Opc = X86::VPBLENDMBZrrk;
1939 break;
1940 case X86::VMOVDQU16Z128rrk:
1941 Opc = X86::VPBLENDMWZ128rrk;
1942 break;
1943 case X86::VMOVDQU16Z256rrk:
1944 Opc = X86::VPBLENDMWZ256rrk;
1945 break;
1946 case X86::VMOVDQU16Zrrk:
1947 Opc = X86::VPBLENDMWZrrk;
1948 break;
1949 case X86::VMOVDQU32Z128rrk:
1950 Opc = X86::VPBLENDMDZ128rrk;
1951 break;
1952 case X86::VMOVDQU32Z256rrk:
1953 Opc = X86::VPBLENDMDZ256rrk;
1954 break;
1955 case X86::VMOVDQU32Zrrk:
1956 Opc = X86::VPBLENDMDZrrk;
1957 break;
1958 case X86::VMOVDQU64Z128rrk:
1959 Opc = X86::VPBLENDMQZ128rrk;
1960 break;
1961 case X86::VMOVDQU64Z256rrk:
1962 Opc = X86::VPBLENDMQZ256rrk;
1963 break;
1964 case X86::VMOVDQU64Zrrk:
1965 Opc = X86::VPBLENDMQZrrk;
1966 break;
1967 case X86::VMOVUPDZ128rrk:
1968 Opc = X86::VBLENDMPDZ128rrk;
1969 break;
1970 case X86::VMOVUPDZ256rrk:
1971 Opc = X86::VBLENDMPDZ256rrk;
1972 break;
1973 case X86::VMOVUPDZrrk:
1974 Opc = X86::VBLENDMPDZrrk;
1975 break;
1976 case X86::VMOVUPSZ128rrk:
1977 Opc = X86::VBLENDMPSZ128rrk;
1978 break;
1979 case X86::VMOVUPSZ256rrk:
1980 Opc = X86::VBLENDMPSZ256rrk;
1981 break;
1982 case X86::VMOVUPSZrrk:
1983 Opc = X86::VBLENDMPSZrrk;
1984 break;
1985 case X86::VMOVDQA32Z128rrk:
1986 Opc = X86::VPBLENDMDZ128rrk;
1987 break;
1988 case X86::VMOVDQA32Z256rrk:
1989 Opc = X86::VPBLENDMDZ256rrk;
1990 break;
1991 case X86::VMOVDQA32Zrrk:
1992 Opc = X86::VPBLENDMDZrrk;
1993 break;
1994 case X86::VMOVDQA64Z128rrk:
1995 Opc = X86::VPBLENDMQZ128rrk;
1996 break;
1997 case X86::VMOVDQA64Z256rrk:
1998 Opc = X86::VPBLENDMQZ256rrk;
1999 break;
2000 case X86::VMOVDQA64Zrrk:
2001 Opc = X86::VPBLENDMQZrrk;
2002 break;
2003 case X86::VMOVAPDZ128rrk:
2004 Opc = X86::VBLENDMPDZ128rrk;
2005 break;
2006 case X86::VMOVAPDZ256rrk:
2007 Opc = X86::VBLENDMPDZ256rrk;
2008 break;
2009 case X86::VMOVAPDZrrk:
2010 Opc = X86::VBLENDMPDZrrk;
2011 break;
2012 case X86::VMOVAPSZ128rrk:
2013 Opc = X86::VBLENDMPSZ128rrk;
2014 break;
2015 case X86::VMOVAPSZ256rrk:
2016 Opc = X86::VBLENDMPSZ256rrk;
2017 break;
2018 case X86::VMOVAPSZrrk:
2019 Opc = X86::VBLENDMPSZrrk;
2020 break;
2021 }
2022
2023 NewMI = BuildMI(MF, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Opc))
2024 .add(MO: Dest)
2025 .add(MO: MI.getOperand(i: 2))
2026 .add(MO: Src)
2027 .add(MO: MI.getOperand(i: 3));
2028 NumRegOperands = 4;
2029 break;
2030 }
2031 }
2032#undef CASE_NF
2033
2034 if (!NewMI)
2035 return nullptr;
2036
2037 if (LV) { // Update live variables
2038 for (unsigned I = 0; I < NumRegOperands; ++I) {
2039 MachineOperand &Op = MI.getOperand(i: I);
2040 if (Op.isReg() && (Op.isDead() || Op.isKill()))
2041 LV->replaceKillInstruction(Reg: Op.getReg(), OldMI&: MI, NewMI&: *NewMI);
2042 }
2043 }
2044
2045 MachineBasicBlock &MBB = *MI.getParent();
2046 MBB.insert(I: MI.getIterator(), M: NewMI); // Insert the new inst
2047
2048 if (LIS) {
2049 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *NewMI);
2050 if (SrcReg)
2051 LIS->getInterval(Reg: SrcReg);
2052 if (SrcReg2)
2053 LIS->getInterval(Reg: SrcReg2);
2054 }
2055
2056 return NewMI;
2057}
2058
2059/// This determines which of three possible cases of a three source commute
2060/// the source indexes correspond to taking into account any mask operands.
2061/// All prevents commuting a passthru operand. Returns -1 if the commute isn't
2062/// possible.
2063/// Case 0 - Possible to commute the first and second operands.
2064/// Case 1 - Possible to commute the first and third operands.
2065/// Case 2 - Possible to commute the second and third operands.
2066static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1,
2067 unsigned SrcOpIdx2) {
2068 // Put the lowest index to SrcOpIdx1 to simplify the checks below.
2069 if (SrcOpIdx1 > SrcOpIdx2)
2070 std::swap(a&: SrcOpIdx1, b&: SrcOpIdx2);
2071
2072 unsigned Op1 = 1, Op2 = 2, Op3 = 3;
2073 if (X86II::isKMasked(TSFlags)) {
2074 Op2++;
2075 Op3++;
2076 }
2077
2078 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2)
2079 return 0;
2080 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3)
2081 return 1;
2082 if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3)
2083 return 2;
2084 llvm_unreachable("Unknown three src commute case.");
2085}
2086
2087unsigned X86InstrInfo::getFMA3OpcodeToCommuteOperands(
2088 const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2,
2089 const X86InstrFMA3Group &FMA3Group) const {
2090
2091 unsigned Opc = MI.getOpcode();
2092
2093 // TODO: Commuting the 1st operand of FMA*_Int requires some additional
2094 // analysis. The commute optimization is legal only if all users of FMA*_Int
2095 // use only the lowest element of the FMA*_Int instruction. Such analysis are
2096 // not implemented yet. So, just return 0 in that case.
2097 // When such analysis are available this place will be the right place for
2098 // calling it.
2099 assert(!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) &&
2100 "Intrinsic instructions can't commute operand 1");
2101
2102 // Determine which case this commute is or if it can't be done.
2103 unsigned Case =
2104 getThreeSrcCommuteCase(TSFlags: MI.getDesc().TSFlags, SrcOpIdx1, SrcOpIdx2);
2105 assert(Case < 3 && "Unexpected case number!");
2106
2107 // Define the FMA forms mapping array that helps to map input FMA form
2108 // to output FMA form to preserve the operation semantics after
2109 // commuting the operands.
2110 const unsigned Form132Index = 0;
2111 const unsigned Form213Index = 1;
2112 const unsigned Form231Index = 2;
2113 static const unsigned FormMapping[][3] = {
2114 // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2;
2115 // FMA132 A, C, b; ==> FMA231 C, A, b;
2116 // FMA213 B, A, c; ==> FMA213 A, B, c;
2117 // FMA231 C, A, b; ==> FMA132 A, C, b;
2118 {Form231Index, Form213Index, Form132Index},
2119 // 1: SrcOpIdx1 == 1 && SrcOpIdx2 == 3;
2120 // FMA132 A, c, B; ==> FMA132 B, c, A;
2121 // FMA213 B, a, C; ==> FMA231 C, a, B;
2122 // FMA231 C, a, B; ==> FMA213 B, a, C;
2123 {Form132Index, Form231Index, Form213Index},
2124 // 2: SrcOpIdx1 == 2 && SrcOpIdx2 == 3;
2125 // FMA132 a, C, B; ==> FMA213 a, B, C;
2126 // FMA213 b, A, C; ==> FMA132 b, C, A;
2127 // FMA231 c, A, B; ==> FMA231 c, B, A;
2128 {Form213Index, Form132Index, Form231Index}};
2129
2130 unsigned FMAForms[3];
2131 FMAForms[0] = FMA3Group.get132Opcode();
2132 FMAForms[1] = FMA3Group.get213Opcode();
2133 FMAForms[2] = FMA3Group.get231Opcode();
2134
2135 // Everything is ready, just adjust the FMA opcode and return it.
2136 for (unsigned FormIndex = 0; FormIndex < 3; FormIndex++)
2137 if (Opc == FMAForms[FormIndex])
2138 return FMAForms[FormMapping[Case][FormIndex]];
2139
2140 llvm_unreachable("Illegal FMA3 format");
2141}
2142
2143static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1,
2144 unsigned SrcOpIdx2) {
2145 // Determine which case this commute is or if it can't be done.
2146 unsigned Case =
2147 getThreeSrcCommuteCase(TSFlags: MI.getDesc().TSFlags, SrcOpIdx1, SrcOpIdx2);
2148 assert(Case < 3 && "Unexpected case value!");
2149
2150 // For each case we need to swap two pairs of bits in the final immediate.
2151 static const uint8_t SwapMasks[3][4] = {
2152 {0x04, 0x10, 0x08, 0x20}, // Swap bits 2/4 and 3/5.
2153 {0x02, 0x10, 0x08, 0x40}, // Swap bits 1/4 and 3/6.
2154 {0x02, 0x04, 0x20, 0x40}, // Swap bits 1/2 and 5/6.
2155 };
2156
2157 uint8_t Imm = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
2158 // Clear out the bits we are swapping.
2159 uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] |
2160 SwapMasks[Case][2] | SwapMasks[Case][3]);
2161 // If the immediate had a bit of the pair set, then set the opposite bit.
2162 if (Imm & SwapMasks[Case][0])
2163 NewImm |= SwapMasks[Case][1];
2164 if (Imm & SwapMasks[Case][1])
2165 NewImm |= SwapMasks[Case][0];
2166 if (Imm & SwapMasks[Case][2])
2167 NewImm |= SwapMasks[Case][3];
2168 if (Imm & SwapMasks[Case][3])
2169 NewImm |= SwapMasks[Case][2];
2170 MI.getOperand(i: MI.getNumOperands() - 1).setImm(NewImm);
2171}
2172
2173// Returns true if this is a VPERMI2 or VPERMT2 instruction that can be
2174// commuted.
2175static bool isCommutableVPERMV3Instruction(unsigned Opcode) {
2176#define VPERM_CASES(Suffix) \
2177 case X86::VPERMI2##Suffix##Z128rr: \
2178 case X86::VPERMT2##Suffix##Z128rr: \
2179 case X86::VPERMI2##Suffix##Z256rr: \
2180 case X86::VPERMT2##Suffix##Z256rr: \
2181 case X86::VPERMI2##Suffix##Zrr: \
2182 case X86::VPERMT2##Suffix##Zrr: \
2183 case X86::VPERMI2##Suffix##Z128rm: \
2184 case X86::VPERMT2##Suffix##Z128rm: \
2185 case X86::VPERMI2##Suffix##Z256rm: \
2186 case X86::VPERMT2##Suffix##Z256rm: \
2187 case X86::VPERMI2##Suffix##Zrm: \
2188 case X86::VPERMT2##Suffix##Zrm: \
2189 case X86::VPERMI2##Suffix##Z128rrkz: \
2190 case X86::VPERMT2##Suffix##Z128rrkz: \
2191 case X86::VPERMI2##Suffix##Z256rrkz: \
2192 case X86::VPERMT2##Suffix##Z256rrkz: \
2193 case X86::VPERMI2##Suffix##Zrrkz: \
2194 case X86::VPERMT2##Suffix##Zrrkz: \
2195 case X86::VPERMI2##Suffix##Z128rmkz: \
2196 case X86::VPERMT2##Suffix##Z128rmkz: \
2197 case X86::VPERMI2##Suffix##Z256rmkz: \
2198 case X86::VPERMT2##Suffix##Z256rmkz: \
2199 case X86::VPERMI2##Suffix##Zrmkz: \
2200 case X86::VPERMT2##Suffix##Zrmkz:
2201
2202#define VPERM_CASES_BROADCAST(Suffix) \
2203 VPERM_CASES(Suffix) \
2204 case X86::VPERMI2##Suffix##Z128rmb: \
2205 case X86::VPERMT2##Suffix##Z128rmb: \
2206 case X86::VPERMI2##Suffix##Z256rmb: \
2207 case X86::VPERMT2##Suffix##Z256rmb: \
2208 case X86::VPERMI2##Suffix##Zrmb: \
2209 case X86::VPERMT2##Suffix##Zrmb: \
2210 case X86::VPERMI2##Suffix##Z128rmbkz: \
2211 case X86::VPERMT2##Suffix##Z128rmbkz: \
2212 case X86::VPERMI2##Suffix##Z256rmbkz: \
2213 case X86::VPERMT2##Suffix##Z256rmbkz: \
2214 case X86::VPERMI2##Suffix##Zrmbkz: \
2215 case X86::VPERMT2##Suffix##Zrmbkz:
2216
2217 switch (Opcode) {
2218 default:
2219 return false;
2220 VPERM_CASES(B)
2221 VPERM_CASES_BROADCAST(D)
2222 VPERM_CASES_BROADCAST(PD)
2223 VPERM_CASES_BROADCAST(PS)
2224 VPERM_CASES_BROADCAST(Q)
2225 VPERM_CASES(W)
2226 return true;
2227 }
2228#undef VPERM_CASES_BROADCAST
2229#undef VPERM_CASES
2230}
2231
2232// Returns commuted opcode for VPERMI2 and VPERMT2 instructions by switching
2233// from the I opcode to the T opcode and vice versa.
2234static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) {
2235#define VPERM_CASES(Orig, New) \
2236 case X86::Orig##Z128rr: \
2237 return X86::New##Z128rr; \
2238 case X86::Orig##Z128rrkz: \
2239 return X86::New##Z128rrkz; \
2240 case X86::Orig##Z128rm: \
2241 return X86::New##Z128rm; \
2242 case X86::Orig##Z128rmkz: \
2243 return X86::New##Z128rmkz; \
2244 case X86::Orig##Z256rr: \
2245 return X86::New##Z256rr; \
2246 case X86::Orig##Z256rrkz: \
2247 return X86::New##Z256rrkz; \
2248 case X86::Orig##Z256rm: \
2249 return X86::New##Z256rm; \
2250 case X86::Orig##Z256rmkz: \
2251 return X86::New##Z256rmkz; \
2252 case X86::Orig##Zrr: \
2253 return X86::New##Zrr; \
2254 case X86::Orig##Zrrkz: \
2255 return X86::New##Zrrkz; \
2256 case X86::Orig##Zrm: \
2257 return X86::New##Zrm; \
2258 case X86::Orig##Zrmkz: \
2259 return X86::New##Zrmkz;
2260
2261#define VPERM_CASES_BROADCAST(Orig, New) \
2262 VPERM_CASES(Orig, New) \
2263 case X86::Orig##Z128rmb: \
2264 return X86::New##Z128rmb; \
2265 case X86::Orig##Z128rmbkz: \
2266 return X86::New##Z128rmbkz; \
2267 case X86::Orig##Z256rmb: \
2268 return X86::New##Z256rmb; \
2269 case X86::Orig##Z256rmbkz: \
2270 return X86::New##Z256rmbkz; \
2271 case X86::Orig##Zrmb: \
2272 return X86::New##Zrmb; \
2273 case X86::Orig##Zrmbkz: \
2274 return X86::New##Zrmbkz;
2275
2276 switch (Opcode) {
2277 VPERM_CASES(VPERMI2B, VPERMT2B)
2278 VPERM_CASES_BROADCAST(VPERMI2D, VPERMT2D)
2279 VPERM_CASES_BROADCAST(VPERMI2PD, VPERMT2PD)
2280 VPERM_CASES_BROADCAST(VPERMI2PS, VPERMT2PS)
2281 VPERM_CASES_BROADCAST(VPERMI2Q, VPERMT2Q)
2282 VPERM_CASES(VPERMI2W, VPERMT2W)
2283 VPERM_CASES(VPERMT2B, VPERMI2B)
2284 VPERM_CASES_BROADCAST(VPERMT2D, VPERMI2D)
2285 VPERM_CASES_BROADCAST(VPERMT2PD, VPERMI2PD)
2286 VPERM_CASES_BROADCAST(VPERMT2PS, VPERMI2PS)
2287 VPERM_CASES_BROADCAST(VPERMT2Q, VPERMI2Q)
2288 VPERM_CASES(VPERMT2W, VPERMI2W)
2289 }
2290
2291 llvm_unreachable("Unreachable!");
2292#undef VPERM_CASES_BROADCAST
2293#undef VPERM_CASES
2294}
2295
2296MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
2297 unsigned OpIdx1,
2298 unsigned OpIdx2) const {
2299 auto CloneIfNew = [&](MachineInstr &MI) {
2300 return std::exchange(obj&: NewMI, new_val: false)
2301 ? MI.getParent()->getParent()->CloneMachineInstr(Orig: &MI)
2302 : &MI;
2303 };
2304 MachineInstr *WorkingMI = nullptr;
2305 unsigned Opc = MI.getOpcode();
2306
2307#define CASE_ND(OP) \
2308 case X86::OP: \
2309 case X86::OP##_ND:
2310
2311 switch (Opc) {
2312 // SHLD B, C, I <-> SHRD C, B, (BitWidth - I)
2313 CASE_ND(SHRD16rri8)
2314 CASE_ND(SHLD16rri8)
2315 CASE_ND(SHRD32rri8)
2316 CASE_ND(SHLD32rri8)
2317 CASE_ND(SHRD64rri8)
2318 CASE_ND(SHLD64rri8) {
2319 unsigned Size;
2320 switch (Opc) {
2321 default:
2322 llvm_unreachable("Unreachable!");
2323#define FROM_TO_SIZE(A, B, S) \
2324 case X86::A: \
2325 Opc = X86::B; \
2326 Size = S; \
2327 break; \
2328 case X86::A##_ND: \
2329 Opc = X86::B##_ND; \
2330 Size = S; \
2331 break; \
2332 case X86::B: \
2333 Opc = X86::A; \
2334 Size = S; \
2335 break; \
2336 case X86::B##_ND: \
2337 Opc = X86::A##_ND; \
2338 Size = S; \
2339 break;
2340
2341 FROM_TO_SIZE(SHRD16rri8, SHLD16rri8, 16)
2342 FROM_TO_SIZE(SHRD32rri8, SHLD32rri8, 32)
2343 FROM_TO_SIZE(SHRD64rri8, SHLD64rri8, 64)
2344#undef FROM_TO_SIZE
2345 }
2346 WorkingMI = CloneIfNew(MI);
2347 WorkingMI->setDesc(get(Opcode: Opc));
2348 WorkingMI->getOperand(i: 3).setImm(Size - MI.getOperand(i: 3).getImm());
2349 break;
2350 }
2351 case X86::PFSUBrr:
2352 case X86::PFSUBRrr:
2353 // PFSUB x, y: x = x - y
2354 // PFSUBR x, y: x = y - x
2355 WorkingMI = CloneIfNew(MI);
2356 WorkingMI->setDesc(
2357 get(Opcode: X86::PFSUBRrr == Opc ? X86::PFSUBrr : X86::PFSUBRrr));
2358 break;
2359 case X86::BLENDPDrri:
2360 case X86::BLENDPSrri:
2361 case X86::PBLENDWrri:
2362 case X86::VBLENDPDrri:
2363 case X86::VBLENDPSrri:
2364 case X86::VBLENDPDYrri:
2365 case X86::VBLENDPSYrri:
2366 case X86::VPBLENDDrri:
2367 case X86::VPBLENDWrri:
2368 case X86::VPBLENDDYrri:
2369 case X86::VPBLENDWYrri: {
2370 int8_t Mask;
2371 switch (Opc) {
2372 default:
2373 llvm_unreachable("Unreachable!");
2374 case X86::BLENDPDrri:
2375 Mask = (int8_t)0x03;
2376 break;
2377 case X86::BLENDPSrri:
2378 Mask = (int8_t)0x0F;
2379 break;
2380 case X86::PBLENDWrri:
2381 Mask = (int8_t)0xFF;
2382 break;
2383 case X86::VBLENDPDrri:
2384 Mask = (int8_t)0x03;
2385 break;
2386 case X86::VBLENDPSrri:
2387 Mask = (int8_t)0x0F;
2388 break;
2389 case X86::VBLENDPDYrri:
2390 Mask = (int8_t)0x0F;
2391 break;
2392 case X86::VBLENDPSYrri:
2393 Mask = (int8_t)0xFF;
2394 break;
2395 case X86::VPBLENDDrri:
2396 Mask = (int8_t)0x0F;
2397 break;
2398 case X86::VPBLENDWrri:
2399 Mask = (int8_t)0xFF;
2400 break;
2401 case X86::VPBLENDDYrri:
2402 Mask = (int8_t)0xFF;
2403 break;
2404 case X86::VPBLENDWYrri:
2405 Mask = (int8_t)0xFF;
2406 break;
2407 }
2408 // Only the least significant bits of Imm are used.
2409 // Using int8_t to ensure it will be sign extended to the int64_t that
2410 // setImm takes in order to match isel behavior.
2411 int8_t Imm = MI.getOperand(i: 3).getImm() & Mask;
2412 WorkingMI = CloneIfNew(MI);
2413 WorkingMI->getOperand(i: 3).setImm(Mask ^ Imm);
2414 break;
2415 }
2416 case X86::INSERTPSrri:
2417 case X86::VINSERTPSrri:
2418 case X86::VINSERTPSZrri: {
2419 unsigned Imm = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
2420 unsigned ZMask = Imm & 15;
2421 unsigned DstIdx = (Imm >> 4) & 3;
2422 unsigned SrcIdx = (Imm >> 6) & 3;
2423
2424 // We can commute insertps if we zero 2 of the elements, the insertion is
2425 // "inline" and we don't override the insertion with a zero.
2426 if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 &&
2427 llvm::popcount(Value: ZMask) == 2) {
2428 unsigned AltIdx = llvm::countr_zero(Val: (ZMask | (1 << DstIdx)) ^ 15);
2429 assert(AltIdx < 4 && "Illegal insertion index");
2430 unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask;
2431 WorkingMI = CloneIfNew(MI);
2432 WorkingMI->getOperand(i: MI.getNumOperands() - 1).setImm(AltImm);
2433 break;
2434 }
2435 return nullptr;
2436 }
2437 case X86::MOVSDrr:
2438 case X86::MOVSSrr:
2439 case X86::VMOVSDrr:
2440 case X86::VMOVSSrr: {
2441 // On SSE41 or later we can commute a MOVSS/MOVSD to a BLENDPS/BLENDPD.
2442 if (Subtarget.hasSSE41()) {
2443 unsigned Mask;
2444 switch (Opc) {
2445 default:
2446 llvm_unreachable("Unreachable!");
2447 case X86::MOVSDrr:
2448 Opc = X86::BLENDPDrri;
2449 Mask = 0x02;
2450 break;
2451 case X86::MOVSSrr:
2452 Opc = X86::BLENDPSrri;
2453 Mask = 0x0E;
2454 break;
2455 case X86::VMOVSDrr:
2456 Opc = X86::VBLENDPDrri;
2457 Mask = 0x02;
2458 break;
2459 case X86::VMOVSSrr:
2460 Opc = X86::VBLENDPSrri;
2461 Mask = 0x0E;
2462 break;
2463 }
2464
2465 WorkingMI = CloneIfNew(MI);
2466 WorkingMI->setDesc(get(Opcode: Opc));
2467 WorkingMI->addOperand(Op: MachineOperand::CreateImm(Val: Mask));
2468 break;
2469 }
2470
2471 assert(Opc == X86::MOVSDrr && "Only MOVSD can commute to SHUFPD");
2472 WorkingMI = CloneIfNew(MI);
2473 WorkingMI->setDesc(get(Opcode: X86::SHUFPDrri));
2474 WorkingMI->addOperand(Op: MachineOperand::CreateImm(Val: 0x02));
2475 break;
2476 }
2477 case X86::SHUFPDrri: {
2478 // Commute to MOVSD.
2479 assert(MI.getOperand(3).getImm() == 0x02 && "Unexpected immediate!");
2480 WorkingMI = CloneIfNew(MI);
2481 WorkingMI->setDesc(get(Opcode: X86::MOVSDrr));
2482 WorkingMI->removeOperand(OpNo: 3);
2483 break;
2484 }
2485 case X86::PCLMULQDQrri:
2486 case X86::VPCLMULQDQrri:
2487 case X86::VPCLMULQDQYrri:
2488 case X86::VPCLMULQDQZrri:
2489 case X86::VPCLMULQDQZ128rri:
2490 case X86::VPCLMULQDQZ256rri: {
2491 // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0]
2492 // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0]
2493 unsigned Imm = MI.getOperand(i: 3).getImm();
2494 unsigned Src1Hi = Imm & 0x01;
2495 unsigned Src2Hi = Imm & 0x10;
2496 WorkingMI = CloneIfNew(MI);
2497 WorkingMI->getOperand(i: 3).setImm((Src1Hi << 4) | (Src2Hi >> 4));
2498 break;
2499 }
2500 case X86::VPCMPBZ128rri:
2501 case X86::VPCMPUBZ128rri:
2502 case X86::VPCMPBZ256rri:
2503 case X86::VPCMPUBZ256rri:
2504 case X86::VPCMPBZrri:
2505 case X86::VPCMPUBZrri:
2506 case X86::VPCMPDZ128rri:
2507 case X86::VPCMPUDZ128rri:
2508 case X86::VPCMPDZ256rri:
2509 case X86::VPCMPUDZ256rri:
2510 case X86::VPCMPDZrri:
2511 case X86::VPCMPUDZrri:
2512 case X86::VPCMPQZ128rri:
2513 case X86::VPCMPUQZ128rri:
2514 case X86::VPCMPQZ256rri:
2515 case X86::VPCMPUQZ256rri:
2516 case X86::VPCMPQZrri:
2517 case X86::VPCMPUQZrri:
2518 case X86::VPCMPWZ128rri:
2519 case X86::VPCMPUWZ128rri:
2520 case X86::VPCMPWZ256rri:
2521 case X86::VPCMPUWZ256rri:
2522 case X86::VPCMPWZrri:
2523 case X86::VPCMPUWZrri:
2524 case X86::VPCMPBZ128rrik:
2525 case X86::VPCMPUBZ128rrik:
2526 case X86::VPCMPBZ256rrik:
2527 case X86::VPCMPUBZ256rrik:
2528 case X86::VPCMPBZrrik:
2529 case X86::VPCMPUBZrrik:
2530 case X86::VPCMPDZ128rrik:
2531 case X86::VPCMPUDZ128rrik:
2532 case X86::VPCMPDZ256rrik:
2533 case X86::VPCMPUDZ256rrik:
2534 case X86::VPCMPDZrrik:
2535 case X86::VPCMPUDZrrik:
2536 case X86::VPCMPQZ128rrik:
2537 case X86::VPCMPUQZ128rrik:
2538 case X86::VPCMPQZ256rrik:
2539 case X86::VPCMPUQZ256rrik:
2540 case X86::VPCMPQZrrik:
2541 case X86::VPCMPUQZrrik:
2542 case X86::VPCMPWZ128rrik:
2543 case X86::VPCMPUWZ128rrik:
2544 case X86::VPCMPWZ256rrik:
2545 case X86::VPCMPUWZ256rrik:
2546 case X86::VPCMPWZrrik:
2547 case X86::VPCMPUWZrrik:
2548 WorkingMI = CloneIfNew(MI);
2549 // Flip comparison mode immediate (if necessary).
2550 WorkingMI->getOperand(i: MI.getNumOperands() - 1)
2551 .setImm(X86::getSwappedVPCMPImm(
2552 Imm: MI.getOperand(i: MI.getNumOperands() - 1).getImm() & 0x7));
2553 break;
2554 case X86::VPCOMBri:
2555 case X86::VPCOMUBri:
2556 case X86::VPCOMDri:
2557 case X86::VPCOMUDri:
2558 case X86::VPCOMQri:
2559 case X86::VPCOMUQri:
2560 case X86::VPCOMWri:
2561 case X86::VPCOMUWri:
2562 WorkingMI = CloneIfNew(MI);
2563 // Flip comparison mode immediate (if necessary).
2564 WorkingMI->getOperand(i: 3).setImm(
2565 X86::getSwappedVPCOMImm(Imm: MI.getOperand(i: 3).getImm() & 0x7));
2566 break;
2567 case X86::VCMPSDZrri:
2568 case X86::VCMPSSZrri:
2569 case X86::VCMPPDZrri:
2570 case X86::VCMPPSZrri:
2571 case X86::VCMPSHZrri:
2572 case X86::VCMPPHZrri:
2573 case X86::VCMPPHZ128rri:
2574 case X86::VCMPPHZ256rri:
2575 case X86::VCMPPDZ128rri:
2576 case X86::VCMPPSZ128rri:
2577 case X86::VCMPPDZ256rri:
2578 case X86::VCMPPSZ256rri:
2579 case X86::VCMPPDZrrik:
2580 case X86::VCMPPSZrrik:
2581 case X86::VCMPPHZrrik:
2582 case X86::VCMPPDZ128rrik:
2583 case X86::VCMPPSZ128rrik:
2584 case X86::VCMPPHZ128rrik:
2585 case X86::VCMPPDZ256rrik:
2586 case X86::VCMPPSZ256rrik:
2587 case X86::VCMPPHZ256rrik:
2588 WorkingMI = CloneIfNew(MI);
2589 WorkingMI->getOperand(i: MI.getNumExplicitOperands() - 1)
2590 .setImm(X86::getSwappedVCMPImm(
2591 Imm: MI.getOperand(i: MI.getNumExplicitOperands() - 1).getImm() & 0x1f));
2592 break;
2593 case X86::VPERM2F128rri:
2594 case X86::VPERM2I128rri:
2595 // Flip permute source immediate.
2596 // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi.
2597 // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi.
2598 WorkingMI = CloneIfNew(MI);
2599 WorkingMI->getOperand(i: 3).setImm((MI.getOperand(i: 3).getImm() & 0xFF) ^ 0x22);
2600 break;
2601 case X86::MOVHLPSrr:
2602 case X86::UNPCKHPDrr:
2603 case X86::VMOVHLPSrr:
2604 case X86::VUNPCKHPDrr:
2605 case X86::VMOVHLPSZrr:
2606 case X86::VUNPCKHPDZ128rr:
2607 assert(Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!");
2608
2609 switch (Opc) {
2610 default:
2611 llvm_unreachable("Unreachable!");
2612 case X86::MOVHLPSrr:
2613 Opc = X86::UNPCKHPDrr;
2614 break;
2615 case X86::UNPCKHPDrr:
2616 Opc = X86::MOVHLPSrr;
2617 break;
2618 case X86::VMOVHLPSrr:
2619 Opc = X86::VUNPCKHPDrr;
2620 break;
2621 case X86::VUNPCKHPDrr:
2622 Opc = X86::VMOVHLPSrr;
2623 break;
2624 case X86::VMOVHLPSZrr:
2625 Opc = X86::VUNPCKHPDZ128rr;
2626 break;
2627 case X86::VUNPCKHPDZ128rr:
2628 Opc = X86::VMOVHLPSZrr;
2629 break;
2630 }
2631 WorkingMI = CloneIfNew(MI);
2632 WorkingMI->setDesc(get(Opcode: Opc));
2633 break;
2634 CASE_ND(CMOV16rr)
2635 CASE_ND(CMOV32rr)
2636 CASE_ND(CMOV64rr) {
2637 WorkingMI = CloneIfNew(MI);
2638 unsigned OpNo = MI.getDesc().getNumOperands() - 1;
2639 X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(i: OpNo).getImm());
2640 WorkingMI->getOperand(i: OpNo).setImm(X86::GetOppositeBranchCondition(CC));
2641 break;
2642 }
2643 case X86::VPTERNLOGDZrri:
2644 case X86::VPTERNLOGDZrmi:
2645 case X86::VPTERNLOGDZ128rri:
2646 case X86::VPTERNLOGDZ128rmi:
2647 case X86::VPTERNLOGDZ256rri:
2648 case X86::VPTERNLOGDZ256rmi:
2649 case X86::VPTERNLOGQZrri:
2650 case X86::VPTERNLOGQZrmi:
2651 case X86::VPTERNLOGQZ128rri:
2652 case X86::VPTERNLOGQZ128rmi:
2653 case X86::VPTERNLOGQZ256rri:
2654 case X86::VPTERNLOGQZ256rmi:
2655 case X86::VPTERNLOGDZrrik:
2656 case X86::VPTERNLOGDZ128rrik:
2657 case X86::VPTERNLOGDZ256rrik:
2658 case X86::VPTERNLOGQZrrik:
2659 case X86::VPTERNLOGQZ128rrik:
2660 case X86::VPTERNLOGQZ256rrik:
2661 case X86::VPTERNLOGDZrrikz:
2662 case X86::VPTERNLOGDZrmikz:
2663 case X86::VPTERNLOGDZ128rrikz:
2664 case X86::VPTERNLOGDZ128rmikz:
2665 case X86::VPTERNLOGDZ256rrikz:
2666 case X86::VPTERNLOGDZ256rmikz:
2667 case X86::VPTERNLOGQZrrikz:
2668 case X86::VPTERNLOGQZrmikz:
2669 case X86::VPTERNLOGQZ128rrikz:
2670 case X86::VPTERNLOGQZ128rmikz:
2671 case X86::VPTERNLOGQZ256rrikz:
2672 case X86::VPTERNLOGQZ256rmikz:
2673 case X86::VPTERNLOGDZ128rmbi:
2674 case X86::VPTERNLOGDZ256rmbi:
2675 case X86::VPTERNLOGDZrmbi:
2676 case X86::VPTERNLOGQZ128rmbi:
2677 case X86::VPTERNLOGQZ256rmbi:
2678 case X86::VPTERNLOGQZrmbi:
2679 case X86::VPTERNLOGDZ128rmbikz:
2680 case X86::VPTERNLOGDZ256rmbikz:
2681 case X86::VPTERNLOGDZrmbikz:
2682 case X86::VPTERNLOGQZ128rmbikz:
2683 case X86::VPTERNLOGQZ256rmbikz:
2684 case X86::VPTERNLOGQZrmbikz: {
2685 WorkingMI = CloneIfNew(MI);
2686 commuteVPTERNLOG(MI&: *WorkingMI, SrcOpIdx1: OpIdx1, SrcOpIdx2: OpIdx2);
2687 break;
2688 }
2689 default:
2690 if (isCommutableVPERMV3Instruction(Opcode: Opc)) {
2691 WorkingMI = CloneIfNew(MI);
2692 WorkingMI->setDesc(get(Opcode: getCommutedVPERMV3Opcode(Opcode: Opc)));
2693 break;
2694 }
2695
2696 if (auto *FMA3Group = getFMA3Group(Opcode: Opc, TSFlags: MI.getDesc().TSFlags)) {
2697 WorkingMI = CloneIfNew(MI);
2698 WorkingMI->setDesc(
2699 get(Opcode: getFMA3OpcodeToCommuteOperands(MI, SrcOpIdx1: OpIdx1, SrcOpIdx2: OpIdx2, FMA3Group: *FMA3Group)));
2700 break;
2701 }
2702 }
2703 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2704}
2705
2706bool X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI,
2707 unsigned &SrcOpIdx1,
2708 unsigned &SrcOpIdx2,
2709 bool IsIntrinsic) const {
2710 uint64_t TSFlags = MI.getDesc().TSFlags;
2711
2712 unsigned FirstCommutableVecOp = 1;
2713 unsigned LastCommutableVecOp = 3;
2714 unsigned KMaskOp = -1U;
2715 if (X86II::isKMasked(TSFlags)) {
2716 // For k-zero-masked operations it is Ok to commute the first vector
2717 // operand. Unless this is an intrinsic instruction.
2718 // For regular k-masked operations a conservative choice is done as the
2719 // elements of the first vector operand, for which the corresponding bit
2720 // in the k-mask operand is set to 0, are copied to the result of the
2721 // instruction.
2722 // TODO/FIXME: The commute still may be legal if it is known that the
2723 // k-mask operand is set to either all ones or all zeroes.
2724 // It is also Ok to commute the 1st operand if all users of MI use only
2725 // the elements enabled by the k-mask operand. For example,
2726 // v4 = VFMADD213PSZrk v1, k, v2, v3; // v1[i] = k[i] ? v2[i]*v1[i]+v3[i]
2727 // : v1[i];
2728 // VMOVAPSZmrk <mem_addr>, k, v4; // this is the ONLY user of v4 ->
2729 // // Ok, to commute v1 in FMADD213PSZrk.
2730
2731 // The k-mask operand has index = 2 for masked and zero-masked operations.
2732 KMaskOp = 2;
2733
2734 // The operand with index = 1 is used as a source for those elements for
2735 // which the corresponding bit in the k-mask is set to 0.
2736 if (X86II::isKMergeMasked(TSFlags) || IsIntrinsic)
2737 FirstCommutableVecOp = 3;
2738
2739 LastCommutableVecOp++;
2740 } else if (IsIntrinsic) {
2741 // Commuting the first operand of an intrinsic instruction isn't possible
2742 // unless we can prove that only the lowest element of the result is used.
2743 FirstCommutableVecOp = 2;
2744 }
2745
2746 if (isMem(MI, Op: LastCommutableVecOp))
2747 LastCommutableVecOp--;
2748
2749 // Only the first RegOpsNum operands are commutable.
2750 // Also, the value 'CommuteAnyOperandIndex' is valid here as it means
2751 // that the operand is not specified/fixed.
2752 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
2753 (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp ||
2754 SrcOpIdx1 == KMaskOp))
2755 return false;
2756 if (SrcOpIdx2 != CommuteAnyOperandIndex &&
2757 (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp ||
2758 SrcOpIdx2 == KMaskOp))
2759 return false;
2760
2761 // Look for two different register operands assumed to be commutable
2762 // regardless of the FMA opcode. The FMA opcode is adjusted later.
2763 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
2764 SrcOpIdx2 == CommuteAnyOperandIndex) {
2765 unsigned CommutableOpIdx2 = SrcOpIdx2;
2766
2767 // At least one of operands to be commuted is not specified and
2768 // this method is free to choose appropriate commutable operands.
2769 if (SrcOpIdx1 == SrcOpIdx2)
2770 // Both of operands are not fixed. By default set one of commutable
2771 // operands to the last register operand of the instruction.
2772 CommutableOpIdx2 = LastCommutableVecOp;
2773 else if (SrcOpIdx2 == CommuteAnyOperandIndex)
2774 // Only one of operands is not fixed.
2775 CommutableOpIdx2 = SrcOpIdx1;
2776
2777 // CommutableOpIdx2 is well defined now. Let's choose another commutable
2778 // operand and assign its index to CommutableOpIdx1.
2779 Register Op2Reg = MI.getOperand(i: CommutableOpIdx2).getReg();
2780
2781 unsigned CommutableOpIdx1;
2782 for (CommutableOpIdx1 = LastCommutableVecOp;
2783 CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) {
2784 // Just ignore and skip the k-mask operand.
2785 if (CommutableOpIdx1 == KMaskOp)
2786 continue;
2787
2788 // The commuted operands must have different registers.
2789 // Otherwise, the commute transformation does not change anything and
2790 // is useless then.
2791 if (Op2Reg != MI.getOperand(i: CommutableOpIdx1).getReg())
2792 break;
2793 }
2794
2795 // No appropriate commutable operands were found.
2796 if (CommutableOpIdx1 < FirstCommutableVecOp)
2797 return false;
2798
2799 // Assign the found pair of commutable indices to SrcOpIdx1 and SrcOpidx2
2800 // to return those values.
2801 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
2802 CommutableOpIdx2))
2803 return false;
2804 }
2805
2806 return true;
2807}
2808
2809bool X86InstrInfo::findCommutedOpIndices(const MachineInstr &MI,
2810 unsigned &SrcOpIdx1,
2811 unsigned &SrcOpIdx2) const {
2812 const MCInstrDesc &Desc = MI.getDesc();
2813 if (!Desc.isCommutable())
2814 return false;
2815
2816 switch (MI.getOpcode()) {
2817 case X86::CMPSDrri:
2818 case X86::CMPSSrri:
2819 case X86::CMPPDrri:
2820 case X86::CMPPSrri:
2821 case X86::VCMPSDrri:
2822 case X86::VCMPSSrri:
2823 case X86::VCMPPDrri:
2824 case X86::VCMPPSrri:
2825 case X86::VCMPPDYrri:
2826 case X86::VCMPPSYrri:
2827 case X86::VCMPSDZrri:
2828 case X86::VCMPSSZrri:
2829 case X86::VCMPPDZrri:
2830 case X86::VCMPPSZrri:
2831 case X86::VCMPSHZrri:
2832 case X86::VCMPPHZrri:
2833 case X86::VCMPPHZ128rri:
2834 case X86::VCMPPHZ256rri:
2835 case X86::VCMPPDZ128rri:
2836 case X86::VCMPPSZ128rri:
2837 case X86::VCMPPDZ256rri:
2838 case X86::VCMPPSZ256rri:
2839 case X86::VCMPPDZrrik:
2840 case X86::VCMPPSZrrik:
2841 case X86::VCMPPHZrrik:
2842 case X86::VCMPPDZ128rrik:
2843 case X86::VCMPPSZ128rrik:
2844 case X86::VCMPPHZ128rrik:
2845 case X86::VCMPPDZ256rrik:
2846 case X86::VCMPPSZ256rrik:
2847 case X86::VCMPPHZ256rrik: {
2848 unsigned OpOffset = X86II::isKMasked(TSFlags: Desc.TSFlags) ? 1 : 0;
2849
2850 // Float comparison can be safely commuted for
2851 // Ordered/Unordered/Equal/NotEqual tests
2852 unsigned Imm = MI.getOperand(i: 3 + OpOffset).getImm() & 0x7;
2853 switch (Imm) {
2854 default:
2855 // EVEX versions can be commuted.
2856 if ((Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX)
2857 break;
2858 return false;
2859 case 0x00: // EQUAL
2860 case 0x03: // UNORDERED
2861 case 0x04: // NOT EQUAL
2862 case 0x07: // ORDERED
2863 break;
2864 }
2865
2866 // The indices of the commutable operands are 1 and 2 (or 2 and 3
2867 // when masked).
2868 // Assign them to the returned operand indices here.
2869 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1 + OpOffset,
2870 CommutableOpIdx2: 2 + OpOffset);
2871 }
2872 case X86::MOVSSrr:
2873 // X86::MOVSDrr is always commutable. MOVSS is only commutable if we can
2874 // form sse4.1 blend. We assume VMOVSSrr/VMOVSDrr is always commutable since
2875 // AVX implies sse4.1.
2876 if (Subtarget.hasSSE41())
2877 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2878 return false;
2879 case X86::SHUFPDrri:
2880 // We can commute this to MOVSD.
2881 if (MI.getOperand(i: 3).getImm() == 0x02)
2882 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2883 return false;
2884 case X86::MOVHLPSrr:
2885 case X86::UNPCKHPDrr:
2886 case X86::VMOVHLPSrr:
2887 case X86::VUNPCKHPDrr:
2888 case X86::VMOVHLPSZrr:
2889 case X86::VUNPCKHPDZ128rr:
2890 if (Subtarget.hasSSE2())
2891 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2892 return false;
2893 case X86::VPTERNLOGDZrri:
2894 case X86::VPTERNLOGDZrmi:
2895 case X86::VPTERNLOGDZ128rri:
2896 case X86::VPTERNLOGDZ128rmi:
2897 case X86::VPTERNLOGDZ256rri:
2898 case X86::VPTERNLOGDZ256rmi:
2899 case X86::VPTERNLOGQZrri:
2900 case X86::VPTERNLOGQZrmi:
2901 case X86::VPTERNLOGQZ128rri:
2902 case X86::VPTERNLOGQZ128rmi:
2903 case X86::VPTERNLOGQZ256rri:
2904 case X86::VPTERNLOGQZ256rmi:
2905 case X86::VPTERNLOGDZrrik:
2906 case X86::VPTERNLOGDZ128rrik:
2907 case X86::VPTERNLOGDZ256rrik:
2908 case X86::VPTERNLOGQZrrik:
2909 case X86::VPTERNLOGQZ128rrik:
2910 case X86::VPTERNLOGQZ256rrik:
2911 case X86::VPTERNLOGDZrrikz:
2912 case X86::VPTERNLOGDZrmikz:
2913 case X86::VPTERNLOGDZ128rrikz:
2914 case X86::VPTERNLOGDZ128rmikz:
2915 case X86::VPTERNLOGDZ256rrikz:
2916 case X86::VPTERNLOGDZ256rmikz:
2917 case X86::VPTERNLOGQZrrikz:
2918 case X86::VPTERNLOGQZrmikz:
2919 case X86::VPTERNLOGQZ128rrikz:
2920 case X86::VPTERNLOGQZ128rmikz:
2921 case X86::VPTERNLOGQZ256rrikz:
2922 case X86::VPTERNLOGQZ256rmikz:
2923 case X86::VPTERNLOGDZ128rmbi:
2924 case X86::VPTERNLOGDZ256rmbi:
2925 case X86::VPTERNLOGDZrmbi:
2926 case X86::VPTERNLOGQZ128rmbi:
2927 case X86::VPTERNLOGQZ256rmbi:
2928 case X86::VPTERNLOGQZrmbi:
2929 case X86::VPTERNLOGDZ128rmbikz:
2930 case X86::VPTERNLOGDZ256rmbikz:
2931 case X86::VPTERNLOGDZrmbikz:
2932 case X86::VPTERNLOGQZ128rmbikz:
2933 case X86::VPTERNLOGQZ256rmbikz:
2934 case X86::VPTERNLOGQZrmbikz:
2935 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2936 case X86::VPDPWSSDYrr:
2937 case X86::VPDPWSSDrr:
2938 case X86::VPDPWSSDSYrr:
2939 case X86::VPDPWSSDSrr:
2940 case X86::VPDPWUUDrr:
2941 case X86::VPDPWUUDYrr:
2942 case X86::VPDPWUUDSrr:
2943 case X86::VPDPWUUDSYrr:
2944 case X86::VPDPBSSDSrr:
2945 case X86::VPDPBSSDSYrr:
2946 case X86::VPDPBSSDrr:
2947 case X86::VPDPBSSDYrr:
2948 case X86::VPDPBUUDSrr:
2949 case X86::VPDPBUUDSYrr:
2950 case X86::VPDPBUUDrr:
2951 case X86::VPDPBUUDYrr:
2952 case X86::VPDPBSSDSZ128rr:
2953 case X86::VPDPBSSDSZ128rrk:
2954 case X86::VPDPBSSDSZ128rrkz:
2955 case X86::VPDPBSSDSZ256rr:
2956 case X86::VPDPBSSDSZ256rrk:
2957 case X86::VPDPBSSDSZ256rrkz:
2958 case X86::VPDPBSSDSZrr:
2959 case X86::VPDPBSSDSZrrk:
2960 case X86::VPDPBSSDSZrrkz:
2961 case X86::VPDPBSSDZ128rr:
2962 case X86::VPDPBSSDZ128rrk:
2963 case X86::VPDPBSSDZ128rrkz:
2964 case X86::VPDPBSSDZ256rr:
2965 case X86::VPDPBSSDZ256rrk:
2966 case X86::VPDPBSSDZ256rrkz:
2967 case X86::VPDPBSSDZrr:
2968 case X86::VPDPBSSDZrrk:
2969 case X86::VPDPBSSDZrrkz:
2970 case X86::VPDPBUUDSZ128rr:
2971 case X86::VPDPBUUDSZ128rrk:
2972 case X86::VPDPBUUDSZ128rrkz:
2973 case X86::VPDPBUUDSZ256rr:
2974 case X86::VPDPBUUDSZ256rrk:
2975 case X86::VPDPBUUDSZ256rrkz:
2976 case X86::VPDPBUUDSZrr:
2977 case X86::VPDPBUUDSZrrk:
2978 case X86::VPDPBUUDSZrrkz:
2979 case X86::VPDPBUUDZ128rr:
2980 case X86::VPDPBUUDZ128rrk:
2981 case X86::VPDPBUUDZ128rrkz:
2982 case X86::VPDPBUUDZ256rr:
2983 case X86::VPDPBUUDZ256rrk:
2984 case X86::VPDPBUUDZ256rrkz:
2985 case X86::VPDPBUUDZrr:
2986 case X86::VPDPBUUDZrrk:
2987 case X86::VPDPBUUDZrrkz:
2988 case X86::VPDPWSSDZ128rr:
2989 case X86::VPDPWSSDZ128rrk:
2990 case X86::VPDPWSSDZ128rrkz:
2991 case X86::VPDPWSSDZ256rr:
2992 case X86::VPDPWSSDZ256rrk:
2993 case X86::VPDPWSSDZ256rrkz:
2994 case X86::VPDPWSSDZrr:
2995 case X86::VPDPWSSDZrrk:
2996 case X86::VPDPWSSDZrrkz:
2997 case X86::VPDPWSSDSZ128rr:
2998 case X86::VPDPWSSDSZ128rrk:
2999 case X86::VPDPWSSDSZ128rrkz:
3000 case X86::VPDPWSSDSZ256rr:
3001 case X86::VPDPWSSDSZ256rrk:
3002 case X86::VPDPWSSDSZ256rrkz:
3003 case X86::VPDPWSSDSZrr:
3004 case X86::VPDPWSSDSZrrk:
3005 case X86::VPDPWSSDSZrrkz:
3006 case X86::VPDPWUUDZ128rr:
3007 case X86::VPDPWUUDZ128rrk:
3008 case X86::VPDPWUUDZ128rrkz:
3009 case X86::VPDPWUUDZ256rr:
3010 case X86::VPDPWUUDZ256rrk:
3011 case X86::VPDPWUUDZ256rrkz:
3012 case X86::VPDPWUUDZrr:
3013 case X86::VPDPWUUDZrrk:
3014 case X86::VPDPWUUDZrrkz:
3015 case X86::VPDPWUUDSZ128rr:
3016 case X86::VPDPWUUDSZ128rrk:
3017 case X86::VPDPWUUDSZ128rrkz:
3018 case X86::VPDPWUUDSZ256rr:
3019 case X86::VPDPWUUDSZ256rrk:
3020 case X86::VPDPWUUDSZ256rrkz:
3021 case X86::VPDPWUUDSZrr:
3022 case X86::VPDPWUUDSZrrk:
3023 case X86::VPDPWUUDSZrrkz:
3024 case X86::VPMADD52HUQrr:
3025 case X86::VPMADD52HUQYrr:
3026 case X86::VPMADD52HUQZ128r:
3027 case X86::VPMADD52HUQZ128rk:
3028 case X86::VPMADD52HUQZ128rkz:
3029 case X86::VPMADD52HUQZ256r:
3030 case X86::VPMADD52HUQZ256rk:
3031 case X86::VPMADD52HUQZ256rkz:
3032 case X86::VPMADD52HUQZr:
3033 case X86::VPMADD52HUQZrk:
3034 case X86::VPMADD52HUQZrkz:
3035 case X86::VPMADD52LUQrr:
3036 case X86::VPMADD52LUQYrr:
3037 case X86::VPMADD52LUQZ128r:
3038 case X86::VPMADD52LUQZ128rk:
3039 case X86::VPMADD52LUQZ128rkz:
3040 case X86::VPMADD52LUQZ256r:
3041 case X86::VPMADD52LUQZ256rk:
3042 case X86::VPMADD52LUQZ256rkz:
3043 case X86::VPMADD52LUQZr:
3044 case X86::VPMADD52LUQZrk:
3045 case X86::VPMADD52LUQZrkz:
3046 case X86::VFMADDCPHZr:
3047 case X86::VFMADDCPHZrk:
3048 case X86::VFMADDCPHZrkz:
3049 case X86::VFMADDCPHZ128r:
3050 case X86::VFMADDCPHZ128rk:
3051 case X86::VFMADDCPHZ128rkz:
3052 case X86::VFMADDCPHZ256r:
3053 case X86::VFMADDCPHZ256rk:
3054 case X86::VFMADDCPHZ256rkz:
3055 case X86::VFMADDCSHZr:
3056 case X86::VFMADDCSHZrk:
3057 case X86::VFMADDCSHZrkz: {
3058 unsigned CommutableOpIdx1 = 2;
3059 unsigned CommutableOpIdx2 = 3;
3060 if (X86II::isKMasked(TSFlags: Desc.TSFlags)) {
3061 // Skip the mask register.
3062 ++CommutableOpIdx1;
3063 ++CommutableOpIdx2;
3064 }
3065 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
3066 CommutableOpIdx2))
3067 return false;
3068 if (!MI.getOperand(i: SrcOpIdx1).isReg() || !MI.getOperand(i: SrcOpIdx2).isReg())
3069 // No idea.
3070 return false;
3071 return true;
3072 }
3073
3074 default:
3075 const X86InstrFMA3Group *FMA3Group =
3076 getFMA3Group(Opcode: MI.getOpcode(), TSFlags: MI.getDesc().TSFlags);
3077 if (FMA3Group)
3078 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2,
3079 IsIntrinsic: FMA3Group->isIntrinsic());
3080
3081 // Handled masked instructions since we need to skip over the mask input
3082 // and the preserved input.
3083 if (X86II::isKMasked(TSFlags: Desc.TSFlags)) {
3084 // First assume that the first input is the mask operand and skip past it.
3085 unsigned CommutableOpIdx1 = Desc.getNumDefs() + 1;
3086 unsigned CommutableOpIdx2 = Desc.getNumDefs() + 2;
3087 // Check if the first input is tied. If there isn't one then we only
3088 // need to skip the mask operand which we did above.
3089 if ((MI.getDesc().getOperandConstraint(OpNum: Desc.getNumDefs(),
3090 Constraint: MCOI::TIED_TO) != -1)) {
3091 // If this is zero masking instruction with a tied operand, we need to
3092 // move the first index back to the first input since this must
3093 // be a 3 input instruction and we want the first two non-mask inputs.
3094 // Otherwise this is a 2 input instruction with a preserved input and
3095 // mask, so we need to move the indices to skip one more input.
3096 if (X86II::isKMergeMasked(TSFlags: Desc.TSFlags)) {
3097 ++CommutableOpIdx1;
3098 ++CommutableOpIdx2;
3099 } else {
3100 --CommutableOpIdx1;
3101 }
3102 }
3103
3104 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
3105 CommutableOpIdx2))
3106 return false;
3107
3108 if (!MI.getOperand(i: SrcOpIdx1).isReg() ||
3109 !MI.getOperand(i: SrcOpIdx2).isReg())
3110 // No idea.
3111 return false;
3112 return true;
3113 }
3114
3115 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
3116 }
3117 return false;
3118}
3119
3120static bool isConvertibleLEA(MachineInstr *MI) {
3121 unsigned Opcode = MI->getOpcode();
3122 if (Opcode != X86::LEA32r && Opcode != X86::LEA64r &&
3123 Opcode != X86::LEA64_32r)
3124 return false;
3125
3126 const MachineOperand &Scale = MI->getOperand(i: 1 + X86::AddrScaleAmt);
3127 const MachineOperand &Disp = MI->getOperand(i: 1 + X86::AddrDisp);
3128 const MachineOperand &Segment = MI->getOperand(i: 1 + X86::AddrSegmentReg);
3129
3130 if (Segment.getReg() != 0 || !Disp.isImm() || Disp.getImm() != 0 ||
3131 Scale.getImm() > 1)
3132 return false;
3133
3134 return true;
3135}
3136
3137bool X86InstrInfo::hasCommutePreference(MachineInstr &MI, bool &Commute) const {
3138 // Currently we're interested in following sequence only.
3139 // r3 = lea r1, r2
3140 // r5 = add r3, r4
3141 // Both r3 and r4 are killed in add, we hope the add instruction has the
3142 // operand order
3143 // r5 = add r4, r3
3144 // So later in X86FixupLEAs the lea instruction can be rewritten as add.
3145 unsigned Opcode = MI.getOpcode();
3146 if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr)
3147 return false;
3148
3149 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3150 Register Reg1 = MI.getOperand(i: 1).getReg();
3151 Register Reg2 = MI.getOperand(i: 2).getReg();
3152
3153 // Check if Reg1 comes from LEA in the same MBB.
3154 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg: Reg1)) {
3155 if (isConvertibleLEA(MI: Inst) && Inst->getParent() == MI.getParent()) {
3156 Commute = true;
3157 return true;
3158 }
3159 }
3160
3161 // Check if Reg2 comes from LEA in the same MBB.
3162 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg: Reg2)) {
3163 if (isConvertibleLEA(MI: Inst) && Inst->getParent() == MI.getParent()) {
3164 Commute = false;
3165 return true;
3166 }
3167 }
3168
3169 return false;
3170}
3171
3172int X86::getCondSrcNoFromDesc(const MCInstrDesc &MCID) {
3173 unsigned Opcode = MCID.getOpcode();
3174 if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isSETZUCC(Opcode) ||
3175 X86::isCMOVCC(Opcode) || X86::isCFCMOVCC(Opcode) ||
3176 X86::isCCMPCC(Opcode) || X86::isCTESTCC(Opcode)))
3177 return -1;
3178 // Assume that condition code is always the last use operand.
3179 unsigned NumUses = MCID.getNumOperands() - MCID.getNumDefs();
3180 return NumUses - 1;
3181}
3182
3183X86::CondCode X86::getCondFromMI(const MachineInstr &MI) {
3184 const MCInstrDesc &MCID = MI.getDesc();
3185 int CondNo = getCondSrcNoFromDesc(MCID);
3186 if (CondNo < 0)
3187 return X86::COND_INVALID;
3188 CondNo += MCID.getNumDefs();
3189 return static_cast<X86::CondCode>(MI.getOperand(i: CondNo).getImm());
3190}
3191
3192X86::CondCode X86::getCondFromBranch(const MachineInstr &MI) {
3193 return X86::isJCC(Opcode: MI.getOpcode()) ? X86::getCondFromMI(MI)
3194 : X86::COND_INVALID;
3195}
3196
3197X86::CondCode X86::getCondFromSETCC(const MachineInstr &MI) {
3198 return X86::isSETCC(Opcode: MI.getOpcode()) || X86::isSETZUCC(Opcode: MI.getOpcode())
3199 ? X86::getCondFromMI(MI)
3200 : X86::COND_INVALID;
3201}
3202
3203X86::CondCode X86::getCondFromCMov(const MachineInstr &MI) {
3204 return X86::isCMOVCC(Opcode: MI.getOpcode()) ? X86::getCondFromMI(MI)
3205 : X86::COND_INVALID;
3206}
3207
3208X86::CondCode X86::getCondFromCFCMov(const MachineInstr &MI) {
3209 return X86::isCFCMOVCC(Opcode: MI.getOpcode()) ? X86::getCondFromMI(MI)
3210 : X86::COND_INVALID;
3211}
3212
3213X86::CondCode X86::getCondFromCCMP(const MachineInstr &MI) {
3214 return X86::isCCMPCC(Opcode: MI.getOpcode()) || X86::isCTESTCC(Opcode: MI.getOpcode())
3215 ? X86::getCondFromMI(MI)
3216 : X86::COND_INVALID;
3217}
3218
3219int X86::getCCMPCondFlagsFromCondCode(X86::CondCode CC) {
3220 // CCMP/CTEST has two conditional operands:
3221 // - SCC: source conditonal code (same as CMOV)
3222 // - DCF: destination conditional flags, which has 4 valid bits
3223 //
3224 // +----+----+----+----+
3225 // | OF | SF | ZF | CF |
3226 // +----+----+----+----+
3227 //
3228 // If SCC(source conditional code) evaluates to false, CCMP/CTEST will updates
3229 // the conditional flags by as follows:
3230 //
3231 // OF = DCF.OF
3232 // SF = DCF.SF
3233 // ZF = DCF.ZF
3234 // CF = DCF.CF
3235 // PF = DCF.CF
3236 // AF = 0 (Auxiliary Carry Flag)
3237 //
3238 // Otherwise, the CMP or TEST is executed and it updates the
3239 // CSPAZO flags normally.
3240 //
3241 // NOTE:
3242 // If SCC = P, then SCC evaluates to true regardless of the CSPAZO value.
3243 // If SCC = NP, then SCC evaluates to false regardless of the CSPAZO value.
3244
3245 enum { CF = 1, ZF = 2, SF = 4, OF = 8, PF = CF };
3246
3247 switch (CC) {
3248 default:
3249 llvm_unreachable("Illegal condition code!");
3250 case X86::COND_NO:
3251 case X86::COND_NE:
3252 case X86::COND_GE:
3253 case X86::COND_G:
3254 case X86::COND_AE:
3255 case X86::COND_A:
3256 case X86::COND_NS:
3257 case X86::COND_NP:
3258 return 0;
3259 case X86::COND_O:
3260 return OF;
3261 case X86::COND_B:
3262 case X86::COND_BE:
3263 return CF;
3264 break;
3265 case X86::COND_E:
3266 case X86::COND_LE:
3267 return ZF;
3268 case X86::COND_S:
3269 case X86::COND_L:
3270 return SF;
3271 case X86::COND_P:
3272 return PF;
3273 }
3274}
3275
3276#define GET_X86_NF_TRANSFORM_TABLE
3277#define GET_X86_ND2NONND_TABLE
3278#include "X86GenInstrMapping.inc"
3279
3280static unsigned getNewOpcFromTable(ArrayRef<X86TableEntry> Table,
3281 unsigned Opc) {
3282 const auto I = llvm::lower_bound(Range&: Table, Value&: Opc);
3283 return (I == Table.end() || I->OldOpc != Opc) ? 0U : I->NewOpc;
3284}
3285unsigned X86::getNFVariant(unsigned Opc) {
3286#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
3287 // Make sure the tables are sorted.
3288 static std::atomic<bool> NFTableChecked(false);
3289 if (!NFTableChecked.load(std::memory_order_relaxed)) {
3290 assert(llvm::is_sorted(X86NFTransformTable) &&
3291 "X86NFTransformTable is not sorted!");
3292 NFTableChecked.store(true, std::memory_order_relaxed);
3293 }
3294#endif
3295 return getNewOpcFromTable(Table: X86NFTransformTable, Opc);
3296}
3297
3298unsigned X86::getNFVariantIfClobberRemovable(const MachineInstr &MI,
3299 const TargetRegisterInfo *TRI) {
3300 if (!MI.registerDefIsDead(Reg: X86::EFLAGS, TRI))
3301 return 0;
3302 // For the instructions are ADDrm/ADDmr with relocation, we'll skip the
3303 // optimization for replacing non-NF with NF. This is to keep backward
3304 // compatiblity with old version of linkers without APX relocation type
3305 // support on Linux OS.
3306 if (!X86EnableAPXForRelocation && isAddMemInstrWithRelocation(MI))
3307 return 0;
3308 return X86::getNFVariant(Opc: MI.getOpcode());
3309}
3310
3311unsigned X86::getNonNDVariant(unsigned Opc) {
3312#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
3313 // Make sure the tables are sorted.
3314 static std::atomic<bool> NDTableChecked(false);
3315 if (!NDTableChecked.load(std::memory_order_relaxed)) {
3316 assert(llvm::is_sorted(X86ND2NonNDTable) &&
3317 "X86ND2NonNDTableis not sorted!");
3318 NDTableChecked.store(true, std::memory_order_relaxed);
3319 }
3320#endif
3321 return getNewOpcFromTable(Table: X86ND2NonNDTable, Opc);
3322}
3323
3324/// Return the inverse of the specified condition,
3325/// e.g. turning COND_E to COND_NE.
3326X86::CondCode X86::GetOppositeBranchCondition(X86::CondCode CC) {
3327 switch (CC) {
3328 default:
3329 llvm_unreachable("Illegal condition code!");
3330 case X86::COND_E:
3331 return X86::COND_NE;
3332 case X86::COND_NE:
3333 return X86::COND_E;
3334 case X86::COND_L:
3335 return X86::COND_GE;
3336 case X86::COND_LE:
3337 return X86::COND_G;
3338 case X86::COND_G:
3339 return X86::COND_LE;
3340 case X86::COND_GE:
3341 return X86::COND_L;
3342 case X86::COND_B:
3343 return X86::COND_AE;
3344 case X86::COND_BE:
3345 return X86::COND_A;
3346 case X86::COND_A:
3347 return X86::COND_BE;
3348 case X86::COND_AE:
3349 return X86::COND_B;
3350 case X86::COND_S:
3351 return X86::COND_NS;
3352 case X86::COND_NS:
3353 return X86::COND_S;
3354 case X86::COND_P:
3355 return X86::COND_NP;
3356 case X86::COND_NP:
3357 return X86::COND_P;
3358 case X86::COND_O:
3359 return X86::COND_NO;
3360 case X86::COND_NO:
3361 return X86::COND_O;
3362 case X86::COND_NE_OR_P:
3363 return X86::COND_E_AND_NP;
3364 case X86::COND_E_AND_NP:
3365 return X86::COND_NE_OR_P;
3366 }
3367}
3368
3369/// Assuming the flags are set by MI(a,b), return the condition code if we
3370/// modify the instructions such that flags are set by MI(b,a).
3371static X86::CondCode getSwappedCondition(X86::CondCode CC) {
3372 switch (CC) {
3373 default:
3374 return X86::COND_INVALID;
3375 case X86::COND_E:
3376 return X86::COND_E;
3377 case X86::COND_NE:
3378 return X86::COND_NE;
3379 case X86::COND_L:
3380 return X86::COND_G;
3381 case X86::COND_LE:
3382 return X86::COND_GE;
3383 case X86::COND_G:
3384 return X86::COND_L;
3385 case X86::COND_GE:
3386 return X86::COND_LE;
3387 case X86::COND_B:
3388 return X86::COND_A;
3389 case X86::COND_BE:
3390 return X86::COND_AE;
3391 case X86::COND_A:
3392 return X86::COND_B;
3393 case X86::COND_AE:
3394 return X86::COND_BE;
3395 }
3396}
3397
3398std::pair<X86::CondCode, bool>
3399X86::getX86ConditionCode(CmpInst::Predicate Predicate) {
3400 X86::CondCode CC = X86::COND_INVALID;
3401 bool NeedSwap = false;
3402 switch (Predicate) {
3403 default:
3404 break;
3405 // Floating-point Predicates
3406 case CmpInst::FCMP_UEQ:
3407 CC = X86::COND_E;
3408 break;
3409 case CmpInst::FCMP_OLT:
3410 NeedSwap = true;
3411 [[fallthrough]];
3412 case CmpInst::FCMP_OGT:
3413 CC = X86::COND_A;
3414 break;
3415 case CmpInst::FCMP_OLE:
3416 NeedSwap = true;
3417 [[fallthrough]];
3418 case CmpInst::FCMP_OGE:
3419 CC = X86::COND_AE;
3420 break;
3421 case CmpInst::FCMP_UGT:
3422 NeedSwap = true;
3423 [[fallthrough]];
3424 case CmpInst::FCMP_ULT:
3425 CC = X86::COND_B;
3426 break;
3427 case CmpInst::FCMP_UGE:
3428 NeedSwap = true;
3429 [[fallthrough]];
3430 case CmpInst::FCMP_ULE:
3431 CC = X86::COND_BE;
3432 break;
3433 case CmpInst::FCMP_ONE:
3434 CC = X86::COND_NE;
3435 break;
3436 case CmpInst::FCMP_UNO:
3437 CC = X86::COND_P;
3438 break;
3439 case CmpInst::FCMP_ORD:
3440 CC = X86::COND_NP;
3441 break;
3442 case CmpInst::FCMP_OEQ:
3443 [[fallthrough]];
3444 case CmpInst::FCMP_UNE:
3445 CC = X86::COND_INVALID;
3446 break;
3447
3448 // Integer Predicates
3449 case CmpInst::ICMP_EQ:
3450 CC = X86::COND_E;
3451 break;
3452 case CmpInst::ICMP_NE:
3453 CC = X86::COND_NE;
3454 break;
3455 case CmpInst::ICMP_UGT:
3456 CC = X86::COND_A;
3457 break;
3458 case CmpInst::ICMP_UGE:
3459 CC = X86::COND_AE;
3460 break;
3461 case CmpInst::ICMP_ULT:
3462 CC = X86::COND_B;
3463 break;
3464 case CmpInst::ICMP_ULE:
3465 CC = X86::COND_BE;
3466 break;
3467 case CmpInst::ICMP_SGT:
3468 CC = X86::COND_G;
3469 break;
3470 case CmpInst::ICMP_SGE:
3471 CC = X86::COND_GE;
3472 break;
3473 case CmpInst::ICMP_SLT:
3474 CC = X86::COND_L;
3475 break;
3476 case CmpInst::ICMP_SLE:
3477 CC = X86::COND_LE;
3478 break;
3479 }
3480
3481 return std::make_pair(x&: CC, y&: NeedSwap);
3482}
3483
3484/// Return a cmov opcode for the given register size in bytes, and operand type.
3485unsigned X86::getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand,
3486 bool HasNDD) {
3487 switch (RegBytes) {
3488 default:
3489 llvm_unreachable("Illegal register size!");
3490#define GET_ND_IF_ENABLED(OPC) (HasNDD ? OPC##_ND : OPC)
3491 case 2:
3492 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV16rm)
3493 : GET_ND_IF_ENABLED(X86::CMOV16rr);
3494 case 4:
3495 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV32rm)
3496 : GET_ND_IF_ENABLED(X86::CMOV32rr);
3497 case 8:
3498 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV64rm)
3499 : GET_ND_IF_ENABLED(X86::CMOV64rr);
3500 }
3501}
3502
3503unsigned X86::getMOVriOpcode(bool Use64BitReg, int64_t Imm) {
3504 if (!Use64BitReg)
3505 return X86::MOV32ri;
3506
3507 if (isUInt<32>(x: Imm))
3508 return X86::MOV32ri64;
3509 if (isInt<32>(x: Imm))
3510 return X86::MOV64ri32;
3511 return X86::MOV64ri;
3512}
3513
3514/// Get the VPCMP immediate for the given condition.
3515unsigned X86::getVPCMPImmForCond(ISD::CondCode CC) {
3516 switch (CC) {
3517 default:
3518 llvm_unreachable("Unexpected SETCC condition");
3519 case ISD::SETNE:
3520 return 4;
3521 case ISD::SETEQ:
3522 return 0;
3523 case ISD::SETULT:
3524 case ISD::SETLT:
3525 return 1;
3526 case ISD::SETUGT:
3527 case ISD::SETGT:
3528 return 6;
3529 case ISD::SETUGE:
3530 case ISD::SETGE:
3531 return 5;
3532 case ISD::SETULE:
3533 case ISD::SETLE:
3534 return 2;
3535 }
3536}
3537
3538/// Get the VPCMP immediate if the operands are swapped.
3539unsigned X86::getSwappedVPCMPImm(unsigned Imm) {
3540 switch (Imm) {
3541 default:
3542 llvm_unreachable("Unreachable!");
3543 case 0x01:
3544 Imm = 0x06;
3545 break; // LT -> NLE
3546 case 0x02:
3547 Imm = 0x05;
3548 break; // LE -> NLT
3549 case 0x05:
3550 Imm = 0x02;
3551 break; // NLT -> LE
3552 case 0x06:
3553 Imm = 0x01;
3554 break; // NLE -> LT
3555 case 0x00: // EQ
3556 case 0x03: // FALSE
3557 case 0x04: // NE
3558 case 0x07: // TRUE
3559 break;
3560 }
3561
3562 return Imm;
3563}
3564
3565/// Get the VPCOM immediate if the operands are swapped.
3566unsigned X86::getSwappedVPCOMImm(unsigned Imm) {
3567 switch (Imm) {
3568 default:
3569 llvm_unreachable("Unreachable!");
3570 case 0x00:
3571 Imm = 0x02;
3572 break; // LT -> GT
3573 case 0x01:
3574 Imm = 0x03;
3575 break; // LE -> GE
3576 case 0x02:
3577 Imm = 0x00;
3578 break; // GT -> LT
3579 case 0x03:
3580 Imm = 0x01;
3581 break; // GE -> LE
3582 case 0x04: // EQ
3583 case 0x05: // NE
3584 case 0x06: // FALSE
3585 case 0x07: // TRUE
3586 break;
3587 }
3588
3589 return Imm;
3590}
3591
3592/// Get the VCMP immediate if the operands are swapped.
3593unsigned X86::getSwappedVCMPImm(unsigned Imm) {
3594 // Only need the lower 2 bits to distinquish.
3595 switch (Imm & 0x3) {
3596 default:
3597 llvm_unreachable("Unreachable!");
3598 case 0x00:
3599 case 0x03:
3600 // EQ/NE/TRUE/FALSE/ORD/UNORD don't change immediate when commuted.
3601 break;
3602 case 0x01:
3603 case 0x02:
3604 // Need to toggle bits 3:0. Bit 4 stays the same.
3605 Imm ^= 0xf;
3606 break;
3607 }
3608
3609 return Imm;
3610}
3611
3612unsigned X86::getVectorRegisterWidth(const MCOperandInfo &Info) {
3613 if (Info.RegClass == X86::VR128RegClassID ||
3614 Info.RegClass == X86::VR128XRegClassID)
3615 return 128;
3616 if (Info.RegClass == X86::VR256RegClassID ||
3617 Info.RegClass == X86::VR256XRegClassID)
3618 return 256;
3619 if (Info.RegClass == X86::VR512RegClassID)
3620 return 512;
3621 llvm_unreachable("Unknown register class!");
3622}
3623
3624/// Return true if the Reg is X87 register.
3625static bool isX87Reg(Register Reg) {
3626 return (Reg == X86::FPCW || Reg == X86::FPSW ||
3627 (Reg >= X86::ST0 && Reg <= X86::ST7));
3628}
3629
3630/// check if the instruction is X87 instruction
3631bool X86::isX87Instruction(MachineInstr &MI) {
3632 // Call and inlineasm defs X87 register, so we special case it here because
3633 // otherwise calls are incorrectly flagged as x87 instructions
3634 // as a result.
3635 if (MI.isCall() || MI.isInlineAsm())
3636 return false;
3637 for (const MachineOperand &MO : MI.operands()) {
3638 if (!MO.isReg())
3639 continue;
3640 if (isX87Reg(Reg: MO.getReg()))
3641 return true;
3642 }
3643 return false;
3644}
3645
3646int X86::getFirstAddrOperandIdx(const MachineInstr &MI) {
3647 auto IsMemOp = [](const MCOperandInfo &OpInfo) {
3648 return OpInfo.OperandType == MCOI::OPERAND_MEMORY;
3649 };
3650
3651 const MCInstrDesc &Desc = MI.getDesc();
3652
3653 // Directly invoke the MC-layer routine for real (i.e., non-pseudo)
3654 // instructions (fast case).
3655 if (!X86II::isPseudo(TSFlags: Desc.TSFlags)) {
3656 int MemRefIdx = X86II::getMemoryOperandNo(TSFlags: Desc.TSFlags);
3657 if (MemRefIdx >= 0)
3658 return MemRefIdx + X86II::getOperandBias(Desc);
3659#ifdef EXPENSIVE_CHECKS
3660 assert(none_of(Desc.operands(), IsMemOp) &&
3661 "Got false negative from X86II::getMemoryOperandNo()!");
3662#endif
3663 return -1;
3664 }
3665
3666 // Otherwise, handle pseudo instructions by examining the type of their
3667 // operands (slow case). An instruction cannot have a memory reference if it
3668 // has fewer than AddrNumOperands (= 5) explicit operands.
3669 unsigned NumOps = Desc.getNumOperands();
3670 if (NumOps < X86::AddrNumOperands) {
3671#ifdef EXPENSIVE_CHECKS
3672 assert(none_of(Desc.operands(), IsMemOp) &&
3673 "Expected no operands to have OPERAND_MEMORY type!");
3674#endif
3675 return -1;
3676 }
3677
3678 // The first operand with type OPERAND_MEMORY indicates the start of a memory
3679 // reference. We expect the following AddrNumOperand-1 operands to also have
3680 // OPERAND_MEMORY type.
3681 for (unsigned I = 0, E = NumOps - X86::AddrNumOperands; I != E; ++I) {
3682 if (IsMemOp(Desc.operands()[I])) {
3683#ifdef EXPENSIVE_CHECKS
3684 assert(std::all_of(Desc.operands().begin() + I,
3685 Desc.operands().begin() + I + X86::AddrNumOperands,
3686 IsMemOp) &&
3687 "Expected all five operands in the memory reference to have "
3688 "OPERAND_MEMORY type!");
3689#endif
3690 return I;
3691 }
3692 }
3693
3694 return -1;
3695}
3696
3697const Constant *X86::getConstantFromPool(const MachineInstr &MI,
3698 unsigned OpNo) {
3699 assert(MI.getNumOperands() >= (OpNo + X86::AddrNumOperands) &&
3700 "Unexpected number of operands!");
3701
3702 const MachineOperand &Index = MI.getOperand(i: OpNo + X86::AddrIndexReg);
3703 if (!Index.isReg() || Index.getReg() != X86::NoRegister)
3704 return nullptr;
3705
3706 const MachineOperand &Disp = MI.getOperand(i: OpNo + X86::AddrDisp);
3707 if (!Disp.isCPI() || Disp.getOffset() != 0)
3708 return nullptr;
3709
3710 ArrayRef<MachineConstantPoolEntry> Constants =
3711 MI.getParent()->getParent()->getConstantPool()->getConstants();
3712 const MachineConstantPoolEntry &ConstantEntry = Constants[Disp.getIndex()];
3713
3714 // Bail if this is a machine constant pool entry, we won't be able to dig out
3715 // anything useful.
3716 if (ConstantEntry.isMachineConstantPoolEntry())
3717 return nullptr;
3718
3719 return ConstantEntry.Val.ConstVal;
3720}
3721
3722bool X86InstrInfo::isUnconditionalTailCall(const MachineInstr &MI) const {
3723 switch (MI.getOpcode()) {
3724 case X86::TCRETURNdi:
3725 case X86::TCRETURNri:
3726 case X86::TCRETURNmi:
3727 case X86::TCRETURNdi64:
3728 case X86::TCRETURNri64:
3729 case X86::TCRETURNri64_ImpCall:
3730 case X86::TCRETURNmi64:
3731 return true;
3732 default:
3733 return false;
3734 }
3735}
3736
3737bool X86InstrInfo::canMakeTailCallConditional(
3738 SmallVectorImpl<MachineOperand> &BranchCond,
3739 const MachineInstr &TailCall) const {
3740
3741 const MachineFunction *MF = TailCall.getMF();
3742
3743 if (MF->getTarget().getCodeModel() == CodeModel::Kernel) {
3744 // Kernel patches thunk calls in runtime, these should never be conditional.
3745 const MachineOperand &Target = TailCall.getOperand(i: 0);
3746 if (Target.isSymbol()) {
3747 StringRef Symbol(Target.getSymbolName());
3748 // this is currently only relevant to r11/kernel indirect thunk.
3749 if (Symbol == "__x86_indirect_thunk_r11")
3750 return false;
3751 }
3752 }
3753
3754 if (TailCall.getOpcode() != X86::TCRETURNdi &&
3755 TailCall.getOpcode() != X86::TCRETURNdi64) {
3756 // Only direct calls can be done with a conditional branch.
3757 return false;
3758 }
3759
3760 if (Subtarget.isTargetWin64() && MF->hasWinCFI()) {
3761 // Conditional tail calls confuse the Win64 unwinder.
3762 return false;
3763 }
3764
3765 assert(BranchCond.size() == 1);
3766 if (BranchCond[0].getImm() > X86::LAST_VALID_COND) {
3767 // Can't make a conditional tail call with this condition.
3768 return false;
3769 }
3770
3771 const X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>();
3772 if (X86FI->getTCReturnAddrDelta() != 0 ||
3773 TailCall.getOperand(i: 1).getImm() != 0) {
3774 // A conditional tail call cannot do any stack adjustment.
3775 return false;
3776 }
3777
3778 return true;
3779}
3780
3781void X86InstrInfo::replaceBranchWithTailCall(
3782 MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &BranchCond,
3783 const MachineInstr &TailCall) const {
3784 assert(canMakeTailCallConditional(BranchCond, TailCall));
3785
3786 MachineBasicBlock::iterator I = MBB.end();
3787 while (I != MBB.begin()) {
3788 --I;
3789 if (I->isDebugInstr())
3790 continue;
3791 if (!I->isBranch())
3792 assert(0 && "Can't find the branch to replace!");
3793
3794 X86::CondCode CC = X86::getCondFromBranch(MI: *I);
3795 assert(BranchCond.size() == 1);
3796 if (CC != BranchCond[0].getImm())
3797 continue;
3798
3799 break;
3800 }
3801
3802 unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc
3803 : X86::TCRETURNdi64cc;
3804
3805 auto MIB = BuildMI(BB&: MBB, I, MIMD: MBB.findDebugLoc(MBBI: I), MCID: get(Opcode: Opc));
3806 MIB->addOperand(Op: TailCall.getOperand(i: 0)); // Destination.
3807 MIB.addImm(Val: 0); // Stack offset (not used).
3808 MIB->addOperand(Op: BranchCond[0]); // Condition.
3809 MIB.copyImplicitOps(OtherMI: TailCall); // Regmask and (imp-used) parameters.
3810
3811 // Add implicit uses and defs of all live regs potentially clobbered by the
3812 // call. This way they still appear live across the call.
3813 LivePhysRegs LiveRegs(getRegisterInfo());
3814 LiveRegs.addLiveOuts(MBB);
3815 SmallVector<std::pair<MCPhysReg, const MachineOperand *>, 8> Clobbers;
3816 LiveRegs.stepForward(MI: *MIB, Clobbers);
3817 for (const auto &C : Clobbers) {
3818 MIB.addReg(RegNo: C.first, Flags: RegState::Implicit);
3819 MIB.addReg(RegNo: C.first, Flags: RegState::Implicit | RegState::Define);
3820 }
3821
3822 I->eraseFromParent();
3823}
3824
3825// Given a MBB and its TBB, find the FBB which was a fallthrough MBB (it may
3826// not be a fallthrough MBB now due to layout changes). Return nullptr if the
3827// fallthrough MBB cannot be identified.
3828static MachineBasicBlock *getFallThroughMBB(MachineBasicBlock *MBB,
3829 MachineBasicBlock *TBB) {
3830 // Look for non-EHPad successors other than TBB. If we find exactly one, it
3831 // is the fallthrough MBB. If we find zero, then TBB is both the target MBB
3832 // and fallthrough MBB. If we find more than one, we cannot identify the
3833 // fallthrough MBB and should return nullptr.
3834 MachineBasicBlock *FallthroughBB = nullptr;
3835 for (MachineBasicBlock *Succ : MBB->successors()) {
3836 if (Succ->isEHPad() || (Succ == TBB && FallthroughBB))
3837 continue;
3838 // Return a nullptr if we found more than one fallthrough successor.
3839 if (FallthroughBB && FallthroughBB != TBB)
3840 return nullptr;
3841 FallthroughBB = Succ;
3842 }
3843 return FallthroughBB;
3844}
3845
3846bool X86InstrInfo::analyzeBranchImpl(
3847 MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB,
3848 SmallVectorImpl<MachineOperand> &Cond,
3849 SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const {
3850
3851 // Start from the bottom of the block and work up, examining the
3852 // terminator instructions.
3853 MachineBasicBlock::iterator I = MBB.end();
3854 MachineBasicBlock::iterator UnCondBrIter = MBB.end();
3855 while (I != MBB.begin()) {
3856 --I;
3857 if (I->isDebugInstr())
3858 continue;
3859
3860 // Working from the bottom, when we see a non-terminator instruction, we're
3861 // done.
3862 if (!isUnpredicatedTerminator(MI: *I))
3863 break;
3864
3865 // A terminator that isn't a branch can't easily be handled by this
3866 // analysis.
3867 if (!I->isBranch())
3868 return true;
3869
3870 // Handle unconditional branches.
3871 if (I->getOpcode() == X86::JMP_1) {
3872 UnCondBrIter = I;
3873
3874 if (!AllowModify) {
3875 TBB = I->getOperand(i: 0).getMBB();
3876 continue;
3877 }
3878
3879 // If the block has any instructions after a JMP, delete them.
3880 MBB.erase(I: std::next(x: I), E: MBB.end());
3881
3882 Cond.clear();
3883 FBB = nullptr;
3884
3885 // Delete the JMP if it's equivalent to a fall-through.
3886 if (MBB.isLayoutSuccessor(MBB: I->getOperand(i: 0).getMBB())) {
3887 TBB = nullptr;
3888 I->eraseFromParent();
3889 I = MBB.end();
3890 UnCondBrIter = MBB.end();
3891 continue;
3892 }
3893
3894 // TBB is used to indicate the unconditional destination.
3895 TBB = I->getOperand(i: 0).getMBB();
3896 continue;
3897 }
3898
3899 // Handle conditional branches.
3900 X86::CondCode BranchCode = X86::getCondFromBranch(MI: *I);
3901 if (BranchCode == X86::COND_INVALID)
3902 return true; // Can't handle indirect branch.
3903
3904 // In practice we should never have an undef eflags operand, if we do
3905 // abort here as we are not prepared to preserve the flag.
3906 if (I->findRegisterUseOperand(Reg: X86::EFLAGS, /*TRI=*/nullptr)->isUndef())
3907 return true;
3908
3909 // Working from the bottom, handle the first conditional branch.
3910 if (Cond.empty()) {
3911 FBB = TBB;
3912 TBB = I->getOperand(i: 0).getMBB();
3913 Cond.push_back(Elt: MachineOperand::CreateImm(Val: BranchCode));
3914 CondBranches.push_back(Elt: &*I);
3915 continue;
3916 }
3917
3918 // Handle subsequent conditional branches. Only handle the case where all
3919 // conditional branches branch to the same destination and their condition
3920 // opcodes fit one of the special multi-branch idioms.
3921 assert(Cond.size() == 1);
3922 assert(TBB);
3923
3924 // If the conditions are the same, we can leave them alone.
3925 X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm();
3926 auto NewTBB = I->getOperand(i: 0).getMBB();
3927 if (OldBranchCode == BranchCode && TBB == NewTBB)
3928 continue;
3929
3930 // If they differ, see if they fit one of the known patterns. Theoretically,
3931 // we could handle more patterns here, but we shouldn't expect to see them
3932 // if instruction selection has done a reasonable job.
3933 if (TBB == NewTBB &&
3934 ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) ||
3935 (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) {
3936 BranchCode = X86::COND_NE_OR_P;
3937 } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) ||
3938 (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) {
3939 if (NewTBB != (FBB ? FBB : getFallThroughMBB(MBB: &MBB, TBB)))
3940 return true;
3941
3942 // X86::COND_E_AND_NP usually has two different branch destinations.
3943 //
3944 // JP B1
3945 // JE B2
3946 // JMP B1
3947 // B1:
3948 // B2:
3949 //
3950 // Here this condition branches to B2 only if NP && E. It has another
3951 // equivalent form:
3952 //
3953 // JNE B1
3954 // JNP B2
3955 // JMP B1
3956 // B1:
3957 // B2:
3958 //
3959 // Similarly it branches to B2 only if E && NP. That is why this condition
3960 // is named with COND_E_AND_NP.
3961 BranchCode = X86::COND_E_AND_NP;
3962 } else
3963 return true;
3964
3965 // Update the MachineOperand.
3966 Cond[0].setImm(BranchCode);
3967 CondBranches.push_back(Elt: &*I);
3968 }
3969
3970 return false;
3971}
3972
3973bool X86InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
3974 MachineBasicBlock *&TBB,
3975 MachineBasicBlock *&FBB,
3976 SmallVectorImpl<MachineOperand> &Cond,
3977 bool AllowModify) const {
3978 SmallVector<MachineInstr *, 4> CondBranches;
3979 return analyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify);
3980}
3981
3982static int getJumpTableIndexFromAddr(const MachineInstr &MI) {
3983 const MCInstrDesc &Desc = MI.getDesc();
3984 int MemRefBegin = X86II::getMemoryOperandNo(TSFlags: Desc.TSFlags);
3985 assert(MemRefBegin >= 0 && "instr should have memory operand");
3986 MemRefBegin += X86II::getOperandBias(Desc);
3987
3988 const MachineOperand &MO = MI.getOperand(i: MemRefBegin + X86::AddrDisp);
3989 if (!MO.isJTI())
3990 return -1;
3991
3992 return MO.getIndex();
3993}
3994
3995static int getJumpTableIndexFromReg(const MachineRegisterInfo &MRI,
3996 Register Reg) {
3997 if (!Reg.isVirtual())
3998 return -1;
3999 MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
4000 if (MI == nullptr)
4001 return -1;
4002 unsigned Opcode = MI->getOpcode();
4003 if (Opcode != X86::LEA64r && Opcode != X86::LEA32r)
4004 return -1;
4005 return getJumpTableIndexFromAddr(MI: *MI);
4006}
4007
4008int X86InstrInfo::getJumpTableIndex(const MachineInstr &MI) const {
4009 unsigned Opcode = MI.getOpcode();
4010 // Switch-jump pattern for non-PIC code looks like:
4011 // JMP64m $noreg, 8, %X, %jump-table.X, $noreg
4012 if (Opcode == X86::JMP64m || Opcode == X86::JMP32m) {
4013 return getJumpTableIndexFromAddr(MI);
4014 }
4015 // The pattern for PIC code looks like:
4016 // %0 = LEA64r $rip, 1, $noreg, %jump-table.X
4017 // %1 = MOVSX64rm32 %0, 4, XX, 0, $noreg
4018 // %2 = ADD64rr %1, %0
4019 // JMP64r %2
4020 if (Opcode == X86::JMP64r || Opcode == X86::JMP32r) {
4021 Register Reg = MI.getOperand(i: 0).getReg();
4022 if (!Reg.isVirtual())
4023 return -1;
4024 const MachineFunction &MF = *MI.getParent()->getParent();
4025 const MachineRegisterInfo &MRI = MF.getRegInfo();
4026 MachineInstr *Add = MRI.getUniqueVRegDef(Reg);
4027 if (Add == nullptr)
4028 return -1;
4029 if (Add->getOpcode() != X86::ADD64rr && Add->getOpcode() != X86::ADD32rr)
4030 return -1;
4031 int JTI1 = getJumpTableIndexFromReg(MRI, Reg: Add->getOperand(i: 1).getReg());
4032 if (JTI1 >= 0)
4033 return JTI1;
4034 int JTI2 = getJumpTableIndexFromReg(MRI, Reg: Add->getOperand(i: 2).getReg());
4035 if (JTI2 >= 0)
4036 return JTI2;
4037 }
4038 return -1;
4039}
4040
4041bool X86InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
4042 MachineBranchPredicate &MBP,
4043 bool AllowModify) const {
4044 using namespace std::placeholders;
4045
4046 SmallVector<MachineOperand, 4> Cond;
4047 SmallVector<MachineInstr *, 4> CondBranches;
4048 if (analyzeBranchImpl(MBB, TBB&: MBP.TrueDest, FBB&: MBP.FalseDest, Cond, CondBranches,
4049 AllowModify))
4050 return true;
4051
4052 if (Cond.size() != 1)
4053 return true;
4054
4055 assert(MBP.TrueDest && "expected!");
4056
4057 if (!MBP.FalseDest)
4058 MBP.FalseDest = MBB.getNextNode();
4059
4060 const TargetRegisterInfo *TRI = &getRegisterInfo();
4061
4062 MachineInstr *ConditionDef = nullptr;
4063 bool SingleUseCondition = true;
4064
4065 for (MachineInstr &MI : llvm::drop_begin(RangeOrContainer: llvm::reverse(C&: MBB))) {
4066 if (MI.modifiesRegister(Reg: X86::EFLAGS, TRI)) {
4067 ConditionDef = &MI;
4068 break;
4069 }
4070
4071 if (MI.readsRegister(Reg: X86::EFLAGS, TRI))
4072 SingleUseCondition = false;
4073 }
4074
4075 if (!ConditionDef)
4076 return true;
4077
4078 if (SingleUseCondition) {
4079 for (auto *Succ : MBB.successors())
4080 if (Succ->isLiveIn(Reg: X86::EFLAGS))
4081 SingleUseCondition = false;
4082 }
4083
4084 MBP.ConditionDef = ConditionDef;
4085 MBP.SingleUseCondition = SingleUseCondition;
4086
4087 // Currently we only recognize the simple pattern:
4088 //
4089 // test %reg, %reg
4090 // je %label
4091 //
4092 const unsigned TestOpcode =
4093 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr;
4094
4095 if (ConditionDef->getOpcode() == TestOpcode &&
4096 ConditionDef->getNumOperands() == 3 &&
4097 ConditionDef->getOperand(i: 0).isIdenticalTo(Other: ConditionDef->getOperand(i: 1)) &&
4098 (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) {
4099 MBP.LHS = ConditionDef->getOperand(i: 0);
4100 MBP.RHS = MachineOperand::CreateImm(Val: 0);
4101 MBP.Predicate = Cond[0].getImm() == X86::COND_NE
4102 ? MachineBranchPredicate::PRED_NE
4103 : MachineBranchPredicate::PRED_EQ;
4104 return false;
4105 }
4106
4107 return true;
4108}
4109
4110unsigned X86InstrInfo::removeBranch(MachineBasicBlock &MBB,
4111 int *BytesRemoved) const {
4112 assert(!BytesRemoved && "code size not handled");
4113
4114 MachineBasicBlock::iterator I = MBB.end();
4115 unsigned Count = 0;
4116
4117 while (I != MBB.begin()) {
4118 --I;
4119 if (I->isDebugInstr())
4120 continue;
4121 if (I->getOpcode() != X86::JMP_1 &&
4122 X86::getCondFromBranch(MI: *I) == X86::COND_INVALID)
4123 break;
4124 // Remove the branch.
4125 I->eraseFromParent();
4126 I = MBB.end();
4127 ++Count;
4128 }
4129
4130 return Count;
4131}
4132
4133unsigned X86InstrInfo::insertBranch(MachineBasicBlock &MBB,
4134 MachineBasicBlock *TBB,
4135 MachineBasicBlock *FBB,
4136 ArrayRef<MachineOperand> Cond,
4137 const DebugLoc &DL, int *BytesAdded) const {
4138 // Shouldn't be a fall through.
4139 assert(TBB && "insertBranch must not be told to insert a fallthrough");
4140 assert((Cond.size() == 1 || Cond.size() == 0) &&
4141 "X86 branch conditions have one component!");
4142 assert(!BytesAdded && "code size not handled");
4143
4144 if (Cond.empty()) {
4145 // Unconditional branch?
4146 assert(!FBB && "Unconditional branch with multiple successors!");
4147 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: X86::JMP_1)).addMBB(MBB: TBB);
4148 return 1;
4149 }
4150
4151 // If FBB is null, it is implied to be a fall-through block.
4152 bool FallThru = FBB == nullptr;
4153
4154 // Conditional branch.
4155 unsigned Count = 0;
4156 X86::CondCode CC = (X86::CondCode)Cond[0].getImm();
4157 switch (CC) {
4158 case X86::COND_NE_OR_P:
4159 // Synthesize NE_OR_P with two branches.
4160 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: X86::JCC_1)).addMBB(MBB: TBB).addImm(Val: X86::COND_NE);
4161 ++Count;
4162 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: X86::JCC_1)).addMBB(MBB: TBB).addImm(Val: X86::COND_P);
4163 ++Count;
4164 break;
4165 case X86::COND_E_AND_NP:
4166 // Use the next block of MBB as FBB if it is null.
4167 if (FBB == nullptr) {
4168 FBB = getFallThroughMBB(MBB: &MBB, TBB);
4169 assert(FBB && "MBB cannot be the last block in function when the false "
4170 "body is a fall-through.");
4171 }
4172 // Synthesize COND_E_AND_NP with two branches.
4173 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: X86::JCC_1)).addMBB(MBB: FBB).addImm(Val: X86::COND_NE);
4174 ++Count;
4175 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: X86::JCC_1)).addMBB(MBB: TBB).addImm(Val: X86::COND_NP);
4176 ++Count;
4177 break;
4178 default: {
4179 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: X86::JCC_1)).addMBB(MBB: TBB).addImm(Val: CC);
4180 ++Count;
4181 }
4182 }
4183 if (!FallThru) {
4184 // Two-way Conditional branch. Insert the second branch.
4185 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: X86::JMP_1)).addMBB(MBB: FBB);
4186 ++Count;
4187 }
4188 return Count;
4189}
4190
4191bool X86InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
4192 ArrayRef<MachineOperand> Cond,
4193 Register DstReg, Register TrueReg,
4194 Register FalseReg, int &CondCycles,
4195 int &TrueCycles, int &FalseCycles) const {
4196 // Not all subtargets have cmov instructions.
4197 if (!Subtarget.canUseCMOV())
4198 return false;
4199 if (Cond.size() != 1)
4200 return false;
4201 // We cannot do the composite conditions, at least not in SSA form.
4202 if ((X86::CondCode)Cond[0].getImm() > X86::LAST_VALID_COND)
4203 return false;
4204
4205 // Check register classes.
4206 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4207 const TargetRegisterClass *RC =
4208 RI.getCommonSubClass(A: MRI.getRegClass(Reg: TrueReg), B: MRI.getRegClass(Reg: FalseReg));
4209 if (!RC)
4210 return false;
4211
4212 // We have cmov instructions for 16, 32, and 64 bit general purpose registers.
4213 if (X86::GR16RegClass.hasSubClassEq(RC) ||
4214 X86::GR32RegClass.hasSubClassEq(RC) ||
4215 X86::GR64RegClass.hasSubClassEq(RC)) {
4216 // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy
4217 // Bridge. Probably Ivy Bridge as well.
4218 CondCycles = 2;
4219 TrueCycles = 2;
4220 FalseCycles = 2;
4221 return true;
4222 }
4223
4224 // Can't do vectors.
4225 return false;
4226}
4227
4228void X86InstrInfo::insertSelect(MachineBasicBlock &MBB,
4229 MachineBasicBlock::iterator I,
4230 const DebugLoc &DL, Register DstReg,
4231 ArrayRef<MachineOperand> Cond, Register TrueReg,
4232 Register FalseReg) const {
4233 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4234 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
4235 const TargetRegisterClass &RC = *MRI.getRegClass(Reg: DstReg);
4236 assert(Cond.size() == 1 && "Invalid Cond array");
4237 unsigned Opc =
4238 X86::getCMovOpcode(RegBytes: TRI.getRegSizeInBits(RC) / 8,
4239 HasMemoryOperand: false /*HasMemoryOperand*/, HasNDD: Subtarget.hasNDD());
4240 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
4241 .addReg(RegNo: FalseReg)
4242 .addReg(RegNo: TrueReg)
4243 .addImm(Val: Cond[0].getImm());
4244}
4245
4246/// Test if the given register is a physical h register.
4247static bool isHReg(Register Reg) {
4248 return X86::GR8_ABCD_HRegClass.contains(Reg);
4249}
4250
4251// Try and copy between VR128/VR64 and GR64 registers.
4252static unsigned CopyToFromAsymmetricReg(Register DestReg, Register SrcReg,
4253 const X86Subtarget &Subtarget) {
4254 bool HasAVX = Subtarget.hasAVX();
4255 bool HasAVX512 = Subtarget.hasAVX512();
4256 bool HasEGPR = Subtarget.hasEGPR();
4257
4258 // SrcReg(MaskReg) -> DestReg(GR64)
4259 // SrcReg(MaskReg) -> DestReg(GR32)
4260
4261 // All KMASK RegClasses hold the same k registers, can be tested against
4262 // anyone.
4263 if (X86::VK16RegClass.contains(Reg: SrcReg)) {
4264 if (X86::GR64RegClass.contains(Reg: DestReg)) {
4265 assert(Subtarget.hasBWI());
4266 return HasEGPR ? X86::KMOVQrk_EVEX : X86::KMOVQrk;
4267 }
4268 if (X86::GR32RegClass.contains(Reg: DestReg))
4269 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDrk_EVEX : X86::KMOVDrk)
4270 : (HasEGPR ? X86::KMOVWrk_EVEX : X86::KMOVWrk);
4271 }
4272
4273 // SrcReg(GR64) -> DestReg(MaskReg)
4274 // SrcReg(GR32) -> DestReg(MaskReg)
4275
4276 // All KMASK RegClasses hold the same k registers, can be tested against
4277 // anyone.
4278 if (X86::VK16RegClass.contains(Reg: DestReg)) {
4279 if (X86::GR64RegClass.contains(Reg: SrcReg)) {
4280 assert(Subtarget.hasBWI());
4281 return HasEGPR ? X86::KMOVQkr_EVEX : X86::KMOVQkr;
4282 }
4283 if (X86::GR32RegClass.contains(Reg: SrcReg))
4284 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDkr_EVEX : X86::KMOVDkr)
4285 : (HasEGPR ? X86::KMOVWkr_EVEX : X86::KMOVWkr);
4286 }
4287
4288 // SrcReg(VR128) -> DestReg(GR64)
4289 // SrcReg(VR64) -> DestReg(GR64)
4290 // SrcReg(GR64) -> DestReg(VR128)
4291 // SrcReg(GR64) -> DestReg(VR64)
4292
4293 if (X86::GR64RegClass.contains(Reg: DestReg)) {
4294 if (X86::VR128XRegClass.contains(Reg: SrcReg))
4295 // Copy from a VR128 register to a GR64 register.
4296 return HasAVX512 ? X86::VMOVPQIto64Zrr
4297 : HasAVX ? X86::VMOVPQIto64rr
4298 : X86::MOVPQIto64rr;
4299 if (X86::VR64RegClass.contains(Reg: SrcReg))
4300 // Copy from a VR64 register to a GR64 register.
4301 return X86::MMX_MOVD64from64rr;
4302 } else if (X86::GR64RegClass.contains(Reg: SrcReg)) {
4303 // Copy from a GR64 register to a VR128 register.
4304 if (X86::VR128XRegClass.contains(Reg: DestReg))
4305 return HasAVX512 ? X86::VMOV64toPQIZrr
4306 : HasAVX ? X86::VMOV64toPQIrr
4307 : X86::MOV64toPQIrr;
4308 // Copy from a GR64 register to a VR64 register.
4309 if (X86::VR64RegClass.contains(Reg: DestReg))
4310 return X86::MMX_MOVD64to64rr;
4311 }
4312
4313 // SrcReg(VR128) -> DestReg(GR32)
4314 // SrcReg(GR32) -> DestReg(VR128)
4315
4316 if (X86::GR32RegClass.contains(Reg: DestReg) &&
4317 X86::VR128XRegClass.contains(Reg: SrcReg))
4318 // Copy from a VR128 register to a GR32 register.
4319 return HasAVX512 ? X86::VMOVPDI2DIZrr
4320 : HasAVX ? X86::VMOVPDI2DIrr
4321 : X86::MOVPDI2DIrr;
4322
4323 if (X86::VR128XRegClass.contains(Reg: DestReg) &&
4324 X86::GR32RegClass.contains(Reg: SrcReg))
4325 // Copy from a GR32 register to a VR128 register.
4326 return HasAVX512 ? X86::VMOVDI2PDIZrr
4327 : HasAVX ? X86::VMOVDI2PDIrr
4328 : X86::MOVDI2PDIrr;
4329
4330 return 0;
4331}
4332
4333void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
4334 MachineBasicBlock::iterator MI,
4335 const DebugLoc &DL, Register DestReg,
4336 Register SrcReg, bool KillSrc,
4337 bool RenamableDest, bool RenamableSrc) const {
4338 // First deal with the normal symmetric copies.
4339 bool HasAVX = Subtarget.hasAVX();
4340 bool HasVLX = Subtarget.hasVLX();
4341 bool HasEGPR = Subtarget.hasEGPR();
4342 unsigned Opc = 0;
4343 if (X86::GR64RegClass.contains(Reg1: DestReg, Reg2: SrcReg))
4344 Opc = X86::MOV64rr;
4345 else if (X86::GR32RegClass.contains(Reg1: DestReg, Reg2: SrcReg))
4346 Opc = X86::MOV32rr;
4347 else if (X86::GR16RegClass.contains(Reg1: DestReg, Reg2: SrcReg))
4348 Opc = X86::MOV16rr;
4349 else if (X86::GR8RegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
4350 // Copying to or from a physical H register on x86-64 requires a NOREX
4351 // move. Otherwise use a normal move.
4352 if ((isHReg(Reg: DestReg) || isHReg(Reg: SrcReg)) && Subtarget.is64Bit()) {
4353 Opc = X86::MOV8rr_NOREX;
4354 // Both operands must be encodable without an REX prefix.
4355 assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) &&
4356 "8-bit H register can not be copied outside GR8_NOREX");
4357 } else
4358 Opc = X86::MOV8rr;
4359 } else if (X86::VR64RegClass.contains(Reg1: DestReg, Reg2: SrcReg))
4360 Opc = X86::MMX_MOVQ64rr;
4361 else if (X86::VR128XRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
4362 if (HasVLX)
4363 Opc = X86::VMOVAPSZ128rr;
4364 else if (X86::VR128RegClass.contains(Reg1: DestReg, Reg2: SrcReg))
4365 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr;
4366 else {
4367 // If this an extended register and we don't have VLX we need to use a
4368 // 512-bit move.
4369 Opc = X86::VMOVAPSZrr;
4370 const TargetRegisterInfo *TRI = &getRegisterInfo();
4371 DestReg =
4372 TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: X86::sub_xmm, RC: &X86::VR512RegClass);
4373 SrcReg =
4374 TRI->getMatchingSuperReg(Reg: SrcReg, SubIdx: X86::sub_xmm, RC: &X86::VR512RegClass);
4375 }
4376 } else if (X86::VR256XRegClass.contains(Reg1: DestReg, Reg2: SrcReg)) {
4377 if (HasVLX)
4378 Opc = X86::VMOVAPSZ256rr;
4379 else if (X86::VR256RegClass.contains(Reg1: DestReg, Reg2: SrcReg))
4380 Opc = X86::VMOVAPSYrr;
4381 else {
4382 // If this an extended register and we don't have VLX we need to use a
4383 // 512-bit move.
4384 Opc = X86::VMOVAPSZrr;
4385 const TargetRegisterInfo *TRI = &getRegisterInfo();
4386 DestReg =
4387 TRI->getMatchingSuperReg(Reg: DestReg, SubIdx: X86::sub_ymm, RC: &X86::VR512RegClass);
4388 SrcReg =
4389 TRI->getMatchingSuperReg(Reg: SrcReg, SubIdx: X86::sub_ymm, RC: &X86::VR512RegClass);
4390 }
4391 } else if (X86::VR512RegClass.contains(Reg1: DestReg, Reg2: SrcReg))
4392 Opc = X86::VMOVAPSZrr;
4393 // All KMASK RegClasses hold the same k registers, can be tested against
4394 // anyone.
4395 else if (X86::VK16RegClass.contains(Reg1: DestReg, Reg2: SrcReg))
4396 Opc = Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVQkk)
4397 : (HasEGPR ? X86::KMOVWkk_EVEX : X86::KMOVWkk);
4398
4399 if (!Opc)
4400 Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, Subtarget);
4401
4402 if (Opc) {
4403 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
4404 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
4405 return;
4406 }
4407
4408 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) {
4409 // FIXME: We use a fatal error here because historically LLVM has tried
4410 // lower some of these physreg copies and we want to ensure we get
4411 // reasonable bug reports if someone encounters a case no other testing
4412 // found. This path should be removed after the LLVM 7 release.
4413 report_fatal_error(reason: "Unable to copy EFLAGS physical register!");
4414 }
4415
4416 LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to "
4417 << RI.getName(DestReg) << '\n');
4418 report_fatal_error(reason: "Cannot emit physreg copy instruction");
4419}
4420
4421std::optional<DestSourcePair>
4422X86InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
4423 if (MI.isMoveReg()) {
4424 // FIXME: Dirty hack for apparent invariant that doesn't hold when
4425 // subreg_to_reg is coalesced with ordinary copies, such that the bits that
4426 // were asserted as 0 are now undef.
4427 if (MI.getOperand(i: 0).isUndef() && MI.getOperand(i: 0).getSubReg())
4428 return std::nullopt;
4429
4430 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
4431 }
4432 return std::nullopt;
4433}
4434
4435static unsigned getLoadStoreOpcodeForFP16(bool Load, const X86Subtarget &STI) {
4436 if (STI.hasFP16())
4437 return Load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr;
4438 if (Load)
4439 return X86::MOVSHPrm;
4440 return X86::MOVSHPmr;
4441}
4442
4443static unsigned getLoadStoreRegOpcode(Register Reg,
4444 const TargetRegisterClass *RC,
4445 bool IsStackAligned,
4446 const X86Subtarget &STI, bool Load) {
4447 bool HasAVX = STI.hasAVX();
4448 bool HasAVX512 = STI.hasAVX512();
4449 bool HasVLX = STI.hasVLX();
4450 bool HasEGPR = STI.hasEGPR();
4451
4452 assert(RC != nullptr && "Invalid target register class");
4453 switch (STI.getRegisterInfo()->getSpillSize(RC: *RC)) {
4454 default:
4455 llvm_unreachable("Unknown spill size");
4456 case 1:
4457 assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass");
4458 if (STI.is64Bit())
4459 // Copying to or from a physical H register on x86-64 requires a NOREX
4460 // move. Otherwise use a normal move.
4461 if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC))
4462 return Load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX;
4463 return Load ? X86::MOV8rm : X86::MOV8mr;
4464 case 2:
4465 if (X86::VK16RegClass.hasSubClassEq(RC))
4466 return Load ? (HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm)
4467 : (HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk);
4468 assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass");
4469 return Load ? X86::MOV16rm : X86::MOV16mr;
4470 case 4:
4471 if (X86::GR32RegClass.hasSubClassEq(RC))
4472 return Load ? X86::MOV32rm : X86::MOV32mr;
4473 if (X86::FR32XRegClass.hasSubClassEq(RC))
4474 return Load ? (HasAVX512 ? X86::VMOVSSZrm_alt
4475 : HasAVX ? X86::VMOVSSrm_alt
4476 : X86::MOVSSrm_alt)
4477 : (HasAVX512 ? X86::VMOVSSZmr
4478 : HasAVX ? X86::VMOVSSmr
4479 : X86::MOVSSmr);
4480 if (X86::RFP32RegClass.hasSubClassEq(RC))
4481 return Load ? X86::LD_Fp32m : X86::ST_Fp32m;
4482 if (X86::VK32RegClass.hasSubClassEq(RC)) {
4483 assert(STI.hasBWI() && "KMOVD requires BWI");
4484 return Load ? (HasEGPR ? X86::KMOVDkm_EVEX : X86::KMOVDkm)
4485 : (HasEGPR ? X86::KMOVDmk_EVEX : X86::KMOVDmk);
4486 }
4487 // All of these mask pair classes have the same spill size, the same kind
4488 // of kmov instructions can be used with all of them.
4489 if (X86::VK1PAIRRegClass.hasSubClassEq(RC) ||
4490 X86::VK2PAIRRegClass.hasSubClassEq(RC) ||
4491 X86::VK4PAIRRegClass.hasSubClassEq(RC) ||
4492 X86::VK8PAIRRegClass.hasSubClassEq(RC) ||
4493 X86::VK16PAIRRegClass.hasSubClassEq(RC))
4494 return Load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE;
4495 if (X86::FR16RegClass.hasSubClassEq(RC) ||
4496 X86::FR16XRegClass.hasSubClassEq(RC))
4497 return getLoadStoreOpcodeForFP16(Load, STI);
4498 llvm_unreachable("Unknown 4-byte regclass");
4499 case 8:
4500 if (X86::GR64RegClass.hasSubClassEq(RC))
4501 return Load ? X86::MOV64rm : X86::MOV64mr;
4502 if (X86::FR64XRegClass.hasSubClassEq(RC))
4503 return Load ? (HasAVX512 ? X86::VMOVSDZrm_alt
4504 : HasAVX ? X86::VMOVSDrm_alt
4505 : X86::MOVSDrm_alt)
4506 : (HasAVX512 ? X86::VMOVSDZmr
4507 : HasAVX ? X86::VMOVSDmr
4508 : X86::MOVSDmr);
4509 if (X86::VR64RegClass.hasSubClassEq(RC))
4510 return Load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr;
4511 if (X86::RFP64RegClass.hasSubClassEq(RC))
4512 return Load ? X86::LD_Fp64m : X86::ST_Fp64m;
4513 if (X86::VK64RegClass.hasSubClassEq(RC)) {
4514 assert(STI.hasBWI() && "KMOVQ requires BWI");
4515 return Load ? (HasEGPR ? X86::KMOVQkm_EVEX : X86::KMOVQkm)
4516 : (HasEGPR ? X86::KMOVQmk_EVEX : X86::KMOVQmk);
4517 }
4518 llvm_unreachable("Unknown 8-byte regclass");
4519 case 10:
4520 assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass");
4521 return Load ? X86::LD_Fp80m : X86::ST_FpP80m;
4522 case 16: {
4523 if (X86::VR128XRegClass.hasSubClassEq(RC)) {
4524 // If stack is realigned we can use aligned stores.
4525 if (IsStackAligned)
4526 return Load ? (HasVLX ? X86::VMOVAPSZ128rm
4527 : HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX
4528 : HasAVX ? X86::VMOVAPSrm
4529 : X86::MOVAPSrm)
4530 : (HasVLX ? X86::VMOVAPSZ128mr
4531 : HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX
4532 : HasAVX ? X86::VMOVAPSmr
4533 : X86::MOVAPSmr);
4534 else
4535 return Load ? (HasVLX ? X86::VMOVUPSZ128rm
4536 : HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX
4537 : HasAVX ? X86::VMOVUPSrm
4538 : X86::MOVUPSrm)
4539 : (HasVLX ? X86::VMOVUPSZ128mr
4540 : HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX
4541 : HasAVX ? X86::VMOVUPSmr
4542 : X86::MOVUPSmr);
4543 }
4544 llvm_unreachable("Unknown 16-byte regclass");
4545 }
4546 case 32:
4547 assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass");
4548 // If stack is realigned we can use aligned stores.
4549 if (IsStackAligned)
4550 return Load ? (HasVLX ? X86::VMOVAPSZ256rm
4551 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
4552 : X86::VMOVAPSYrm)
4553 : (HasVLX ? X86::VMOVAPSZ256mr
4554 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
4555 : X86::VMOVAPSYmr);
4556 else
4557 return Load ? (HasVLX ? X86::VMOVUPSZ256rm
4558 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
4559 : X86::VMOVUPSYrm)
4560 : (HasVLX ? X86::VMOVUPSZ256mr
4561 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
4562 : X86::VMOVUPSYmr);
4563 case 64:
4564 assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass");
4565 assert(STI.hasAVX512() && "Using 512-bit register requires AVX512");
4566 if (IsStackAligned)
4567 return Load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
4568 else
4569 return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
4570 case 1024:
4571 assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass");
4572 assert(STI.hasAMXTILE() && "Using 8*1024-bit register requires AMX-TILE");
4573#define GET_EGPR_IF_ENABLED(OPC) (STI.hasEGPR() ? OPC##_EVEX : OPC)
4574 return Load ? GET_EGPR_IF_ENABLED(X86::TILELOADD)
4575 : GET_EGPR_IF_ENABLED(X86::TILESTORED);
4576#undef GET_EGPR_IF_ENABLED
4577 }
4578}
4579
4580std::optional<ExtAddrMode>
4581X86InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI,
4582 const TargetRegisterInfo *TRI) const {
4583 const MCInstrDesc &Desc = MemI.getDesc();
4584 int MemRefBegin = X86II::getMemoryOperandNo(TSFlags: Desc.TSFlags);
4585 if (MemRefBegin < 0)
4586 return std::nullopt;
4587
4588 MemRefBegin += X86II::getOperandBias(Desc);
4589
4590 auto &BaseOp = MemI.getOperand(i: MemRefBegin + X86::AddrBaseReg);
4591 if (!BaseOp.isReg()) // Can be an MO_FrameIndex
4592 return std::nullopt;
4593
4594 const MachineOperand &DispMO = MemI.getOperand(i: MemRefBegin + X86::AddrDisp);
4595 // Displacement can be symbolic
4596 if (!DispMO.isImm())
4597 return std::nullopt;
4598
4599 ExtAddrMode AM;
4600 AM.BaseReg = BaseOp.getReg();
4601 AM.ScaledReg = MemI.getOperand(i: MemRefBegin + X86::AddrIndexReg).getReg();
4602 AM.Scale = MemI.getOperand(i: MemRefBegin + X86::AddrScaleAmt).getImm();
4603 AM.Displacement = DispMO.getImm();
4604 return AM;
4605}
4606
4607bool X86InstrInfo::verifyInstruction(const MachineInstr &MI,
4608 StringRef &ErrInfo) const {
4609 std::optional<ExtAddrMode> AMOrNone = getAddrModeFromMemoryOp(MemI: MI, TRI: nullptr);
4610 if (!AMOrNone)
4611 return true;
4612
4613 ExtAddrMode AM = *AMOrNone;
4614 assert(AM.Form == ExtAddrMode::Formula::Basic);
4615 if (AM.ScaledReg != X86::NoRegister) {
4616 switch (AM.Scale) {
4617 case 1:
4618 case 2:
4619 case 4:
4620 case 8:
4621 break;
4622 default:
4623 ErrInfo = "Scale factor in address must be 1, 2, 4 or 8";
4624 return false;
4625 }
4626 }
4627 if (!isInt<32>(x: AM.Displacement)) {
4628 ErrInfo = "Displacement in address must fit into 32-bit signed "
4629 "integer";
4630 return false;
4631 }
4632
4633 return true;
4634}
4635
4636bool X86InstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
4637 const Register Reg,
4638 int64_t &ImmVal) const {
4639 Register MovReg = Reg;
4640 const MachineInstr *MovMI = &MI;
4641
4642 // Follow use-def for SUBREG_TO_REG to find the real move immediate
4643 // instruction. It is quite common for x86-64.
4644 if (MI.isSubregToReg()) {
4645 // We use following pattern to setup 64b immediate.
4646 // %8:gr32 = MOV32r0 implicit-def dead $eflags
4647 // %6:gr64 = SUBREG_TO_REG killed %8:gr32, %subreg.sub_32bit
4648 unsigned SubIdx = MI.getOperand(i: 2).getImm();
4649 MovReg = MI.getOperand(i: 1).getReg();
4650 if (SubIdx != X86::sub_32bit)
4651 return false;
4652 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4653 MovMI = MRI.getUniqueVRegDef(Reg: MovReg);
4654 if (!MovMI)
4655 return false;
4656 }
4657
4658 if (MovMI->getOpcode() == X86::MOV32r0 &&
4659 MovMI->getOperand(i: 0).getReg() == MovReg) {
4660 ImmVal = 0;
4661 return true;
4662 }
4663
4664 if (MovMI->getOpcode() != X86::MOV32ri &&
4665 MovMI->getOpcode() != X86::MOV64ri &&
4666 MovMI->getOpcode() != X86::MOV32ri64 && MovMI->getOpcode() != X86::MOV8ri)
4667 return false;
4668 // Mov Src can be a global address.
4669 if (!MovMI->getOperand(i: 1).isImm() || MovMI->getOperand(i: 0).getReg() != MovReg)
4670 return false;
4671 ImmVal = MovMI->getOperand(i: 1).getImm();
4672 return true;
4673}
4674
4675bool X86InstrInfo::preservesZeroValueInReg(
4676 const MachineInstr *MI, const Register NullValueReg,
4677 const TargetRegisterInfo *TRI) const {
4678 if (!MI->modifiesRegister(Reg: NullValueReg, TRI))
4679 return true;
4680 switch (MI->getOpcode()) {
4681 // Shift right/left of a null unto itself is still a null, i.e. rax = shl rax
4682 // X.
4683 case X86::SHR64ri:
4684 case X86::SHR32ri:
4685 case X86::SHL64ri:
4686 case X86::SHL32ri:
4687 assert(MI->getOperand(0).isDef() && MI->getOperand(1).isUse() &&
4688 "expected for shift opcode!");
4689 return MI->getOperand(i: 0).getReg() == NullValueReg &&
4690 MI->getOperand(i: 1).getReg() == NullValueReg;
4691 // Zero extend of a sub-reg of NullValueReg into itself does not change the
4692 // null value.
4693 case X86::MOV32rr:
4694 return llvm::all_of(Range: MI->operands(), P: [&](const MachineOperand &MO) {
4695 return TRI->isSubRegisterEq(RegA: NullValueReg, RegB: MO.getReg());
4696 });
4697 default:
4698 return false;
4699 }
4700 llvm_unreachable("Should be handled above!");
4701}
4702
4703bool X86InstrInfo::getMemOperandsWithOffsetWidth(
4704 const MachineInstr &MemOp, SmallVectorImpl<const MachineOperand *> &BaseOps,
4705 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
4706 const TargetRegisterInfo *TRI) const {
4707 const MCInstrDesc &Desc = MemOp.getDesc();
4708 int MemRefBegin = X86II::getMemoryOperandNo(TSFlags: Desc.TSFlags);
4709 if (MemRefBegin < 0)
4710 return false;
4711
4712 MemRefBegin += X86II::getOperandBias(Desc);
4713
4714 const MachineOperand *BaseOp =
4715 &MemOp.getOperand(i: MemRefBegin + X86::AddrBaseReg);
4716 if (!BaseOp->isReg()) // Can be an MO_FrameIndex
4717 return false;
4718
4719 if (MemOp.getOperand(i: MemRefBegin + X86::AddrScaleAmt).getImm() != 1)
4720 return false;
4721
4722 if (MemOp.getOperand(i: MemRefBegin + X86::AddrIndexReg).getReg() !=
4723 X86::NoRegister)
4724 return false;
4725
4726 const MachineOperand &DispMO = MemOp.getOperand(i: MemRefBegin + X86::AddrDisp);
4727
4728 // Displacement can be symbolic
4729 if (!DispMO.isImm())
4730 return false;
4731
4732 Offset = DispMO.getImm();
4733
4734 if (!BaseOp->isReg())
4735 return false;
4736
4737 OffsetIsScalable = false;
4738 // FIXME: Relying on memoperands() may not be right thing to do here. Check
4739 // with X86 maintainers, and fix it accordingly. For now, it is ok, since
4740 // there is no use of `Width` for X86 back-end at the moment.
4741 Width = !MemOp.memoperands_empty() ? MemOp.memoperands().front()->getSize()
4742 : LocationSize::precise(Value: 0);
4743 BaseOps.push_back(Elt: BaseOp);
4744 return true;
4745}
4746
4747static unsigned getStoreRegOpcode(Register SrcReg,
4748 const TargetRegisterClass *RC,
4749 bool IsStackAligned,
4750 const X86Subtarget &STI) {
4751 return getLoadStoreRegOpcode(Reg: SrcReg, RC, IsStackAligned, STI, Load: false);
4752}
4753
4754static unsigned getLoadRegOpcode(Register DestReg,
4755 const TargetRegisterClass *RC,
4756 bool IsStackAligned, const X86Subtarget &STI) {
4757 return getLoadStoreRegOpcode(Reg: DestReg, RC, IsStackAligned, STI, Load: true);
4758}
4759
4760static bool isAMXOpcode(unsigned Opc) {
4761 switch (Opc) {
4762 default:
4763 return false;
4764 case X86::TILELOADD:
4765 case X86::TILESTORED:
4766 case X86::TILELOADD_EVEX:
4767 case X86::TILESTORED_EVEX:
4768 return true;
4769 }
4770}
4771
4772void X86InstrInfo::loadStoreTileReg(MachineBasicBlock &MBB,
4773 MachineBasicBlock::iterator MI,
4774 unsigned Opc, Register Reg, int FrameIdx,
4775 bool isKill) const {
4776 switch (Opc) {
4777 default:
4778 llvm_unreachable("Unexpected special opcode!");
4779 case X86::TILESTORED:
4780 case X86::TILESTORED_EVEX: {
4781 // tilestored %tmm, (%sp, %idx)
4782 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
4783 Register VirtReg = RegInfo.createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
4784 BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc(), MCID: get(Opcode: X86::MOV64ri), DestReg: VirtReg).addImm(Val: 64);
4785 MachineInstr *NewMI =
4786 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc(), MCID: get(Opcode: Opc)), FI: FrameIdx)
4787 .addReg(RegNo: Reg, Flags: getKillRegState(B: isKill));
4788 MachineOperand &MO = NewMI->getOperand(i: X86::AddrIndexReg);
4789 MO.setReg(VirtReg);
4790 MO.setIsKill(true);
4791 break;
4792 }
4793 case X86::TILELOADD:
4794 case X86::TILELOADD_EVEX: {
4795 // tileloadd (%sp, %idx), %tmm
4796 MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
4797 Register VirtReg = RegInfo.createVirtualRegister(RegClass: &X86::GR64_NOSPRegClass);
4798 BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc(), MCID: get(Opcode: X86::MOV64ri), DestReg: VirtReg).addImm(Val: 64);
4799 MachineInstr *NewMI = addFrameReference(
4800 MIB: BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc(), MCID: get(Opcode: Opc), DestReg: Reg), FI: FrameIdx);
4801 MachineOperand &MO = NewMI->getOperand(i: 1 + X86::AddrIndexReg);
4802 MO.setReg(VirtReg);
4803 MO.setIsKill(true);
4804 break;
4805 }
4806 }
4807}
4808
4809void X86InstrInfo::storeRegToStackSlot(
4810 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
4811 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
4812
4813 Register VReg, MachineInstr::MIFlag Flags) const {
4814 const MachineFunction &MF = *MBB.getParent();
4815 const MachineFrameInfo &MFI = MF.getFrameInfo();
4816 assert(MFI.getObjectSize(FrameIdx) >= RI.getSpillSize(*RC) &&
4817 "Stack slot too small for store");
4818
4819 unsigned Alignment = std::max<uint32_t>(a: RI.getSpillSize(RC: *RC), b: 16);
4820 bool isAligned =
4821 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
4822 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(ObjectIdx: FrameIdx));
4823
4824 unsigned Opc = getStoreRegOpcode(SrcReg, RC, IsStackAligned: isAligned, STI: Subtarget);
4825 if (isAMXOpcode(Opc))
4826 loadStoreTileReg(MBB, MI, Opc, Reg: SrcReg, FrameIdx, isKill);
4827 else
4828 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc(), MCID: get(Opcode: Opc)), FI: FrameIdx)
4829 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
4830 .setMIFlag(Flags);
4831}
4832
4833void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
4834 MachineBasicBlock::iterator MI,
4835 Register DestReg, int FrameIdx,
4836 const TargetRegisterClass *RC,
4837 Register VReg, unsigned SubReg,
4838 MachineInstr::MIFlag Flags) const {
4839 const MachineFunction &MF = *MBB.getParent();
4840 const MachineFrameInfo &MFI = MF.getFrameInfo();
4841 assert(MFI.getObjectSize(FrameIdx) >= RI.getSpillSize(*RC) &&
4842 "Load size exceeds stack slot");
4843 unsigned Alignment = std::max<uint32_t>(a: RI.getSpillSize(RC: *RC), b: 16);
4844 bool isAligned =
4845 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
4846 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(ObjectIdx: FrameIdx));
4847
4848 unsigned Opc = getLoadRegOpcode(DestReg, RC, IsStackAligned: isAligned, STI: Subtarget);
4849 if (isAMXOpcode(Opc))
4850 loadStoreTileReg(MBB, MI, Opc, Reg: DestReg, FrameIdx);
4851 else
4852 addFrameReference(MIB: BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc(), MCID: get(Opcode: Opc), DestReg), FI: FrameIdx)
4853 .setMIFlag(Flags);
4854}
4855
4856bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
4857 Register &SrcReg2, int64_t &CmpMask,
4858 int64_t &CmpValue) const {
4859 switch (MI.getOpcode()) {
4860 default:
4861 break;
4862 case X86::CMP64ri32:
4863 case X86::CMP32ri:
4864 case X86::CMP16ri:
4865 case X86::CMP8ri:
4866 SrcReg = MI.getOperand(i: 0).getReg();
4867 SrcReg2 = 0;
4868 if (MI.getOperand(i: 1).isImm()) {
4869 CmpMask = ~0;
4870 CmpValue = MI.getOperand(i: 1).getImm();
4871 } else {
4872 CmpMask = CmpValue = 0;
4873 }
4874 return true;
4875 // A SUB can be used to perform comparison.
4876 CASE_ND(SUB64rm)
4877 CASE_ND(SUB32rm)
4878 CASE_ND(SUB16rm)
4879 CASE_ND(SUB8rm)
4880 SrcReg = MI.getOperand(i: 1).getReg();
4881 SrcReg2 = 0;
4882 CmpMask = 0;
4883 CmpValue = 0;
4884 return true;
4885 CASE_ND(SUB64rr)
4886 CASE_ND(SUB32rr)
4887 CASE_ND(SUB16rr)
4888 CASE_ND(SUB8rr)
4889 SrcReg = MI.getOperand(i: 1).getReg();
4890 SrcReg2 = MI.getOperand(i: 2).getReg();
4891 CmpMask = 0;
4892 CmpValue = 0;
4893 return true;
4894 CASE_ND(SUB64ri32)
4895 CASE_ND(SUB32ri)
4896 CASE_ND(SUB16ri)
4897 CASE_ND(SUB8ri)
4898 SrcReg = MI.getOperand(i: 1).getReg();
4899 SrcReg2 = 0;
4900 if (MI.getOperand(i: 2).isImm()) {
4901 CmpMask = ~0;
4902 CmpValue = MI.getOperand(i: 2).getImm();
4903 } else {
4904 CmpMask = CmpValue = 0;
4905 }
4906 return true;
4907 case X86::CMP64rr:
4908 case X86::CMP32rr:
4909 case X86::CMP16rr:
4910 case X86::CMP8rr:
4911 SrcReg = MI.getOperand(i: 0).getReg();
4912 SrcReg2 = MI.getOperand(i: 1).getReg();
4913 CmpMask = 0;
4914 CmpValue = 0;
4915 return true;
4916 case X86::TEST8rr:
4917 case X86::TEST16rr:
4918 case X86::TEST32rr:
4919 case X86::TEST64rr:
4920 SrcReg = MI.getOperand(i: 0).getReg();
4921 if (MI.getOperand(i: 1).getReg() != SrcReg)
4922 return false;
4923 // Compare against zero.
4924 SrcReg2 = 0;
4925 CmpMask = ~0;
4926 CmpValue = 0;
4927 return true;
4928 case X86::TEST64ri32:
4929 case X86::TEST32ri:
4930 case X86::TEST16ri:
4931 case X86::TEST8ri:
4932 SrcReg = MI.getOperand(i: 0).getReg();
4933 SrcReg2 = 0;
4934 // Force identical compare.
4935 CmpMask = 0;
4936 CmpValue = 0;
4937 return true;
4938 }
4939 return false;
4940}
4941
4942bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
4943 Register SrcReg, Register SrcReg2,
4944 int64_t ImmMask, int64_t ImmValue,
4945 const MachineInstr &OI, bool *IsSwapped,
4946 int64_t *ImmDelta) const {
4947 switch (OI.getOpcode()) {
4948 case X86::CMP64rr:
4949 case X86::CMP32rr:
4950 case X86::CMP16rr:
4951 case X86::CMP8rr:
4952 CASE_ND(SUB64rr)
4953 CASE_ND(SUB32rr)
4954 CASE_ND(SUB16rr)
4955 CASE_ND(SUB8rr) {
4956 Register OISrcReg;
4957 Register OISrcReg2;
4958 int64_t OIMask;
4959 int64_t OIValue;
4960 if (!analyzeCompare(MI: OI, SrcReg&: OISrcReg, SrcReg2&: OISrcReg2, CmpMask&: OIMask, CmpValue&: OIValue) ||
4961 OIMask != ImmMask || OIValue != ImmValue)
4962 return false;
4963 if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
4964 *IsSwapped = false;
4965 return true;
4966 }
4967 if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
4968 *IsSwapped = true;
4969 return true;
4970 }
4971 return false;
4972 }
4973 case X86::CMP64ri32:
4974 case X86::CMP32ri:
4975 case X86::CMP16ri:
4976 case X86::CMP8ri:
4977 case X86::TEST64ri32:
4978 case X86::TEST32ri:
4979 case X86::TEST16ri:
4980 case X86::TEST8ri:
4981 CASE_ND(SUB64ri32)
4982 CASE_ND(SUB32ri)
4983 CASE_ND(SUB16ri)
4984 CASE_ND(SUB8ri)
4985 case X86::TEST64rr:
4986 case X86::TEST32rr:
4987 case X86::TEST16rr:
4988 case X86::TEST8rr: {
4989 if (ImmMask != 0) {
4990 Register OISrcReg;
4991 Register OISrcReg2;
4992 int64_t OIMask;
4993 int64_t OIValue;
4994 if (analyzeCompare(MI: OI, SrcReg&: OISrcReg, SrcReg2&: OISrcReg2, CmpMask&: OIMask, CmpValue&: OIValue) &&
4995 SrcReg == OISrcReg && ImmMask == OIMask) {
4996 if (OIValue == ImmValue) {
4997 *ImmDelta = 0;
4998 return true;
4999 } else if (static_cast<uint64_t>(ImmValue) ==
5000 static_cast<uint64_t>(OIValue) - 1) {
5001 *ImmDelta = -1;
5002 return true;
5003 } else if (static_cast<uint64_t>(ImmValue) ==
5004 static_cast<uint64_t>(OIValue) + 1) {
5005 *ImmDelta = 1;
5006 return true;
5007 } else {
5008 return false;
5009 }
5010 }
5011 }
5012 return FlagI.isIdenticalTo(Other: OI);
5013 }
5014 default:
5015 return false;
5016 }
5017}
5018
5019#define CASE_EVEX(OP) \
5020 case X86::OP: \
5021 case X86::OP##_EVEX:
5022
5023/// Check whether the definition can be converted
5024/// to remove a comparison against zero.
5025inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag,
5026 bool &ClearsOverflowFlag) {
5027 NoSignFlag = false;
5028 ClearsOverflowFlag = false;
5029
5030 // "ELF Handling for Thread-Local Storage" specifies that x86-64 GOTTPOFF, and
5031 // i386 GOTNTPOFF/INDNTPOFF relocations can convert an ADD to a LEA during
5032 // Initial Exec to Local Exec relaxation. In these cases, we must not depend
5033 // on the EFLAGS modification of ADD actually happening in the final binary.
5034 if (MI.getOpcode() == X86::ADD64rm || MI.getOpcode() == X86::ADD32rm) {
5035 unsigned Flags = MI.getOperand(i: 5).getTargetFlags();
5036 if (Flags == X86II::MO_GOTTPOFF || Flags == X86II::MO_INDNTPOFF ||
5037 Flags == X86II::MO_GOTNTPOFF)
5038 return false;
5039 }
5040
5041 switch (MI.getOpcode()) {
5042 default:
5043 return false;
5044
5045 // The shift instructions only modify ZF if their shift count is non-zero.
5046 // N.B.: The processor truncates the shift count depending on the encoding.
5047 CASE_ND(SAR8ri)
5048 CASE_ND(SAR16ri)
5049 CASE_ND(SAR32ri)
5050 CASE_ND(SAR64ri)
5051 CASE_ND(SHR8ri)
5052 CASE_ND(SHR16ri)
5053 CASE_ND(SHR32ri)
5054 CASE_ND(SHR64ri)
5055 return getTruncatedShiftCount(MI, ShiftAmtOperandIdx: 2) != 0;
5056
5057 // Some left shift instructions can be turned into LEA instructions but only
5058 // if their flags aren't used. Avoid transforming such instructions.
5059 CASE_ND(SHL8ri)
5060 CASE_ND(SHL16ri)
5061 CASE_ND(SHL32ri)
5062 CASE_ND(SHL64ri) {
5063 unsigned ShAmt = getTruncatedShiftCount(MI, ShiftAmtOperandIdx: 2);
5064 if (isTruncatedShiftCountForLEA(ShAmt))
5065 return false;
5066 return ShAmt != 0;
5067 }
5068
5069 CASE_ND(SHRD16rri8)
5070 CASE_ND(SHRD32rri8)
5071 CASE_ND(SHRD64rri8)
5072 CASE_ND(SHLD16rri8)
5073 CASE_ND(SHLD32rri8)
5074 CASE_ND(SHLD64rri8)
5075 return getTruncatedShiftCount(MI, ShiftAmtOperandIdx: 3) != 0;
5076
5077 CASE_ND(SUB64ri32)
5078 CASE_ND(SUB32ri)
5079 CASE_ND(SUB16ri)
5080 CASE_ND(SUB8ri)
5081 CASE_ND(SUB64rr)
5082 CASE_ND(SUB32rr)
5083 CASE_ND(SUB16rr)
5084 CASE_ND(SUB8rr)
5085 CASE_ND(SUB64rm)
5086 CASE_ND(SUB32rm)
5087 CASE_ND(SUB16rm)
5088 CASE_ND(SUB8rm)
5089 CASE_ND(DEC64r)
5090 CASE_ND(DEC32r)
5091 CASE_ND(DEC16r)
5092 CASE_ND(DEC8r)
5093 CASE_ND(ADD64ri32)
5094 CASE_ND(ADD32ri)
5095 CASE_ND(ADD16ri)
5096 CASE_ND(ADD8ri)
5097 CASE_ND(ADD64rr)
5098 CASE_ND(ADD32rr)
5099 CASE_ND(ADD16rr)
5100 CASE_ND(ADD8rr)
5101 CASE_ND(ADD64rm)
5102 CASE_ND(ADD32rm)
5103 CASE_ND(ADD16rm)
5104 CASE_ND(ADD8rm)
5105 CASE_ND(INC64r)
5106 CASE_ND(INC32r)
5107 CASE_ND(INC16r)
5108 CASE_ND(INC8r)
5109 CASE_ND(ADC64ri32)
5110 CASE_ND(ADC32ri)
5111 CASE_ND(ADC16ri)
5112 CASE_ND(ADC8ri)
5113 CASE_ND(ADC64rr)
5114 CASE_ND(ADC32rr)
5115 CASE_ND(ADC16rr)
5116 CASE_ND(ADC8rr)
5117 CASE_ND(ADC64rm)
5118 CASE_ND(ADC32rm)
5119 CASE_ND(ADC16rm)
5120 CASE_ND(ADC8rm)
5121 CASE_ND(SBB64ri32)
5122 CASE_ND(SBB32ri)
5123 CASE_ND(SBB16ri)
5124 CASE_ND(SBB8ri)
5125 CASE_ND(SBB64rr)
5126 CASE_ND(SBB32rr)
5127 CASE_ND(SBB16rr)
5128 CASE_ND(SBB8rr)
5129 CASE_ND(SBB64rm)
5130 CASE_ND(SBB32rm)
5131 CASE_ND(SBB16rm)
5132 CASE_ND(SBB8rm)
5133 CASE_ND(NEG8r)
5134 CASE_ND(NEG16r)
5135 CASE_ND(NEG32r)
5136 CASE_ND(NEG64r)
5137 case X86::LZCNT16rr:
5138 case X86::LZCNT16rm:
5139 case X86::LZCNT32rr:
5140 case X86::LZCNT32rm:
5141 case X86::LZCNT64rr:
5142 case X86::LZCNT64rm:
5143 case X86::POPCNT16rr:
5144 case X86::POPCNT16rm:
5145 case X86::POPCNT32rr:
5146 case X86::POPCNT32rm:
5147 case X86::POPCNT64rr:
5148 case X86::POPCNT64rm:
5149 case X86::TZCNT16rr:
5150 case X86::TZCNT16rm:
5151 case X86::TZCNT32rr:
5152 case X86::TZCNT32rm:
5153 case X86::TZCNT64rr:
5154 case X86::TZCNT64rm:
5155 return true;
5156 CASE_ND(AND64ri32)
5157 CASE_ND(AND32ri)
5158 CASE_ND(AND16ri)
5159 CASE_ND(AND8ri)
5160 CASE_ND(AND64rr)
5161 CASE_ND(AND32rr)
5162 CASE_ND(AND16rr)
5163 CASE_ND(AND8rr)
5164 CASE_ND(AND64rm)
5165 CASE_ND(AND32rm)
5166 CASE_ND(AND16rm)
5167 CASE_ND(AND8rm)
5168 CASE_ND(XOR64ri32)
5169 CASE_ND(XOR32ri)
5170 CASE_ND(XOR16ri)
5171 CASE_ND(XOR8ri)
5172 CASE_ND(XOR64rr)
5173 CASE_ND(XOR32rr)
5174 CASE_ND(XOR16rr)
5175 CASE_ND(XOR8rr)
5176 CASE_ND(XOR64rm)
5177 CASE_ND(XOR32rm)
5178 CASE_ND(XOR16rm)
5179 CASE_ND(XOR8rm)
5180 CASE_ND(OR64ri32)
5181 CASE_ND(OR32ri)
5182 CASE_ND(OR16ri)
5183 CASE_ND(OR8ri)
5184 CASE_ND(OR64rr)
5185 CASE_ND(OR32rr)
5186 CASE_ND(OR16rr)
5187 CASE_ND(OR8rr)
5188 CASE_ND(OR64rm)
5189 CASE_ND(OR32rm)
5190 CASE_ND(OR16rm)
5191 CASE_ND(OR8rm)
5192 CASE_EVEX(ANDN32rr)
5193 CASE_EVEX(ANDN32rm)
5194 CASE_EVEX(ANDN64rr)
5195 CASE_EVEX(ANDN64rm)
5196 CASE_EVEX(BLSI32rr)
5197 CASE_EVEX(BLSI32rm)
5198 CASE_EVEX(BLSI64rr)
5199 CASE_EVEX(BLSI64rm)
5200 CASE_EVEX(BLSMSK32rr)
5201 CASE_EVEX(BLSMSK32rm)
5202 CASE_EVEX(BLSMSK64rr)
5203 CASE_EVEX(BLSMSK64rm)
5204 CASE_EVEX(BLSR32rr)
5205 CASE_EVEX(BLSR32rm)
5206 CASE_EVEX(BLSR64rr)
5207 CASE_EVEX(BLSR64rm)
5208 case X86::BLCFILL32rr:
5209 case X86::BLCFILL32rm:
5210 case X86::BLCFILL64rr:
5211 case X86::BLCFILL64rm:
5212 case X86::BLCI32rr:
5213 case X86::BLCI32rm:
5214 case X86::BLCI64rr:
5215 case X86::BLCI64rm:
5216 case X86::BLCIC32rr:
5217 case X86::BLCIC32rm:
5218 case X86::BLCIC64rr:
5219 case X86::BLCIC64rm:
5220 case X86::BLCMSK32rr:
5221 case X86::BLCMSK32rm:
5222 case X86::BLCMSK64rr:
5223 case X86::BLCMSK64rm:
5224 case X86::BLCS32rr:
5225 case X86::BLCS32rm:
5226 case X86::BLCS64rr:
5227 case X86::BLCS64rm:
5228 case X86::BLSFILL32rr:
5229 case X86::BLSFILL32rm:
5230 case X86::BLSFILL64rr:
5231 case X86::BLSFILL64rm:
5232 case X86::BLSIC32rr:
5233 case X86::BLSIC32rm:
5234 case X86::BLSIC64rr:
5235 case X86::BLSIC64rm:
5236 CASE_EVEX(BZHI32rr)
5237 CASE_EVEX(BZHI32rm)
5238 CASE_EVEX(BZHI64rr)
5239 CASE_EVEX(BZHI64rm)
5240 case X86::T1MSKC32rr:
5241 case X86::T1MSKC32rm:
5242 case X86::T1MSKC64rr:
5243 case X86::T1MSKC64rm:
5244 case X86::TZMSK32rr:
5245 case X86::TZMSK32rm:
5246 case X86::TZMSK64rr:
5247 case X86::TZMSK64rm:
5248 // These instructions clear the overflow flag just like TEST.
5249 // FIXME: These are not the only instructions in this switch that clear the
5250 // overflow flag.
5251 ClearsOverflowFlag = true;
5252 return true;
5253 CASE_EVEX(BEXTR32rr)
5254 CASE_EVEX(BEXTR64rr)
5255 CASE_EVEX(BEXTR32rm)
5256 CASE_EVEX(BEXTR64rm)
5257 case X86::BEXTRI32ri:
5258 case X86::BEXTRI32mi:
5259 case X86::BEXTRI64ri:
5260 case X86::BEXTRI64mi:
5261 // BEXTR doesn't update the sign flag so we can't use it. It does clear
5262 // the overflow flag, but that's not useful without the sign flag.
5263 NoSignFlag = true;
5264 return true;
5265 }
5266}
5267
5268/// Check whether the use can be converted to remove a comparison against zero.
5269/// Returns the EFLAGS condition and the operand that we are comparing against zero.
5270static std::pair<X86::CondCode, unsigned> isUseDefConvertible(const MachineInstr &MI) {
5271 switch (MI.getOpcode()) {
5272 default:
5273 return std::make_pair(x: X86::COND_INVALID, y: ~0U);
5274 CASE_ND(NEG8r)
5275 CASE_ND(NEG16r)
5276 CASE_ND(NEG32r)
5277 CASE_ND(NEG64r)
5278 return std::make_pair(x: X86::COND_AE, y: 1U);
5279 case X86::LZCNT16rr:
5280 case X86::LZCNT32rr:
5281 case X86::LZCNT64rr:
5282 return std::make_pair(x: X86::COND_B, y: 1U);
5283 case X86::POPCNT16rr:
5284 case X86::POPCNT32rr:
5285 case X86::POPCNT64rr:
5286 return std::make_pair(x: X86::COND_E, y: 1U);
5287 case X86::TZCNT16rr:
5288 case X86::TZCNT32rr:
5289 case X86::TZCNT64rr:
5290 return std::make_pair(x: X86::COND_B, y: 1U);
5291 case X86::BSF16rr:
5292 case X86::BSF32rr:
5293 case X86::BSF64rr:
5294 case X86::BSR16rr:
5295 case X86::BSR32rr:
5296 case X86::BSR64rr:
5297 return std::make_pair(x: X86::COND_E, y: 2U);
5298 CASE_EVEX(BLSI32rr)
5299 CASE_EVEX(BLSI64rr)
5300 return std::make_pair(x: X86::COND_AE, y: 1U);
5301 CASE_EVEX(BLSR32rr)
5302 CASE_EVEX(BLSR64rr)
5303 CASE_EVEX(BLSMSK32rr)
5304 CASE_EVEX(BLSMSK64rr)
5305 return std::make_pair(x: X86::COND_B, y: 1U);
5306 // TODO: TBM instructions.
5307 }
5308}
5309#undef CASE_EVEX
5310
5311MachineInstr *X86InstrInfo::findDominatingRedundantFlagInstr(
5312 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
5313 int64_t CmpValue, MachineBasicBlock *MultiPredMBB, bool &IsSwapped,
5314 int64_t &ImmDelta,
5315 SmallVectorImpl<std::pair<MachineInstr *, unsigned>> &InstsToUpdate) const {
5316 assert(Subtarget.hasNF() && "NF feature required");
5317 const TargetRegisterInfo *TRI = &getRegisterInfo();
5318
5319 // The caller already scanned MultiPredMBB without finding the producer, so it
5320 // must live in a block that strictly dominates MultiPredMBB. Walk
5321 // predecessors backward to find it and prove dominance, avoiding a
5322 // whole-function MachineDominatorTree that would be rebuilt in O(function
5323 // size) per compare.
5324 //
5325 // The producer's block dominates MultiPredMBB iff every backward path funnels
5326 // through it before a function-entry block, so expand predecessors but stop
5327 // at a block holding the producer. Bail if a predecessor-less block is
5328 // reached without the producer (a path bypasses it) or the producer is found
5329 // in two blocks (neither dominates alone). Within a block, scan backward,
5330 // collecting the NF-convertible EFLAGS clobbers above the producer and
5331 // bailing on any other clobber (it would shadow the producer's flags from
5332 // CmpInstr).
5333 //
5334 // Clobbers are staged in Pending and committed only on success. Visited
5335 // (seeded with MultiPredMBB) stops the walk from revisiting a block or
5336 // re-entering the single-predecessor chain, so none is collected twice.
5337 //
5338 // Each NF conversion trades a compact legacy/EVEX-compressed encoding for a
5339 // wider EVEX (often NDD three-operand) one, growing code size, while the
5340 // reuse only removes a single compare. Cap the total number of conversions
5341 // (those the caller already collected on the single-predecessor chain plus
5342 // those the walk stages) so the reuse cannot bloat code just to delete one
5343 // compare.
5344 MachineInstr *Sub = nullptr;
5345 MachineBasicBlock *SubMBB = nullptr;
5346 SmallVector<std::pair<MachineInstr *, unsigned>, 4> Pending;
5347 SmallPtrSet<MachineBasicBlock *, 8> Visited;
5348 SmallVector<MachineBasicBlock *, 8> Worklist;
5349 Visited.insert(Ptr: MultiPredMBB);
5350 for (MachineBasicBlock *Pred : MultiPredMBB->predecessors())
5351 if (Visited.insert(Ptr: Pred).second)
5352 Worklist.push_back(Elt: Pred);
5353 while (!Worklist.empty()) {
5354 MachineBasicBlock *MBB = Worklist.pop_back_val();
5355 MachineInstr *Producer = nullptr;
5356 for (MachineInstr &Inst : reverse(C&: *MBB)) {
5357 if (!Inst.modifiesRegister(Reg: X86::EFLAGS, TRI))
5358 continue;
5359 if (isRedundantFlagInstr(FlagI: CmpInstr, SrcReg, SrcReg2, ImmMask: CmpMask, ImmValue: CmpValue,
5360 OI: Inst, IsSwapped: &IsSwapped, ImmDelta: &ImmDelta)) {
5361 Producer = &Inst;
5362 break;
5363 }
5364 unsigned NewOpc = X86::getNFVariantIfClobberRemovable(MI: Inst, TRI);
5365 if (!NewOpc)
5366 return nullptr;
5367 if (InstsToUpdate.size() + Pending.size() >= MaxNFConversions)
5368 return nullptr;
5369 Pending.push_back(Elt: std::make_pair(x: &Inst, y&: NewOpc));
5370 }
5371 if (Producer) {
5372 // A producer in a second block means neither dominates alone.
5373 if (Sub && SubMBB != MBB)
5374 return nullptr;
5375 Sub = Producer;
5376 SubMBB = MBB;
5377 continue;
5378 }
5379 // Entry reached without the producer: some path bypasses it.
5380 if (MBB->pred_empty())
5381 return nullptr;
5382 for (MachineBasicBlock *Pred : MBB->predecessors())
5383 if (Visited.insert(Ptr: Pred).second)
5384 Worklist.push_back(Elt: Pred);
5385 }
5386 if (!Sub)
5387 return nullptr;
5388
5389 // The forward condition-code fixup in the caller (OpsToUpdate) only rewrites
5390 // EFLAGS users within CmpMBB. When the producer's flags require a condition
5391 // swap or an immediate adjustment, EFLAGS users elsewhere in the dominated
5392 // region or in CmpMBB's successors (when EFLAGS is live-out) would also need
5393 // rewriting, which is not handled here. Restrict the multi-predecessor case
5394 // to producers that yield identical flags.
5395 if (IsSwapped || ImmDelta != 0)
5396 return nullptr;
5397
5398 InstsToUpdate.append(in_start: Pending.begin(), in_end: Pending.end());
5399 return Sub;
5400}
5401
5402/// Check if there exists an earlier instruction that
5403/// operates on the same source operands and sets flags in the same way as
5404/// Compare; remove Compare if possible.
5405bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
5406 Register SrcReg2, int64_t CmpMask,
5407 int64_t CmpValue,
5408 const MachineRegisterInfo *MRI) const {
5409 // Check whether we can replace SUB with CMP.
5410 switch (CmpInstr.getOpcode()) {
5411 default:
5412 break;
5413 CASE_ND(SUB64ri32)
5414 CASE_ND(SUB32ri)
5415 CASE_ND(SUB16ri)
5416 CASE_ND(SUB8ri)
5417 CASE_ND(SUB64rm)
5418 CASE_ND(SUB32rm)
5419 CASE_ND(SUB16rm)
5420 CASE_ND(SUB8rm)
5421 CASE_ND(SUB64rr)
5422 CASE_ND(SUB32rr)
5423 CASE_ND(SUB16rr)
5424 CASE_ND(SUB8rr) {
5425 if (!MRI->use_nodbg_empty(RegNo: CmpInstr.getOperand(i: 0).getReg()))
5426 return false;
5427 // There is no use of the destination register, we can replace SUB with CMP.
5428 unsigned NewOpcode = 0;
5429#define FROM_TO(A, B) \
5430 CASE_ND(A) NewOpcode = X86::B; \
5431 break;
5432 switch (CmpInstr.getOpcode()) {
5433 default:
5434 llvm_unreachable("Unreachable!");
5435 FROM_TO(SUB64rm, CMP64rm)
5436 FROM_TO(SUB32rm, CMP32rm)
5437 FROM_TO(SUB16rm, CMP16rm)
5438 FROM_TO(SUB8rm, CMP8rm)
5439 FROM_TO(SUB64rr, CMP64rr)
5440 FROM_TO(SUB32rr, CMP32rr)
5441 FROM_TO(SUB16rr, CMP16rr)
5442 FROM_TO(SUB8rr, CMP8rr)
5443 FROM_TO(SUB64ri32, CMP64ri32)
5444 FROM_TO(SUB32ri, CMP32ri)
5445 FROM_TO(SUB16ri, CMP16ri)
5446 FROM_TO(SUB8ri, CMP8ri)
5447 }
5448#undef FROM_TO
5449 CmpInstr.setDesc(get(Opcode: NewOpcode));
5450 CmpInstr.removeOperand(OpNo: 0);
5451 // Mutating this instruction invalidates any debug data associated with it.
5452 CmpInstr.dropDebugNumber();
5453 // Fall through to optimize Cmp if Cmp is CMPrr or CMPri.
5454 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
5455 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
5456 return false;
5457 }
5458 }
5459
5460 // The following code tries to remove the comparison by re-using EFLAGS
5461 // from earlier instructions.
5462
5463 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0);
5464
5465 // Transformation currently requires SSA values.
5466 if (SrcReg2.isPhysical())
5467 return false;
5468 MachineInstr *SrcRegDef = MRI->getVRegDef(Reg: SrcReg);
5469 assert(SrcRegDef && "Must have a definition (SSA)");
5470
5471 MachineInstr *MI = nullptr;
5472 MachineInstr *Sub = nullptr;
5473 MachineInstr *Movr0Inst = nullptr;
5474 SmallVector<std::pair<MachineInstr *, unsigned>, 4> InstsToUpdate;
5475 bool NoSignFlag = false;
5476 bool ClearsOverflowFlag = false;
5477 bool ShouldUpdateCC = false;
5478 bool IsSwapped = false;
5479 bool HasNF = Subtarget.hasNF();
5480 unsigned OpNo = 0;
5481 X86::CondCode NewCC = X86::COND_INVALID;
5482 int64_t ImmDelta = 0;
5483
5484 // Search backward from CmpInstr for the next instruction defining EFLAGS.
5485 const TargetRegisterInfo *TRI = &getRegisterInfo();
5486 MachineBasicBlock &CmpMBB = *CmpInstr.getParent();
5487 MachineBasicBlock::reverse_iterator From =
5488 std::next(x: MachineBasicBlock::reverse_iterator(CmpInstr));
5489 for (MachineBasicBlock *MBB = &CmpMBB;;) {
5490 for (MachineInstr &Inst : make_range(x: From, y: MBB->rend())) {
5491 // Try to use EFLAGS from the instruction defining %SrcReg. Example:
5492 // %eax = addl ...
5493 // ... // EFLAGS not changed
5494 // testl %eax, %eax // <-- can be removed
5495 if (&Inst == SrcRegDef) {
5496 if (IsCmpZero &&
5497 isDefConvertible(MI: Inst, NoSignFlag, ClearsOverflowFlag)) {
5498 MI = &Inst;
5499 break;
5500 }
5501
5502 // Look back for the following pattern, in which case the
5503 // test16rr/test64rr instruction could be erased.
5504 //
5505 // Example for test16rr:
5506 // %reg = and32ri %in_reg, 5
5507 // ... // EFLAGS not changed.
5508 // %src_reg = copy %reg.sub_16bit:gr32
5509 // test16rr %src_reg, %src_reg, implicit-def $eflags
5510 // Example for test64rr:
5511 // %reg = and32ri %in_reg, 5
5512 // ... // EFLAGS not changed.
5513 // %src_reg = subreg_to_reg %reg, %subreg.sub_index
5514 // test64rr %src_reg, %src_reg, implicit-def $eflags
5515 MachineInstr *AndInstr = nullptr;
5516 if (IsCmpZero &&
5517 findRedundantFlagInstr(CmpInstr, CmpValDefInstr&: Inst, MRI, AndInstr: &AndInstr, TRI,
5518 ST: Subtarget, NoSignFlag, ClearsOverflowFlag)) {
5519 assert(AndInstr != nullptr && X86::isAND(AndInstr->getOpcode()));
5520 MI = AndInstr;
5521 break;
5522 }
5523 // Cannot find other candidates before definition of SrcReg.
5524 return false;
5525 }
5526
5527 if (Inst.modifiesRegister(Reg: X86::EFLAGS, TRI)) {
5528 // Try to use EFLAGS produced by an instruction reading %SrcReg.
5529 // Example:
5530 // %eax = ...
5531 // ...
5532 // popcntl %eax
5533 // ... // EFLAGS not changed
5534 // testl %eax, %eax // <-- can be removed
5535 if (IsCmpZero) {
5536 std::tie(args&: NewCC, args&: OpNo) = isUseDefConvertible(MI: Inst);
5537 if (NewCC != X86::COND_INVALID && Inst.getOperand(i: OpNo).isReg() &&
5538 Inst.getOperand(i: OpNo).getReg() == SrcReg) {
5539 ShouldUpdateCC = true;
5540 MI = &Inst;
5541 break;
5542 }
5543 }
5544
5545 // Try to use EFLAGS from an instruction with similar flag results.
5546 // Example:
5547 // sub x, y or cmp x, y
5548 // ... // EFLAGS not changed
5549 // cmp x, y // <-- can be removed
5550 if (isRedundantFlagInstr(FlagI: CmpInstr, SrcReg, SrcReg2, ImmMask: CmpMask, ImmValue: CmpValue,
5551 OI: Inst, IsSwapped: &IsSwapped, ImmDelta: &ImmDelta)) {
5552 Sub = &Inst;
5553 break;
5554 }
5555
5556 // MOV32r0 is implemented with xor which clobbers condition code. It is
5557 // safe to move up, if the definition to EFLAGS is dead and earlier
5558 // instructions do not read or write EFLAGS.
5559 if (!Movr0Inst && Inst.getOpcode() == X86::MOV32r0 &&
5560 Inst.registerDefIsDead(Reg: X86::EFLAGS, TRI)) {
5561 Movr0Inst = &Inst;
5562 continue;
5563 }
5564
5565 // Try to replace non-NF with NF instructions.
5566 if (HasNF) {
5567 unsigned NewOp = X86::getNFVariantIfClobberRemovable(MI: Inst, TRI);
5568 if (!NewOp)
5569 return false;
5570
5571 InstsToUpdate.push_back(Elt: std::make_pair(x: &Inst, y&: NewOp));
5572 continue;
5573 }
5574
5575 // Cannot do anything for any other EFLAG changes.
5576 return false;
5577 }
5578 }
5579
5580 if (MI || Sub)
5581 break;
5582
5583 // Reached the begin of the basic block. If it has exactly one predecessor,
5584 // continue the backward scan there. Otherwise (multiple predecessors), try
5585 // to reuse EFLAGS from a dominating producer (handled below).
5586 if (MBB->pred_size() != 1) {
5587 // The block has multiple predecessors. We can still reuse EFLAGS from an
5588 // equivalent flag producer that dominates CmpInstr, provided every path
5589 // from that producer to CmpInstr only clobbers EFLAGS via instructions
5590 // that have an NF (no-flags) variant (which requires APX). This handles
5591 // patterns like (CMP duplicated by CodeGenPrepare across a diamond):
5592 // entry: cmp %x, C ; br
5593 // bb1: imul ... ; clobbers EFLAGS -> {nf} imul
5594 // bb2: ...
5595 // bb3: cmp %x, C ; <-- redundant, reuse EFLAGS from entry
5596 // cmovcc ...
5597 // The helper caps the total number of NF conversions so this cannot grow
5598 // code size without bound just to delete one compare.
5599 if (HasNF)
5600 Sub = findDominatingRedundantFlagInstr(
5601 CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue, MultiPredMBB: MBB, IsSwapped,
5602 ImmDelta, InstsToUpdate);
5603 if (!Sub)
5604 return false;
5605 break;
5606 }
5607 MBB = *MBB->pred_begin();
5608 From = MBB->rbegin();
5609 }
5610
5611 // Scan forward from the instruction after CmpInstr for uses of EFLAGS.
5612 // It is safe to remove CmpInstr if EFLAGS is redefined or killed.
5613 // If we are done with the basic block, we need to check whether EFLAGS is
5614 // live-out.
5615 bool FlagsMayLiveOut = true;
5616 SmallVector<std::pair<MachineInstr *, X86::CondCode>, 4> OpsToUpdate;
5617 MachineBasicBlock::iterator AfterCmpInstr =
5618 std::next(x: MachineBasicBlock::iterator(CmpInstr));
5619 for (MachineInstr &Instr : make_range(x: AfterCmpInstr, y: CmpMBB.end())) {
5620 bool ModifyEFLAGS = Instr.modifiesRegister(Reg: X86::EFLAGS, TRI);
5621 bool UseEFLAGS = Instr.readsRegister(Reg: X86::EFLAGS, TRI);
5622 // We should check the usage if this instruction uses and updates EFLAGS.
5623 if (!UseEFLAGS && ModifyEFLAGS) {
5624 // It is safe to remove CmpInstr if EFLAGS is updated again.
5625 FlagsMayLiveOut = false;
5626 break;
5627 }
5628 if (!UseEFLAGS && !ModifyEFLAGS)
5629 continue;
5630
5631 // EFLAGS is used by this instruction.
5632 X86::CondCode OldCC = X86::getCondFromMI(MI: Instr);
5633 if ((MI || IsSwapped || ImmDelta != 0) && OldCC == X86::COND_INVALID)
5634 return false;
5635
5636 X86::CondCode ReplacementCC = X86::COND_INVALID;
5637 if (MI) {
5638 switch (OldCC) {
5639 default:
5640 break;
5641 case X86::COND_A:
5642 case X86::COND_AE:
5643 case X86::COND_B:
5644 case X86::COND_BE:
5645 // CF is used, we can't perform this optimization.
5646 return false;
5647 case X86::COND_G:
5648 case X86::COND_GE:
5649 case X86::COND_L:
5650 case X86::COND_LE:
5651 // If SF is used, but the instruction doesn't update the SF, then we
5652 // can't do the optimization.
5653 if (NoSignFlag)
5654 return false;
5655 [[fallthrough]];
5656 case X86::COND_O:
5657 case X86::COND_NO:
5658 // If OF is used, the instruction needs to clear it like CmpZero does.
5659 if (!ClearsOverflowFlag)
5660 return false;
5661 break;
5662 case X86::COND_S:
5663 case X86::COND_NS:
5664 // If SF is used, but the instruction doesn't update the SF, then we
5665 // can't do the optimization.
5666 if (NoSignFlag)
5667 return false;
5668 break;
5669 }
5670
5671 // If we're updating the condition code check if we have to reverse the
5672 // condition.
5673 if (ShouldUpdateCC)
5674 switch (OldCC) {
5675 default:
5676 return false;
5677 case X86::COND_E:
5678 ReplacementCC = NewCC;
5679 break;
5680 case X86::COND_NE:
5681 ReplacementCC = GetOppositeBranchCondition(CC: NewCC);
5682 break;
5683 }
5684 } else if (IsSwapped) {
5685 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code needs
5686 // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
5687 // We swap the condition code and synthesize the new opcode.
5688 ReplacementCC = getSwappedCondition(CC: OldCC);
5689 if (ReplacementCC == X86::COND_INVALID)
5690 return false;
5691 ShouldUpdateCC = true;
5692 } else if (ImmDelta != 0) {
5693 unsigned BitWidth = RI.getRegSizeInBits(RC: *MRI->getRegClass(Reg: SrcReg));
5694 // Shift amount for min/max constants to adjust for 8/16/32 instruction
5695 // sizes.
5696 switch (OldCC) {
5697 case X86::COND_L: // x <s (C + 1) --> x <=s C
5698 if (ImmDelta != 1 || APInt::getSignedMinValue(numBits: BitWidth) == CmpValue)
5699 return false;
5700 ReplacementCC = X86::COND_LE;
5701 break;
5702 case X86::COND_B: // x <u (C + 1) --> x <=u C
5703 if (ImmDelta != 1 || CmpValue == 0)
5704 return false;
5705 ReplacementCC = X86::COND_BE;
5706 break;
5707 case X86::COND_GE: // x >=s (C + 1) --> x >s C
5708 if (ImmDelta != 1 || APInt::getSignedMinValue(numBits: BitWidth) == CmpValue)
5709 return false;
5710 ReplacementCC = X86::COND_G;
5711 break;
5712 case X86::COND_AE: // x >=u (C + 1) --> x >u C
5713 if (ImmDelta != 1 || CmpValue == 0)
5714 return false;
5715 ReplacementCC = X86::COND_A;
5716 break;
5717 case X86::COND_G: // x >s (C - 1) --> x >=s C
5718 if (ImmDelta != -1 || APInt::getSignedMaxValue(numBits: BitWidth) == CmpValue)
5719 return false;
5720 ReplacementCC = X86::COND_GE;
5721 break;
5722 case X86::COND_A: // x >u (C - 1) --> x >=u C
5723 if (ImmDelta != -1 || APInt::getMaxValue(numBits: BitWidth) == CmpValue)
5724 return false;
5725 ReplacementCC = X86::COND_AE;
5726 break;
5727 case X86::COND_LE: // x <=s (C - 1) --> x <s C
5728 if (ImmDelta != -1 || APInt::getSignedMaxValue(numBits: BitWidth) == CmpValue)
5729 return false;
5730 ReplacementCC = X86::COND_L;
5731 break;
5732 case X86::COND_BE: // x <=u (C - 1) --> x <u C
5733 if (ImmDelta != -1 || APInt::getMaxValue(numBits: BitWidth) == CmpValue)
5734 return false;
5735 ReplacementCC = X86::COND_B;
5736 break;
5737 default:
5738 return false;
5739 }
5740 ShouldUpdateCC = true;
5741 }
5742
5743 if (ShouldUpdateCC && ReplacementCC != OldCC) {
5744 // Push the MachineInstr to OpsToUpdate.
5745 // If it is safe to remove CmpInstr, the condition code of these
5746 // instructions will be modified.
5747 OpsToUpdate.push_back(Elt: std::make_pair(x: &Instr, y&: ReplacementCC));
5748 }
5749 if (ModifyEFLAGS || Instr.killsRegister(Reg: X86::EFLAGS, TRI)) {
5750 // It is safe to remove CmpInstr if EFLAGS is updated again or killed.
5751 FlagsMayLiveOut = false;
5752 break;
5753 }
5754 }
5755
5756 // If we have to update users but EFLAGS is live-out abort, since we cannot
5757 // easily find all of the users.
5758 if ((MI != nullptr || ShouldUpdateCC) && FlagsMayLiveOut) {
5759 for (MachineBasicBlock *Successor : CmpMBB.successors())
5760 if (Successor->isLiveIn(Reg: X86::EFLAGS))
5761 return false;
5762 }
5763
5764 // The instruction to be updated is either Sub or MI.
5765 assert((MI == nullptr || Sub == nullptr) && "Should not have Sub and MI set");
5766 Sub = MI != nullptr ? MI : Sub;
5767 MachineBasicBlock *SubBB = Sub->getParent();
5768 // Move Movr0Inst to the appropriate place before Sub.
5769 if (Movr0Inst) {
5770 // Only move within the same block so we don't accidentally move to a
5771 // block with higher execution frequency.
5772 if (&CmpMBB != SubBB)
5773 return false;
5774 // Look backwards until we find a def that doesn't use the current EFLAGS.
5775 MachineBasicBlock::reverse_iterator InsertI = Sub,
5776 InsertE = Sub->getParent()->rend();
5777 for (; InsertI != InsertE; ++InsertI) {
5778 MachineInstr *Instr = &*InsertI;
5779 if (!Instr->readsRegister(Reg: X86::EFLAGS, TRI) &&
5780 Instr->modifiesRegister(Reg: X86::EFLAGS, TRI)) {
5781 Movr0Inst->getParent()->remove(I: Movr0Inst);
5782 Instr->getParent()->insert(I: MachineBasicBlock::iterator(Instr),
5783 MI: Movr0Inst);
5784 break;
5785 }
5786 }
5787 if (InsertI == InsertE)
5788 return false;
5789 }
5790
5791 // Replace non-NF with NF instructions.
5792 for (auto &Inst : InstsToUpdate) {
5793 Inst.first->setDesc(get(Opcode: Inst.second));
5794 Inst.first->removeOperand(
5795 OpNo: Inst.first->findRegisterDefOperandIdx(Reg: X86::EFLAGS, /*TRI=*/nullptr));
5796 }
5797
5798 // Make sure Sub instruction defines EFLAGS and mark the def live.
5799 MachineOperand *FlagDef =
5800 Sub->findRegisterDefOperand(Reg: X86::EFLAGS, /*TRI=*/nullptr);
5801 assert(FlagDef && "Unable to locate a def EFLAGS operand");
5802 FlagDef->setIsDead(false);
5803
5804 CmpInstr.eraseFromParent();
5805
5806 // Modify the condition code of instructions in OpsToUpdate.
5807 for (auto &Op : OpsToUpdate) {
5808 Op.first->getOperand(i: Op.first->getDesc().getNumOperands() - 1)
5809 .setImm(Op.second);
5810 }
5811 // Add EFLAGS to block live-ins between CmpBB and block of flags producer.
5812 // Walk the CFG backward from CmpMBB up to (but excluding) SubBB, marking
5813 // EFLAGS live-in on every block in between. SubBB dominates CmpMBB (whether
5814 // the producer was found by the single-predecessor backward walk or the
5815 // multi-predecessor dominator search), so the walk reaches SubBB on every
5816 // path and never escapes above it. A single-predecessor chain is just the
5817 // degenerate case where every block has exactly one predecessor.
5818 SmallPtrSet<MachineBasicBlock *, 8> Visited;
5819 SmallVector<MachineBasicBlock *, 8> Worklist(1, &CmpMBB);
5820 Visited.insert(Ptr: &CmpMBB);
5821 while (!Worklist.empty()) {
5822 MachineBasicBlock *MBB = Worklist.pop_back_val();
5823 // EFLAGS is produced inside SubBB, so it is not live-in there.
5824 if (MBB == SubBB)
5825 continue;
5826 if (!MBB->isLiveIn(Reg: X86::EFLAGS))
5827 MBB->addLiveIn(PhysReg: X86::EFLAGS);
5828 for (MachineBasicBlock *Pred : MBB->predecessors())
5829 if (Visited.insert(Ptr: Pred).second)
5830 Worklist.push_back(Elt: Pred);
5831 }
5832 return true;
5833}
5834
5835/// \returns true if the instruction can be changed to COPY when imm is 0.
5836static bool canConvert2Copy(unsigned Opc) {
5837 switch (Opc) {
5838 default:
5839 return false;
5840 CASE_ND(ADD64ri32)
5841 CASE_ND(SUB64ri32)
5842 CASE_ND(OR64ri32)
5843 CASE_ND(XOR64ri32)
5844 CASE_ND(ADD32ri)
5845 CASE_ND(SUB32ri)
5846 CASE_ND(OR32ri)
5847 CASE_ND(XOR32ri)
5848 return true;
5849 }
5850}
5851
5852/// Convert an ALUrr opcode to corresponding ALUri opcode. Such as
5853/// ADD32rr ==> ADD32ri
5854static unsigned convertALUrr2ALUri(unsigned Opc) {
5855 switch (Opc) {
5856 default:
5857 return 0;
5858#define FROM_TO(FROM, TO) \
5859 case X86::FROM: \
5860 return X86::TO; \
5861 case X86::FROM##_ND: \
5862 return X86::TO##_ND;
5863 FROM_TO(ADC64rr, ADC64ri32)
5864 FROM_TO(SBB64rr, SBB64ri32)
5865 FROM_TO(AND64rr, AND64ri32)
5866 FROM_TO(OR64rr, OR64ri32)
5867 FROM_TO(XOR64rr, XOR64ri32)
5868 FROM_TO(SHR64rCL, SHR64ri)
5869 FROM_TO(SHL64rCL, SHL64ri)
5870 FROM_TO(SAR64rCL, SAR64ri)
5871 FROM_TO(ROL64rCL, ROL64ri)
5872 FROM_TO(ROR64rCL, ROR64ri)
5873 FROM_TO(RCL64rCL, RCL64ri)
5874 FROM_TO(RCR64rCL, RCR64ri)
5875 FROM_TO(ADD32rr, ADD32ri)
5876 FROM_TO(ADC32rr, ADC32ri)
5877 FROM_TO(SUB32rr, SUB32ri)
5878 FROM_TO(SBB32rr, SBB32ri)
5879 FROM_TO(AND32rr, AND32ri)
5880 FROM_TO(OR32rr, OR32ri)
5881 FROM_TO(XOR32rr, XOR32ri)
5882 FROM_TO(SHR32rCL, SHR32ri)
5883 FROM_TO(SHL32rCL, SHL32ri)
5884 FROM_TO(SAR32rCL, SAR32ri)
5885 FROM_TO(ROL32rCL, ROL32ri)
5886 FROM_TO(ROR32rCL, ROR32ri)
5887 FROM_TO(RCL32rCL, RCL32ri)
5888 FROM_TO(RCR32rCL, RCR32ri)
5889#undef FROM_TO
5890#define FROM_TO(FROM, TO) \
5891 case X86::FROM: \
5892 return X86::TO;
5893 FROM_TO(ADD64rr, ADD64ri32)
5894 FROM_TO(SUB64rr, SUB64ri32)
5895 FROM_TO(TEST64rr, TEST64ri32)
5896 FROM_TO(CTEST64rr, CTEST64ri32)
5897 FROM_TO(CMP64rr, CMP64ri32)
5898 FROM_TO(CCMP64rr, CCMP64ri32)
5899 FROM_TO(TEST32rr, TEST32ri)
5900 FROM_TO(CTEST32rr, CTEST32ri)
5901 FROM_TO(CMP32rr, CMP32ri)
5902 FROM_TO(CCMP32rr, CCMP32ri)
5903#undef FROM_TO
5904 case X86::ADD64rr_ND:
5905 return X86::ADD64ri32_ND;
5906 case X86::SUB64rr_ND:
5907 return X86::SUB64ri32_ND;
5908 }
5909}
5910
5911/// Reg is assigned ImmVal in DefMI, and is used in UseMI.
5912/// If MakeChange is true, this function tries to replace Reg by ImmVal in
5913/// UseMI. If MakeChange is false, just check if folding is possible.
5914//
5915/// \returns true if folding is successful or possible.
5916bool X86InstrInfo::foldImmediateImpl(MachineInstr &UseMI, MachineInstr *DefMI,
5917 Register Reg, int64_t ImmVal,
5918 MachineRegisterInfo *MRI,
5919 bool MakeChange) const {
5920 bool Modified = false;
5921
5922 // 64 bit operations accept sign extended 32 bit immediates.
5923 // 32 bit operations accept all 32 bit immediates, so we don't need to check
5924 // them.
5925 const TargetRegisterClass *RC = nullptr;
5926 if (Reg.isVirtual())
5927 RC = MRI->getRegClass(Reg);
5928 if ((Reg.isPhysical() && X86::GR64RegClass.contains(Reg)) ||
5929 (Reg.isVirtual() && X86::GR64RegClass.hasSubClassEq(RC))) {
5930 if (!isInt<32>(x: ImmVal))
5931 return false;
5932 }
5933
5934 if (UseMI.findRegisterUseOperand(Reg, /*TRI=*/nullptr)->getSubReg())
5935 return false;
5936 // Immediate has larger code size than register. So avoid folding the
5937 // immediate if it has more than 1 use and we are optimizing for size.
5938 if (UseMI.getMF()->getFunction().hasOptSize() && Reg.isVirtual() &&
5939 !MRI->hasOneNonDBGUse(RegNo: Reg))
5940 return false;
5941
5942 unsigned Opc = UseMI.getOpcode();
5943 unsigned NewOpc;
5944 if (Opc == TargetOpcode::COPY) {
5945 Register ToReg = UseMI.getOperand(i: 0).getReg();
5946 const TargetRegisterClass *RC = nullptr;
5947 if (ToReg.isVirtual())
5948 RC = MRI->getRegClass(Reg: ToReg);
5949 bool GR32Reg = (ToReg.isVirtual() && X86::GR32RegClass.hasSubClassEq(RC)) ||
5950 (ToReg.isPhysical() && X86::GR32RegClass.contains(Reg: ToReg));
5951 bool GR64Reg = (ToReg.isVirtual() && X86::GR64RegClass.hasSubClassEq(RC)) ||
5952 (ToReg.isPhysical() && X86::GR64RegClass.contains(Reg: ToReg));
5953 bool GR8Reg = (ToReg.isVirtual() && X86::GR8RegClass.hasSubClassEq(RC)) ||
5954 (ToReg.isPhysical() && X86::GR8RegClass.contains(Reg: ToReg));
5955
5956 if (ImmVal == 0) {
5957 // We have MOV32r0 only.
5958 if (!GR32Reg)
5959 return false;
5960 }
5961
5962 if (GR64Reg) {
5963 if (isUInt<32>(x: ImmVal))
5964 NewOpc = X86::MOV32ri64;
5965 else
5966 NewOpc = X86::MOV64ri;
5967 } else if (GR32Reg) {
5968 NewOpc = X86::MOV32ri;
5969 if (ImmVal == 0) {
5970 // MOV32r0 clobbers EFLAGS.
5971 const TargetRegisterInfo *TRI = &getRegisterInfo();
5972 if (UseMI.getParent()->computeRegisterLiveness(
5973 TRI, Reg: X86::EFLAGS, Before: UseMI) != MachineBasicBlock::LQR_Dead)
5974 return false;
5975
5976 // MOV32r0 is different than other cases because it doesn't encode the
5977 // immediate in the instruction. So we directly modify it here.
5978 if (!MakeChange)
5979 return true;
5980 UseMI.setDesc(get(Opcode: X86::MOV32r0));
5981 UseMI.removeOperand(
5982 OpNo: UseMI.findRegisterUseOperandIdx(Reg, /*TRI=*/nullptr));
5983 UseMI.addOperand(Op: MachineOperand::CreateReg(Reg: X86::EFLAGS, /*isDef=*/true,
5984 /*isImp=*/true,
5985 /*isKill=*/false,
5986 /*isDead=*/true));
5987 Modified = true;
5988 }
5989 } else if (GR8Reg)
5990 NewOpc = X86::MOV8ri;
5991 else
5992 return false;
5993 } else
5994 NewOpc = convertALUrr2ALUri(Opc);
5995
5996 if (!NewOpc)
5997 return false;
5998
5999 // For SUB instructions the immediate can only be the second source operand.
6000 if ((NewOpc == X86::SUB64ri32 || NewOpc == X86::SUB32ri ||
6001 NewOpc == X86::SBB64ri32 || NewOpc == X86::SBB32ri ||
6002 NewOpc == X86::SUB64ri32_ND || NewOpc == X86::SUB32ri_ND ||
6003 NewOpc == X86::SBB64ri32_ND || NewOpc == X86::SBB32ri_ND) &&
6004 UseMI.findRegisterUseOperandIdx(Reg, /*TRI=*/nullptr) != 2)
6005 return false;
6006 // For CMP instructions the immediate can only be at index 1.
6007 if (((NewOpc == X86::CMP64ri32 || NewOpc == X86::CMP32ri) ||
6008 (NewOpc == X86::CCMP64ri32 || NewOpc == X86::CCMP32ri)) &&
6009 UseMI.findRegisterUseOperandIdx(Reg, /*TRI=*/nullptr) != 1)
6010 return false;
6011
6012 using namespace X86;
6013 if (isSHL(Opcode: Opc) || isSHR(Opcode: Opc) || isSAR(Opcode: Opc) || isROL(Opcode: Opc) || isROR(Opcode: Opc) ||
6014 isRCL(Opcode: Opc) || isRCR(Opcode: Opc)) {
6015 unsigned RegIdx = UseMI.findRegisterUseOperandIdx(Reg, /*TRI=*/nullptr);
6016 if (RegIdx < 2)
6017 return false;
6018 if (!isInt<8>(x: ImmVal))
6019 return false;
6020 assert(Reg == X86::CL);
6021
6022 if (!MakeChange)
6023 return true;
6024 UseMI.setDesc(get(Opcode: NewOpc));
6025 UseMI.removeOperand(OpNo: RegIdx);
6026 UseMI.addOperand(Op: MachineOperand::CreateImm(Val: ImmVal));
6027 // Reg is physical register $cl, so we don't know if DefMI is dead through
6028 // MRI. Let the caller handle it, or pass dead-mi-elimination can delete
6029 // the dead physical register define instruction.
6030 return true;
6031 }
6032
6033 if (!MakeChange)
6034 return true;
6035
6036 if (!Modified) {
6037 // Modify the instruction.
6038 if (ImmVal == 0 && canConvert2Copy(Opc: NewOpc) &&
6039 UseMI.registerDefIsDead(Reg: X86::EFLAGS, /*TRI=*/nullptr)) {
6040 // %100 = add %101, 0
6041 // ==>
6042 // %100 = COPY %101
6043 UseMI.setDesc(get(Opcode: TargetOpcode::COPY));
6044 UseMI.removeOperand(
6045 OpNo: UseMI.findRegisterUseOperandIdx(Reg, /*TRI=*/nullptr));
6046 UseMI.removeOperand(
6047 OpNo: UseMI.findRegisterDefOperandIdx(Reg: X86::EFLAGS, /*TRI=*/nullptr));
6048 UseMI.untieRegOperand(OpIdx: 0);
6049 UseMI.clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
6050 UseMI.clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
6051 } else {
6052 unsigned Op1 = 1, Op2 = CommuteAnyOperandIndex;
6053 unsigned ImmOpNum = 2;
6054 if (!UseMI.getOperand(i: 0).isDef()) {
6055 Op1 = 0; // TEST, CMP, CTEST, CCMP
6056 ImmOpNum = 1;
6057 }
6058 if (Opc == TargetOpcode::COPY)
6059 ImmOpNum = 1;
6060 if (findCommutedOpIndices(MI: UseMI, SrcOpIdx1&: Op1, SrcOpIdx2&: Op2) &&
6061 UseMI.getOperand(i: Op1).getReg() == Reg)
6062 commuteInstruction(MI&: UseMI);
6063
6064 assert(UseMI.getOperand(ImmOpNum).getReg() == Reg);
6065 UseMI.setDesc(get(Opcode: NewOpc));
6066 UseMI.getOperand(i: ImmOpNum).ChangeToImmediate(ImmVal);
6067 }
6068 }
6069
6070 if (Reg.isVirtual() && MRI->use_nodbg_empty(RegNo: Reg))
6071 DefMI->eraseFromBundle();
6072
6073 return true;
6074}
6075
6076/// foldImmediate - 'Reg' is known to be defined by a move immediate
6077/// instruction, try to fold the immediate into the use instruction.
6078bool X86InstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
6079 Register Reg, MachineRegisterInfo *MRI) const {
6080 int64_t ImmVal;
6081 if (!getConstValDefinedInReg(MI: DefMI, Reg, ImmVal))
6082 return false;
6083
6084 return foldImmediateImpl(UseMI, DefMI: &DefMI, Reg, ImmVal, MRI, MakeChange: true);
6085}
6086
6087/// Expand a single-def pseudo instruction to a two-addr
6088/// instruction with two undef reads of the register being defined.
6089/// This is used for mapping:
6090/// %xmm4 = V_SET0
6091/// to:
6092/// %xmm4 = PXORrr undef %xmm4, undef %xmm4
6093///
6094static bool Expand2AddrUndef(MachineInstrBuilder &MIB,
6095 const MCInstrDesc &Desc) {
6096 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.");
6097 Register Reg = MIB.getReg(Idx: 0);
6098 MIB->setDesc(Desc);
6099
6100 // MachineInstr::addOperand() will insert explicit operands before any
6101 // implicit operands.
6102 MIB.addReg(RegNo: Reg, Flags: RegState::Undef).addReg(RegNo: Reg, Flags: RegState::Undef);
6103 // But we don't trust that.
6104 assert(MIB.getReg(1) == Reg && MIB.getReg(2) == Reg && "Misplaced operand");
6105 return true;
6106}
6107
6108/// Expand a single-def pseudo instruction to a two-addr
6109/// instruction with two %k0 reads.
6110/// This is used for mapping:
6111/// %k4 = K_SET1
6112/// to:
6113/// %k4 = KXNORrr %k0, %k0
6114static bool Expand2AddrKreg(MachineInstrBuilder &MIB, const MCInstrDesc &Desc,
6115 Register Reg) {
6116 assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.");
6117 MIB->setDesc(Desc);
6118 MIB.addReg(RegNo: Reg, Flags: RegState::Undef).addReg(RegNo: Reg, Flags: RegState::Undef);
6119 return true;
6120}
6121
6122static bool expandMOV32r1(MachineInstrBuilder &MIB, const TargetInstrInfo &TII,
6123 bool MinusOne) {
6124 MachineBasicBlock &MBB = *MIB->getParent();
6125 const DebugLoc &DL = MIB->getDebugLoc();
6126 Register Reg = MIB.getReg(Idx: 0);
6127
6128 // Insert the XOR.
6129 BuildMI(BB&: MBB, I: MIB.getInstr(), MIMD: DL, MCID: TII.get(Opcode: X86::XOR32rr), DestReg: Reg)
6130 .addReg(RegNo: Reg, Flags: RegState::Undef)
6131 .addReg(RegNo: Reg, Flags: RegState::Undef);
6132
6133 // Turn the pseudo into an INC or DEC.
6134 MIB->setDesc(TII.get(Opcode: MinusOne ? X86::DEC32r : X86::INC32r));
6135 MIB.addReg(RegNo: Reg);
6136
6137 return true;
6138}
6139
6140static bool ExpandMOVImmSExti8(MachineInstrBuilder &MIB,
6141 const TargetInstrInfo &TII,
6142 const X86Subtarget &Subtarget) {
6143 MachineBasicBlock &MBB = *MIB->getParent();
6144 const DebugLoc &DL = MIB->getDebugLoc();
6145 int64_t Imm = MIB->getOperand(i: 1).getImm();
6146 assert(Imm != 0 && "Using push/pop for 0 is not efficient.");
6147 MachineBasicBlock::iterator I = MIB.getInstr();
6148
6149 int StackAdjustment;
6150
6151 if (Subtarget.is64Bit()) {
6152 assert(MIB->getOpcode() == X86::MOV64ImmSExti8 ||
6153 MIB->getOpcode() == X86::MOV32ImmSExti8);
6154
6155 // Can't use push/pop lowering if the function might write to the red zone.
6156 X86MachineFunctionInfo *X86FI =
6157 MBB.getParent()->getInfo<X86MachineFunctionInfo>();
6158 if (X86FI->getUsesRedZone()) {
6159 MIB->setDesc(TII.get(Opcode: MIB->getOpcode() == X86::MOV32ImmSExti8
6160 ? X86::MOV32ri
6161 : X86::MOV64ri));
6162 return true;
6163 }
6164
6165 // 64-bit mode doesn't have 32-bit push/pop, so use 64-bit operations and
6166 // widen the register if necessary.
6167 StackAdjustment = 8;
6168 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH64i32)).addImm(Val: Imm);
6169 MIB->setDesc(TII.get(Opcode: X86::POP64r));
6170 MIB->getOperand(i: 0).setReg(getX86SubSuperRegister(Reg: MIB.getReg(Idx: 0), Size: 64));
6171 } else {
6172 assert(MIB->getOpcode() == X86::MOV32ImmSExti8);
6173 StackAdjustment = 4;
6174 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: X86::PUSH32i)).addImm(Val: Imm);
6175 MIB->setDesc(TII.get(Opcode: X86::POP32r));
6176 }
6177 MIB->removeOperand(OpNo: 1);
6178 MIB->addImplicitDefUseOperands(MF&: *MBB.getParent());
6179
6180 // Build CFI if necessary.
6181 MachineFunction &MF = *MBB.getParent();
6182 const X86FrameLowering *TFL = Subtarget.getFrameLowering();
6183 bool IsWin64Prologue = MF.getTarget().getMCAsmInfo().usesWindowsCFI();
6184 bool NeedsDwarfCFI = !IsWin64Prologue && MF.needsFrameMoves();
6185 bool EmitCFI = !TFL->hasFP(MF) && NeedsDwarfCFI;
6186 if (EmitCFI) {
6187 TFL->BuildCFI(
6188 MBB, MBBI: I, DL,
6189 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: StackAdjustment));
6190 TFL->BuildCFI(
6191 MBB, MBBI: std::next(x: I), DL,
6192 CFIInst: MCCFIInstruction::createAdjustCfaOffset(L: nullptr, Adjustment: -StackAdjustment));
6193 }
6194
6195 return true;
6196}
6197
6198// LoadStackGuard has so far only been implemented for 64-bit MachO. Different
6199// code sequence is needed for other targets.
6200static void expandLoadStackGuard(MachineInstrBuilder &MIB,
6201 const TargetInstrInfo &TII) {
6202 MachineBasicBlock &MBB = *MIB->getParent();
6203 const DebugLoc &DL = MIB->getDebugLoc();
6204 Register Reg = MIB.getReg(Idx: 0);
6205 const GlobalValue *GV =
6206 cast<GlobalValue>(Val: (*MIB->memoperands_begin())->getValue());
6207 auto Flags = MachineMemOperand::MOLoad |
6208 MachineMemOperand::MODereferenceable |
6209 MachineMemOperand::MOInvariant;
6210 MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
6211 PtrInfo: MachinePointerInfo::getGOT(MF&: *MBB.getParent()), F: Flags, Size: 8, BaseAlignment: Align(8));
6212 MachineBasicBlock::iterator I = MIB.getInstr();
6213
6214 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: X86::MOV64rm), DestReg: Reg)
6215 .addReg(RegNo: X86::RIP)
6216 .addImm(Val: 1)
6217 .addReg(RegNo: 0)
6218 .addGlobalAddress(GV, Offset: 0, TargetFlags: X86II::MO_GOTPCREL)
6219 .addReg(RegNo: 0)
6220 .addMemOperand(MMO);
6221 MIB->setDebugLoc(DL);
6222 MIB->setDesc(TII.get(Opcode: X86::MOV64rm));
6223 MIB.addReg(RegNo: Reg, Flags: RegState::Kill).addImm(Val: 1).addReg(RegNo: 0).addImm(Val: 0).addReg(RegNo: 0);
6224}
6225
6226static bool expandXorFP(MachineInstrBuilder &MIB, const TargetInstrInfo &TII) {
6227 MachineBasicBlock &MBB = *MIB->getParent();
6228 MachineFunction &MF = *MBB.getParent();
6229 const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
6230 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
6231 unsigned XorOp =
6232 MIB->getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr;
6233 MIB->setDesc(TII.get(Opcode: XorOp));
6234 MIB.addReg(RegNo: TRI->getFrameRegister(MF), Flags: RegState::Undef);
6235 return true;
6236}
6237
6238// This is used to handle spills for 128/256-bit registers when we have AVX512,
6239// but not VLX. If it uses an extended register we need to use an instruction
6240// that loads the lower 128/256-bit, but is available with only AVX512F.
6241static bool expandNOVLXLoad(MachineInstrBuilder &MIB,
6242 const TargetRegisterInfo *TRI,
6243 const MCInstrDesc &LoadDesc,
6244 const MCInstrDesc &BroadcastDesc, unsigned SubIdx) {
6245 Register DestReg = MIB.getReg(Idx: 0);
6246 // Check if DestReg is XMM16-31 or YMM16-31.
6247 if (TRI->getEncodingValue(Reg: DestReg) < 16) {
6248 // We can use a normal VEX encoded load.
6249 MIB->setDesc(LoadDesc);
6250 } else {
6251 // Use a 128/256-bit VBROADCAST instruction.
6252 MIB->setDesc(BroadcastDesc);
6253 // Change the destination to a 512-bit register.
6254 DestReg = TRI->getMatchingSuperReg(Reg: DestReg, SubIdx, RC: &X86::VR512RegClass);
6255 MIB->getOperand(i: 0).setReg(DestReg);
6256 }
6257 return true;
6258}
6259
6260// This is used to handle spills for 128/256-bit registers when we have AVX512,
6261// but not VLX. If it uses an extended register we need to use an instruction
6262// that stores the lower 128/256-bit, but is available with only AVX512F.
6263static bool expandNOVLXStore(MachineInstrBuilder &MIB,
6264 const TargetRegisterInfo *TRI,
6265 const MCInstrDesc &StoreDesc,
6266 const MCInstrDesc &ExtractDesc, unsigned SubIdx) {
6267 Register SrcReg = MIB.getReg(Idx: X86::AddrNumOperands);
6268 // Check if DestReg is XMM16-31 or YMM16-31.
6269 if (TRI->getEncodingValue(Reg: SrcReg) < 16) {
6270 // We can use a normal VEX encoded store.
6271 MIB->setDesc(StoreDesc);
6272 } else {
6273 // Use a VEXTRACTF instruction.
6274 MIB->setDesc(ExtractDesc);
6275 // Change the destination to a 512-bit register.
6276 SrcReg = TRI->getMatchingSuperReg(Reg: SrcReg, SubIdx, RC: &X86::VR512RegClass);
6277 MIB->getOperand(i: X86::AddrNumOperands).setReg(SrcReg);
6278 MIB.addImm(Val: 0x0); // Append immediate to extract from the lower bits.
6279 }
6280
6281 return true;
6282}
6283
6284static bool expandSHXDROT(MachineInstrBuilder &MIB, const MCInstrDesc &Desc) {
6285 MIB->setDesc(Desc);
6286 int64_t ShiftAmt = MIB->getOperand(i: 2).getImm();
6287 // Temporarily remove the immediate so we can add another source register.
6288 MIB->removeOperand(OpNo: 2);
6289 // Add the register. Don't copy the kill flag if there is one.
6290 MIB.addReg(RegNo: MIB.getReg(Idx: 1), Flags: getUndefRegState(B: MIB->getOperand(i: 1).isUndef()));
6291 // Add back the immediate.
6292 MIB.addImm(Val: ShiftAmt);
6293 return true;
6294}
6295
6296static bool expandMOVSHP(MachineInstrBuilder &MIB, MachineInstr &MI,
6297 const TargetInstrInfo &TII, bool HasAVX) {
6298 unsigned NewOpc;
6299 if (MI.getOpcode() == X86::MOVSHPrm) {
6300 NewOpc = HasAVX ? X86::VMOVSSrm : X86::MOVSSrm;
6301 Register Reg = MI.getOperand(i: 0).getReg();
6302 if (Reg > X86::XMM15)
6303 NewOpc = X86::VMOVSSZrm;
6304 } else {
6305 NewOpc = HasAVX ? X86::VMOVSSmr : X86::MOVSSmr;
6306 Register Reg = MI.getOperand(i: 5).getReg();
6307 if (Reg > X86::XMM15)
6308 NewOpc = X86::VMOVSSZmr;
6309 }
6310
6311 MIB->setDesc(TII.get(Opcode: NewOpc));
6312 return true;
6313}
6314
6315bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
6316 bool HasAVX = Subtarget.hasAVX();
6317 MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
6318 switch (MI.getOpcode()) {
6319 case X86::MOV32r0:
6320 return Expand2AddrUndef(MIB, Desc: get(Opcode: X86::XOR32rr));
6321 case X86::MOV32r1:
6322 return expandMOV32r1(MIB, TII: *this, /*MinusOne=*/false);
6323 case X86::MOV32r_1:
6324 return expandMOV32r1(MIB, TII: *this, /*MinusOne=*/true);
6325 case X86::MOV32ImmSExti8:
6326 case X86::MOV64ImmSExti8:
6327 return ExpandMOVImmSExti8(MIB, TII: *this, Subtarget);
6328 case X86::SETB_C32r:
6329 return Expand2AddrUndef(MIB, Desc: get(Opcode: X86::SBB32rr));
6330 case X86::SETB_C64r:
6331 return Expand2AddrUndef(MIB, Desc: get(Opcode: X86::SBB64rr));
6332 case X86::MMX_SET0:
6333 return Expand2AddrUndef(MIB, Desc: get(Opcode: X86::MMX_PXORrr));
6334 case X86::V_SET0:
6335 case X86::FsFLD0SS:
6336 case X86::FsFLD0SD:
6337 case X86::FsFLD0SH:
6338 case X86::FsFLD0F128:
6339 return Expand2AddrUndef(MIB, Desc: get(Opcode: HasAVX ? X86::VXORPSrr : X86::XORPSrr));
6340 case X86::AVX_SET0: {
6341 assert(HasAVX && "AVX not supported");
6342 const TargetRegisterInfo *TRI = &getRegisterInfo();
6343 Register SrcReg = MIB.getReg(Idx: 0);
6344 Register XReg = TRI->getSubReg(Reg: SrcReg, Idx: X86::sub_xmm);
6345 MIB->getOperand(i: 0).setReg(XReg);
6346 Expand2AddrUndef(MIB, Desc: get(Opcode: X86::VXORPSrr));
6347 MIB.addReg(RegNo: SrcReg, Flags: RegState::ImplicitDefine);
6348 return true;
6349 }
6350 case X86::AVX512_128_SET0:
6351 case X86::AVX512_FsFLD0SH:
6352 case X86::AVX512_FsFLD0SS:
6353 case X86::AVX512_FsFLD0SD:
6354 case X86::AVX512_FsFLD0F128: {
6355 bool HasVLX = Subtarget.hasVLX();
6356 Register SrcReg = MIB.getReg(Idx: 0);
6357 const TargetRegisterInfo *TRI = &getRegisterInfo();
6358 if (HasVLX || TRI->getEncodingValue(Reg: SrcReg) < 16)
6359 return Expand2AddrUndef(MIB,
6360 Desc: get(Opcode: HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
6361 // Extended register without VLX. Use a larger XOR.
6362 SrcReg =
6363 TRI->getMatchingSuperReg(Reg: SrcReg, SubIdx: X86::sub_xmm, RC: &X86::VR512RegClass);
6364 MIB->getOperand(i: 0).setReg(SrcReg);
6365 return Expand2AddrUndef(MIB, Desc: get(Opcode: X86::VPXORDZrr));
6366 }
6367 case X86::AVX512_256_SET0:
6368 case X86::AVX512_512_SET0: {
6369 bool HasVLX = Subtarget.hasVLX();
6370 Register SrcReg = MIB.getReg(Idx: 0);
6371 const TargetRegisterInfo *TRI = &getRegisterInfo();
6372 if (HasVLX || TRI->getEncodingValue(Reg: SrcReg) < 16) {
6373 Register XReg = TRI->getSubReg(Reg: SrcReg, Idx: X86::sub_xmm);
6374 MIB->getOperand(i: 0).setReg(XReg);
6375 Expand2AddrUndef(MIB, Desc: get(Opcode: HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
6376 MIB.addReg(RegNo: SrcReg, Flags: RegState::ImplicitDefine);
6377 return true;
6378 }
6379 if (MI.getOpcode() == X86::AVX512_256_SET0) {
6380 // No VLX so we must reference a zmm.
6381 MCRegister ZReg =
6382 TRI->getMatchingSuperReg(Reg: SrcReg, SubIdx: X86::sub_ymm, RC: &X86::VR512RegClass);
6383 MIB->getOperand(i: 0).setReg(ZReg);
6384 }
6385 return Expand2AddrUndef(MIB, Desc: get(Opcode: X86::VPXORDZrr));
6386 }
6387 case X86::MOVSHPmr:
6388 case X86::MOVSHPrm:
6389 return expandMOVSHP(MIB, MI, TII: *this, HasAVX: Subtarget.hasAVX());
6390 case X86::V_SETALLONES:
6391 return Expand2AddrUndef(MIB,
6392 Desc: get(Opcode: HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr));
6393 case X86::AVX2_SETALLONES:
6394 return Expand2AddrUndef(MIB, Desc: get(Opcode: X86::VPCMPEQDYrr));
6395 case X86::AVX1_SETALLONES: {
6396 Register Reg = MIB.getReg(Idx: 0);
6397 // VCMPPSYrri with an immediate 0xf should produce VCMPTRUEPS.
6398 MIB->setDesc(get(Opcode: X86::VCMPPSYrri));
6399 MIB.addReg(RegNo: Reg, Flags: RegState::Undef).addReg(RegNo: Reg, Flags: RegState::Undef).addImm(Val: 0xf);
6400 return true;
6401 }
6402 case X86::AVX512_128_SETALLONES:
6403 case X86::AVX512_256_SETALLONES:
6404 case X86::AVX512_512_SETALLONES: {
6405 Register Reg = MIB.getReg(Idx: 0);
6406 unsigned Opc;
6407 switch (MI.getOpcode()) {
6408 case X86::AVX512_128_SETALLONES: {
6409 if (X86::VR128RegClass.contains(Reg))
6410 return Expand2AddrUndef(MIB, Desc: get(Opcode: X86::VPCMPEQDrr));
6411
6412 Opc = X86::VPTERNLOGDZ128rri;
6413 break;
6414 }
6415 case X86::AVX512_256_SETALLONES: {
6416 if (X86::VR256RegClass.contains(Reg))
6417 return Expand2AddrUndef(MIB, Desc: get(Opcode: X86::VPCMPEQDYrr));
6418
6419 Opc = X86::VPTERNLOGDZ256rri;
6420 break;
6421 }
6422 case X86::AVX512_512_SETALLONES:
6423 Opc = X86::VPTERNLOGDZrri;
6424 break;
6425 }
6426 MIB->setDesc(get(Opcode: Opc));
6427 // VPTERNLOGD needs 3 register inputs and an immediate.
6428 // 0xff will return 1s for any input.
6429 MIB.addReg(RegNo: Reg, Flags: RegState::Undef)
6430 .addReg(RegNo: Reg, Flags: RegState::Undef)
6431 .addReg(RegNo: Reg, Flags: RegState::Undef)
6432 .addImm(Val: 0xff);
6433 return true;
6434 }
6435 case X86::AVX512_512_SEXT_MASK_32:
6436 case X86::AVX512_512_SEXT_MASK_64: {
6437 Register Reg = MIB.getReg(Idx: 0);
6438 Register MaskReg = MIB.getReg(Idx: 1);
6439 RegState MaskState = getRegState(RegOp: MIB->getOperand(i: 1));
6440 unsigned Opc = (MI.getOpcode() == X86::AVX512_512_SEXT_MASK_64)
6441 ? X86::VPTERNLOGQZrrikz
6442 : X86::VPTERNLOGDZrrikz;
6443 MI.removeOperand(OpNo: 1);
6444 MIB->setDesc(get(Opcode: Opc));
6445 // VPTERNLOG needs 3 register inputs and an immediate.
6446 // 0xff will return 1s for any input.
6447 MIB.addReg(RegNo: Reg, Flags: RegState::Undef)
6448 .addReg(RegNo: MaskReg, Flags: MaskState)
6449 .addReg(RegNo: Reg, Flags: RegState::Undef)
6450 .addReg(RegNo: Reg, Flags: RegState::Undef)
6451 .addImm(Val: 0xff);
6452 return true;
6453 }
6454 case X86::VMOVAPSZ128rm_NOVLX:
6455 return expandNOVLXLoad(MIB, TRI: &getRegisterInfo(), LoadDesc: get(Opcode: X86::VMOVAPSrm),
6456 BroadcastDesc: get(Opcode: X86::VBROADCASTF32X4Zrm), SubIdx: X86::sub_xmm);
6457 case X86::VMOVUPSZ128rm_NOVLX:
6458 return expandNOVLXLoad(MIB, TRI: &getRegisterInfo(), LoadDesc: get(Opcode: X86::VMOVUPSrm),
6459 BroadcastDesc: get(Opcode: X86::VBROADCASTF32X4Zrm), SubIdx: X86::sub_xmm);
6460 case X86::VMOVAPSZ256rm_NOVLX:
6461 return expandNOVLXLoad(MIB, TRI: &getRegisterInfo(), LoadDesc: get(Opcode: X86::VMOVAPSYrm),
6462 BroadcastDesc: get(Opcode: X86::VBROADCASTF64X4Zrm), SubIdx: X86::sub_ymm);
6463 case X86::VMOVUPSZ256rm_NOVLX:
6464 return expandNOVLXLoad(MIB, TRI: &getRegisterInfo(), LoadDesc: get(Opcode: X86::VMOVUPSYrm),
6465 BroadcastDesc: get(Opcode: X86::VBROADCASTF64X4Zrm), SubIdx: X86::sub_ymm);
6466 case X86::VMOVAPSZ128mr_NOVLX:
6467 return expandNOVLXStore(MIB, TRI: &getRegisterInfo(), StoreDesc: get(Opcode: X86::VMOVAPSmr),
6468 ExtractDesc: get(Opcode: X86::VEXTRACTF32X4Zmri), SubIdx: X86::sub_xmm);
6469 case X86::VMOVUPSZ128mr_NOVLX:
6470 return expandNOVLXStore(MIB, TRI: &getRegisterInfo(), StoreDesc: get(Opcode: X86::VMOVUPSmr),
6471 ExtractDesc: get(Opcode: X86::VEXTRACTF32X4Zmri), SubIdx: X86::sub_xmm);
6472 case X86::VMOVAPSZ256mr_NOVLX:
6473 return expandNOVLXStore(MIB, TRI: &getRegisterInfo(), StoreDesc: get(Opcode: X86::VMOVAPSYmr),
6474 ExtractDesc: get(Opcode: X86::VEXTRACTF64X4Zmri), SubIdx: X86::sub_ymm);
6475 case X86::VMOVUPSZ256mr_NOVLX:
6476 return expandNOVLXStore(MIB, TRI: &getRegisterInfo(), StoreDesc: get(Opcode: X86::VMOVUPSYmr),
6477 ExtractDesc: get(Opcode: X86::VEXTRACTF64X4Zmri), SubIdx: X86::sub_ymm);
6478 case X86::MOV32ri64: {
6479 Register Reg = MIB.getReg(Idx: 0);
6480 Register Reg32 = RI.getSubReg(Reg, Idx: X86::sub_32bit);
6481 MI.setDesc(get(Opcode: X86::MOV32ri));
6482 MIB->getOperand(i: 0).setReg(Reg32);
6483 MIB.addReg(RegNo: Reg, Flags: RegState::ImplicitDefine);
6484 return true;
6485 }
6486
6487 case X86::RDFLAGS32:
6488 case X86::RDFLAGS64: {
6489 unsigned Is64Bit = MI.getOpcode() == X86::RDFLAGS64;
6490 MachineBasicBlock &MBB = *MIB->getParent();
6491
6492 MachineInstr *NewMI = BuildMI(BB&: MBB, I&: MI, MIMD: MIB->getDebugLoc(),
6493 MCID: get(Opcode: Is64Bit ? X86::PUSHF64 : X86::PUSHF32))
6494 .getInstr();
6495
6496 // Permit reads of the EFLAGS and DF registers without them being defined.
6497 // This intrinsic exists to read external processor state in flags, such as
6498 // the trap flag, interrupt flag, and direction flag, none of which are
6499 // modeled by the backend.
6500 assert(NewMI->getOperand(2).getReg() == X86::EFLAGS &&
6501 "Unexpected register in operand! Should be EFLAGS.");
6502 NewMI->getOperand(i: 2).setIsUndef();
6503 assert(NewMI->getOperand(3).getReg() == X86::DF &&
6504 "Unexpected register in operand! Should be DF.");
6505 NewMI->getOperand(i: 3).setIsUndef();
6506
6507 MIB->setDesc(get(Opcode: Is64Bit ? X86::POP64r : X86::POP32r));
6508 return true;
6509 }
6510
6511 case X86::WRFLAGS32:
6512 case X86::WRFLAGS64: {
6513 unsigned Is64Bit = MI.getOpcode() == X86::WRFLAGS64;
6514 MachineBasicBlock &MBB = *MIB->getParent();
6515
6516 BuildMI(BB&: MBB, I&: MI, MIMD: MIB->getDebugLoc(),
6517 MCID: get(Opcode: Is64Bit ? X86::PUSH64r : X86::PUSH32r))
6518 .addReg(RegNo: MI.getOperand(i: 0).getReg());
6519 BuildMI(BB&: MBB, I&: MI, MIMD: MIB->getDebugLoc(),
6520 MCID: get(Opcode: Is64Bit ? X86::POPF64 : X86::POPF32));
6521 MI.eraseFromParent();
6522 return true;
6523 }
6524
6525 // KNL does not recognize dependency-breaking idioms for mask registers,
6526 // so kxnor %k1, %k1, %k2 has a RAW dependence on %k1.
6527 // Using %k0 as the undef input register is a performance heuristic based
6528 // on the assumption that %k0 is used less frequently than the other mask
6529 // registers, since it is not usable as a write mask.
6530 // FIXME: A more advanced approach would be to choose the best input mask
6531 // register based on context.
6532 case X86::KSET0B:
6533 return Expand2AddrKreg(MIB, Desc: get(Opcode: X86::KXORBkk), Reg: X86::K0);
6534 case X86::KSET0W:
6535 return Expand2AddrKreg(MIB, Desc: get(Opcode: X86::KXORWkk), Reg: X86::K0);
6536 case X86::KSET0D:
6537 return Expand2AddrKreg(MIB, Desc: get(Opcode: X86::KXORDkk), Reg: X86::K0);
6538 case X86::KSET0Q:
6539 return Expand2AddrKreg(MIB, Desc: get(Opcode: X86::KXORQkk), Reg: X86::K0);
6540 case X86::KSET1B:
6541 return Expand2AddrKreg(MIB, Desc: get(Opcode: X86::KXNORBkk), Reg: X86::K0);
6542 case X86::KSET1W:
6543 return Expand2AddrKreg(MIB, Desc: get(Opcode: X86::KXNORWkk), Reg: X86::K0);
6544 case X86::KSET1D:
6545 return Expand2AddrKreg(MIB, Desc: get(Opcode: X86::KXNORDkk), Reg: X86::K0);
6546 case X86::KSET1Q:
6547 return Expand2AddrKreg(MIB, Desc: get(Opcode: X86::KXNORQkk), Reg: X86::K0);
6548 case TargetOpcode::LOAD_STACK_GUARD:
6549 expandLoadStackGuard(MIB, TII: *this);
6550 return true;
6551 case X86::XOR64_FP:
6552 case X86::XOR32_FP:
6553 return expandXorFP(MIB, TII: *this);
6554 case X86::SHLDROT32ri:
6555 return expandSHXDROT(MIB, Desc: get(Opcode: X86::SHLD32rri8));
6556 case X86::SHLDROT64ri:
6557 return expandSHXDROT(MIB, Desc: get(Opcode: X86::SHLD64rri8));
6558 case X86::SHRDROT32ri:
6559 return expandSHXDROT(MIB, Desc: get(Opcode: X86::SHRD32rri8));
6560 case X86::SHRDROT64ri:
6561 return expandSHXDROT(MIB, Desc: get(Opcode: X86::SHRD64rri8));
6562 case X86::ADD8rr_DB:
6563 MIB->setDesc(get(Opcode: X86::OR8rr));
6564 break;
6565 case X86::ADD16rr_DB:
6566 MIB->setDesc(get(Opcode: X86::OR16rr));
6567 break;
6568 case X86::ADD32rr_DB:
6569 MIB->setDesc(get(Opcode: X86::OR32rr));
6570 break;
6571 case X86::ADD64rr_DB:
6572 MIB->setDesc(get(Opcode: X86::OR64rr));
6573 break;
6574 case X86::ADD8ri_DB:
6575 MIB->setDesc(get(Opcode: X86::OR8ri));
6576 break;
6577 case X86::ADD16ri_DB:
6578 MIB->setDesc(get(Opcode: X86::OR16ri));
6579 break;
6580 case X86::ADD32ri_DB:
6581 MIB->setDesc(get(Opcode: X86::OR32ri));
6582 break;
6583 case X86::ADD64ri32_DB:
6584 MIB->setDesc(get(Opcode: X86::OR64ri32));
6585 break;
6586 }
6587 return false;
6588}
6589
6590/// Return true for all instructions that only update
6591/// the first 32 or 64-bits of the destination register and leave the rest
6592/// unmodified. This can be used to avoid folding loads if the instructions
6593/// only update part of the destination register, and the non-updated part is
6594/// not needed. e.g. cvtss2sd, sqrtss. Unfolding the load from these
6595/// instructions breaks the partial register dependency and it can improve
6596/// performance. e.g.:
6597///
6598/// movss (%rdi), %xmm0
6599/// cvtss2sd %xmm0, %xmm0
6600///
6601/// Instead of
6602/// cvtss2sd (%rdi), %xmm0
6603///
6604/// FIXME: This should be turned into a TSFlags.
6605///
6606static bool hasPartialRegUpdate(unsigned Opcode, const X86Subtarget &Subtarget,
6607 bool ForLoadFold = false) {
6608 switch (Opcode) {
6609 case X86::CVTSI2SSrr:
6610 case X86::CVTSI2SSrm:
6611 case X86::CVTSI642SSrr:
6612 case X86::CVTSI642SSrm:
6613 case X86::CVTSI2SDrr:
6614 case X86::CVTSI2SDrm:
6615 case X86::CVTSI642SDrr:
6616 case X86::CVTSI642SDrm:
6617 // Load folding won't effect the undef register update since the input is
6618 // a GPR.
6619 return !ForLoadFold;
6620 case X86::CVTSD2SSrr:
6621 case X86::CVTSD2SSrm:
6622 case X86::CVTSS2SDrr:
6623 case X86::CVTSS2SDrm:
6624 case X86::MOVHPDrm:
6625 case X86::MOVHPSrm:
6626 case X86::MOVLPDrm:
6627 case X86::MOVLPSrm:
6628 case X86::RCPSSr:
6629 case X86::RCPSSm:
6630 case X86::RCPSSr_Int:
6631 case X86::RCPSSm_Int:
6632 case X86::ROUNDSDri:
6633 case X86::ROUNDSDmi:
6634 case X86::ROUNDSSri:
6635 case X86::ROUNDSSmi:
6636 case X86::RSQRTSSr:
6637 case X86::RSQRTSSm:
6638 case X86::RSQRTSSr_Int:
6639 case X86::RSQRTSSm_Int:
6640 case X86::SQRTSSr:
6641 case X86::SQRTSSm:
6642 case X86::SQRTSSr_Int:
6643 case X86::SQRTSSm_Int:
6644 case X86::SQRTSDr:
6645 case X86::SQRTSDm:
6646 case X86::SQRTSDr_Int:
6647 case X86::SQRTSDm_Int:
6648 return true;
6649 case X86::VFCMULCPHZ128rm:
6650 case X86::VFCMULCPHZ128rmb:
6651 case X86::VFCMULCPHZ128rmbkz:
6652 case X86::VFCMULCPHZ128rmkz:
6653 case X86::VFCMULCPHZ128rr:
6654 case X86::VFCMULCPHZ128rrkz:
6655 case X86::VFCMULCPHZ256rm:
6656 case X86::VFCMULCPHZ256rmb:
6657 case X86::VFCMULCPHZ256rmbkz:
6658 case X86::VFCMULCPHZ256rmkz:
6659 case X86::VFCMULCPHZ256rr:
6660 case X86::VFCMULCPHZ256rrkz:
6661 case X86::VFCMULCPHZrm:
6662 case X86::VFCMULCPHZrmb:
6663 case X86::VFCMULCPHZrmbkz:
6664 case X86::VFCMULCPHZrmkz:
6665 case X86::VFCMULCPHZrr:
6666 case X86::VFCMULCPHZrrb:
6667 case X86::VFCMULCPHZrrbkz:
6668 case X86::VFCMULCPHZrrkz:
6669 case X86::VFMULCPHZ128rm:
6670 case X86::VFMULCPHZ128rmb:
6671 case X86::VFMULCPHZ128rmbkz:
6672 case X86::VFMULCPHZ128rmkz:
6673 case X86::VFMULCPHZ128rr:
6674 case X86::VFMULCPHZ128rrkz:
6675 case X86::VFMULCPHZ256rm:
6676 case X86::VFMULCPHZ256rmb:
6677 case X86::VFMULCPHZ256rmbkz:
6678 case X86::VFMULCPHZ256rmkz:
6679 case X86::VFMULCPHZ256rr:
6680 case X86::VFMULCPHZ256rrkz:
6681 case X86::VFMULCPHZrm:
6682 case X86::VFMULCPHZrmb:
6683 case X86::VFMULCPHZrmbkz:
6684 case X86::VFMULCPHZrmkz:
6685 case X86::VFMULCPHZrr:
6686 case X86::VFMULCPHZrrb:
6687 case X86::VFMULCPHZrrbkz:
6688 case X86::VFMULCPHZrrkz:
6689 case X86::VFCMULCSHZrm:
6690 case X86::VFCMULCSHZrmkz:
6691 case X86::VFCMULCSHZrr:
6692 case X86::VFCMULCSHZrrb:
6693 case X86::VFCMULCSHZrrbkz:
6694 case X86::VFCMULCSHZrrkz:
6695 case X86::VFMULCSHZrm:
6696 case X86::VFMULCSHZrmkz:
6697 case X86::VFMULCSHZrr:
6698 case X86::VFMULCSHZrrb:
6699 case X86::VFMULCSHZrrbkz:
6700 case X86::VFMULCSHZrrkz:
6701 return Subtarget.hasMULCFalseDeps();
6702 case X86::VPERMDYrm:
6703 case X86::VPERMDYrr:
6704 case X86::VPERMQYmi:
6705 case X86::VPERMQYri:
6706 case X86::VPERMPSYrm:
6707 case X86::VPERMPSYrr:
6708 case X86::VPERMPDYmi:
6709 case X86::VPERMPDYri:
6710 case X86::VPERMDZ256rm:
6711 case X86::VPERMDZ256rmb:
6712 case X86::VPERMDZ256rmbkz:
6713 case X86::VPERMDZ256rmkz:
6714 case X86::VPERMDZ256rr:
6715 case X86::VPERMDZ256rrkz:
6716 case X86::VPERMDZrm:
6717 case X86::VPERMDZrmb:
6718 case X86::VPERMDZrmbkz:
6719 case X86::VPERMDZrmkz:
6720 case X86::VPERMDZrr:
6721 case X86::VPERMDZrrkz:
6722 case X86::VPERMQZ256mbi:
6723 case X86::VPERMQZ256mbikz:
6724 case X86::VPERMQZ256mi:
6725 case X86::VPERMQZ256mikz:
6726 case X86::VPERMQZ256ri:
6727 case X86::VPERMQZ256rikz:
6728 case X86::VPERMQZ256rm:
6729 case X86::VPERMQZ256rmb:
6730 case X86::VPERMQZ256rmbkz:
6731 case X86::VPERMQZ256rmkz:
6732 case X86::VPERMQZ256rr:
6733 case X86::VPERMQZ256rrkz:
6734 case X86::VPERMQZmbi:
6735 case X86::VPERMQZmbikz:
6736 case X86::VPERMQZmi:
6737 case X86::VPERMQZmikz:
6738 case X86::VPERMQZri:
6739 case X86::VPERMQZrikz:
6740 case X86::VPERMQZrm:
6741 case X86::VPERMQZrmb:
6742 case X86::VPERMQZrmbkz:
6743 case X86::VPERMQZrmkz:
6744 case X86::VPERMQZrr:
6745 case X86::VPERMQZrrkz:
6746 case X86::VPERMPSZ256rm:
6747 case X86::VPERMPSZ256rmb:
6748 case X86::VPERMPSZ256rmbkz:
6749 case X86::VPERMPSZ256rmkz:
6750 case X86::VPERMPSZ256rr:
6751 case X86::VPERMPSZ256rrkz:
6752 case X86::VPERMPSZrm:
6753 case X86::VPERMPSZrmb:
6754 case X86::VPERMPSZrmbkz:
6755 case X86::VPERMPSZrmkz:
6756 case X86::VPERMPSZrr:
6757 case X86::VPERMPSZrrkz:
6758 case X86::VPERMPDZ256mbi:
6759 case X86::VPERMPDZ256mbikz:
6760 case X86::VPERMPDZ256mi:
6761 case X86::VPERMPDZ256mikz:
6762 case X86::VPERMPDZ256ri:
6763 case X86::VPERMPDZ256rikz:
6764 case X86::VPERMPDZ256rm:
6765 case X86::VPERMPDZ256rmb:
6766 case X86::VPERMPDZ256rmbkz:
6767 case X86::VPERMPDZ256rmkz:
6768 case X86::VPERMPDZ256rr:
6769 case X86::VPERMPDZ256rrkz:
6770 case X86::VPERMPDZmbi:
6771 case X86::VPERMPDZmbikz:
6772 case X86::VPERMPDZmi:
6773 case X86::VPERMPDZmikz:
6774 case X86::VPERMPDZri:
6775 case X86::VPERMPDZrikz:
6776 case X86::VPERMPDZrm:
6777 case X86::VPERMPDZrmb:
6778 case X86::VPERMPDZrmbkz:
6779 case X86::VPERMPDZrmkz:
6780 case X86::VPERMPDZrr:
6781 case X86::VPERMPDZrrkz:
6782 return Subtarget.hasPERMFalseDeps();
6783 case X86::VRANGEPDZ128rmbi:
6784 case X86::VRANGEPDZ128rmbikz:
6785 case X86::VRANGEPDZ128rmi:
6786 case X86::VRANGEPDZ128rmikz:
6787 case X86::VRANGEPDZ128rri:
6788 case X86::VRANGEPDZ128rrikz:
6789 case X86::VRANGEPDZ256rmbi:
6790 case X86::VRANGEPDZ256rmbikz:
6791 case X86::VRANGEPDZ256rmi:
6792 case X86::VRANGEPDZ256rmikz:
6793 case X86::VRANGEPDZ256rri:
6794 case X86::VRANGEPDZ256rrikz:
6795 case X86::VRANGEPDZrmbi:
6796 case X86::VRANGEPDZrmbikz:
6797 case X86::VRANGEPDZrmi:
6798 case X86::VRANGEPDZrmikz:
6799 case X86::VRANGEPDZrri:
6800 case X86::VRANGEPDZrrib:
6801 case X86::VRANGEPDZrribkz:
6802 case X86::VRANGEPDZrrikz:
6803 case X86::VRANGEPSZ128rmbi:
6804 case X86::VRANGEPSZ128rmbikz:
6805 case X86::VRANGEPSZ128rmi:
6806 case X86::VRANGEPSZ128rmikz:
6807 case X86::VRANGEPSZ128rri:
6808 case X86::VRANGEPSZ128rrikz:
6809 case X86::VRANGEPSZ256rmbi:
6810 case X86::VRANGEPSZ256rmbikz:
6811 case X86::VRANGEPSZ256rmi:
6812 case X86::VRANGEPSZ256rmikz:
6813 case X86::VRANGEPSZ256rri:
6814 case X86::VRANGEPSZ256rrikz:
6815 case X86::VRANGEPSZrmbi:
6816 case X86::VRANGEPSZrmbikz:
6817 case X86::VRANGEPSZrmi:
6818 case X86::VRANGEPSZrmikz:
6819 case X86::VRANGEPSZrri:
6820 case X86::VRANGEPSZrrib:
6821 case X86::VRANGEPSZrribkz:
6822 case X86::VRANGEPSZrrikz:
6823 case X86::VRANGESDZrmi:
6824 case X86::VRANGESDZrmikz:
6825 case X86::VRANGESDZrri:
6826 case X86::VRANGESDZrrib:
6827 case X86::VRANGESDZrribkz:
6828 case X86::VRANGESDZrrikz:
6829 case X86::VRANGESSZrmi:
6830 case X86::VRANGESSZrmikz:
6831 case X86::VRANGESSZrri:
6832 case X86::VRANGESSZrrib:
6833 case X86::VRANGESSZrribkz:
6834 case X86::VRANGESSZrrikz:
6835 return Subtarget.hasRANGEFalseDeps();
6836 case X86::VGETMANTSSZrmi:
6837 case X86::VGETMANTSSZrmikz:
6838 case X86::VGETMANTSSZrri:
6839 case X86::VGETMANTSSZrrib:
6840 case X86::VGETMANTSSZrribkz:
6841 case X86::VGETMANTSSZrrikz:
6842 case X86::VGETMANTSDZrmi:
6843 case X86::VGETMANTSDZrmikz:
6844 case X86::VGETMANTSDZrri:
6845 case X86::VGETMANTSDZrrib:
6846 case X86::VGETMANTSDZrribkz:
6847 case X86::VGETMANTSDZrrikz:
6848 case X86::VGETMANTSHZrmi:
6849 case X86::VGETMANTSHZrmikz:
6850 case X86::VGETMANTSHZrri:
6851 case X86::VGETMANTSHZrrib:
6852 case X86::VGETMANTSHZrribkz:
6853 case X86::VGETMANTSHZrrikz:
6854 case X86::VGETMANTPSZ128rmbi:
6855 case X86::VGETMANTPSZ128rmbikz:
6856 case X86::VGETMANTPSZ128rmi:
6857 case X86::VGETMANTPSZ128rmikz:
6858 case X86::VGETMANTPSZ256rmbi:
6859 case X86::VGETMANTPSZ256rmbikz:
6860 case X86::VGETMANTPSZ256rmi:
6861 case X86::VGETMANTPSZ256rmikz:
6862 case X86::VGETMANTPSZrmbi:
6863 case X86::VGETMANTPSZrmbikz:
6864 case X86::VGETMANTPSZrmi:
6865 case X86::VGETMANTPSZrmikz:
6866 case X86::VGETMANTPDZ128rmbi:
6867 case X86::VGETMANTPDZ128rmbikz:
6868 case X86::VGETMANTPDZ128rmi:
6869 case X86::VGETMANTPDZ128rmikz:
6870 case X86::VGETMANTPDZ256rmbi:
6871 case X86::VGETMANTPDZ256rmbikz:
6872 case X86::VGETMANTPDZ256rmi:
6873 case X86::VGETMANTPDZ256rmikz:
6874 case X86::VGETMANTPDZrmbi:
6875 case X86::VGETMANTPDZrmbikz:
6876 case X86::VGETMANTPDZrmi:
6877 case X86::VGETMANTPDZrmikz:
6878 return Subtarget.hasGETMANTFalseDeps();
6879 case X86::VPMULLQZ128rm:
6880 case X86::VPMULLQZ128rmb:
6881 case X86::VPMULLQZ128rmbkz:
6882 case X86::VPMULLQZ128rmkz:
6883 case X86::VPMULLQZ128rr:
6884 case X86::VPMULLQZ128rrkz:
6885 case X86::VPMULLQZ256rm:
6886 case X86::VPMULLQZ256rmb:
6887 case X86::VPMULLQZ256rmbkz:
6888 case X86::VPMULLQZ256rmkz:
6889 case X86::VPMULLQZ256rr:
6890 case X86::VPMULLQZ256rrkz:
6891 case X86::VPMULLQZrm:
6892 case X86::VPMULLQZrmb:
6893 case X86::VPMULLQZrmbkz:
6894 case X86::VPMULLQZrmkz:
6895 case X86::VPMULLQZrr:
6896 case X86::VPMULLQZrrkz:
6897 return Subtarget.hasMULLQFalseDeps();
6898 case X86::VPCOMPRESSBZ128rrkz:
6899 case X86::VPCOMPRESSBZ256rrkz:
6900 case X86::VPCOMPRESSBZrrkz:
6901 case X86::VPCOMPRESSWZ128rrkz:
6902 case X86::VPCOMPRESSWZ256rrkz:
6903 case X86::VPCOMPRESSWZrrkz:
6904 case X86::VPCOMPRESSDZ128rrkz:
6905 case X86::VPCOMPRESSDZ256rrkz:
6906 case X86::VPCOMPRESSDZrrkz:
6907 case X86::VPCOMPRESSQZ128rrkz:
6908 case X86::VPCOMPRESSQZ256rrkz:
6909 case X86::VPCOMPRESSQZrrkz:
6910 case X86::VCOMPRESSPSZ128rrkz:
6911 case X86::VCOMPRESSPSZ256rrkz:
6912 case X86::VCOMPRESSPSZrrkz:
6913 case X86::VCOMPRESSPDZ128rrkz:
6914 case X86::VCOMPRESSPDZ256rrkz:
6915 case X86::VCOMPRESSPDZrrkz:
6916 return Subtarget.hasCOMPRESSFalseDeps();
6917 case X86::VPEXPANDBZ128rmkz:
6918 case X86::VPEXPANDBZ128rrkz:
6919 case X86::VPEXPANDBZ256rmkz:
6920 case X86::VPEXPANDBZ256rrkz:
6921 case X86::VPEXPANDBZrmkz:
6922 case X86::VPEXPANDBZrrkz:
6923 case X86::VPEXPANDWZ128rmkz:
6924 case X86::VPEXPANDWZ128rrkz:
6925 case X86::VPEXPANDWZ256rmkz:
6926 case X86::VPEXPANDWZ256rrkz:
6927 case X86::VPEXPANDWZrmkz:
6928 case X86::VPEXPANDWZrrkz:
6929 case X86::VPEXPANDDZ128rmkz:
6930 case X86::VPEXPANDDZ128rrkz:
6931 case X86::VPEXPANDDZ256rmkz:
6932 case X86::VPEXPANDDZ256rrkz:
6933 case X86::VPEXPANDDZrmkz:
6934 case X86::VPEXPANDDZrrkz:
6935 case X86::VPEXPANDQZ128rmkz:
6936 case X86::VPEXPANDQZ128rrkz:
6937 case X86::VPEXPANDQZ256rmkz:
6938 case X86::VPEXPANDQZ256rrkz:
6939 case X86::VPEXPANDQZrmkz:
6940 case X86::VPEXPANDQZrrkz:
6941 case X86::VEXPANDPSZ128rmkz:
6942 case X86::VEXPANDPSZ128rrkz:
6943 case X86::VEXPANDPSZ256rmkz:
6944 case X86::VEXPANDPSZ256rrkz:
6945 case X86::VEXPANDPSZrmkz:
6946 case X86::VEXPANDPSZrrkz:
6947 case X86::VEXPANDPDZ128rmkz:
6948 case X86::VEXPANDPDZ128rrkz:
6949 case X86::VEXPANDPDZ256rmkz:
6950 case X86::VEXPANDPDZ256rrkz:
6951 case X86::VEXPANDPDZrmkz:
6952 case X86::VEXPANDPDZrrkz:
6953 return Subtarget.hasEXPANDFalseDeps();
6954 // GPR
6955 case X86::POPCNT32rm:
6956 case X86::POPCNT32rr:
6957 case X86::POPCNT64rm:
6958 case X86::POPCNT64rr:
6959 return Subtarget.hasPOPCNTFalseDeps();
6960 case X86::LZCNT32rm:
6961 case X86::LZCNT32rr:
6962 case X86::LZCNT64rm:
6963 case X86::LZCNT64rr:
6964 return Subtarget.hasLZCNTFalseDeps();
6965 case X86::TZCNT32rm:
6966 case X86::TZCNT32rr:
6967 case X86::TZCNT64rm:
6968 case X86::TZCNT64rr:
6969 return Subtarget.hasTZCNTFalseDeps();
6970 case X86::BLSR32rr:
6971 case X86::BLSR32rm:
6972 case X86::BLSR64rr:
6973 case X86::BLSR64rm:
6974 case X86::BLSI32rr:
6975 case X86::BLSI32rm:
6976 case X86::BLSI64rr:
6977 case X86::BLSI64rm:
6978 case X86::BLSMSK32rr:
6979 case X86::BLSMSK32rm:
6980 case X86::BLSMSK64rr:
6981 case X86::BLSMSK64rm:
6982 return Subtarget.hasBLSFalseDeps() && !ForLoadFold; // Preserve load folding
6983 }
6984
6985 return false;
6986}
6987
6988/// Inform the BreakFalseDeps pass how many idle
6989/// instructions we would like before a partial register update.
6990unsigned X86InstrInfo::getPartialRegUpdateClearance(
6991 const MachineInstr &MI, unsigned OpNum,
6992 const TargetRegisterInfo *TRI) const {
6993
6994 if (OpNum != 0)
6995 return 0;
6996
6997 // NDD ops with 8/16b results may appear to be partial register
6998 // updates after register allocation.
6999 bool HasNDDPartialWrite = false;
7000 if (X86II::hasNewDataDest(TSFlags: MI.getDesc().TSFlags)) {
7001 Register Reg = MI.getOperand(i: 0).getReg();
7002 if (!Reg.isVirtual())
7003 HasNDDPartialWrite =
7004 X86::GR8RegClass.contains(Reg) || X86::GR16RegClass.contains(Reg);
7005 }
7006
7007 if (!(HasNDDPartialWrite || hasPartialRegUpdate(Opcode: MI.getOpcode(), Subtarget)))
7008 return 0;
7009
7010 // Check if the result register is also used as a source.
7011 // For non-NDD ops, this means a partial update is wanted, hence we return 0.
7012 // For NDD ops, this means it is possible to compress the instruction
7013 // to a legacy form in CompressEVEX, which would create an unwanted partial
7014 // update, so we return the clearance.
7015 const MachineOperand &MO = MI.getOperand(i: 0);
7016 Register Reg = MO.getReg();
7017 bool ReadsReg = false;
7018 if (Reg.isVirtual())
7019 ReadsReg = (MO.readsReg() || MI.readsVirtualRegister(Reg));
7020 else
7021 ReadsReg = MI.readsRegister(Reg, TRI);
7022 if (ReadsReg != HasNDDPartialWrite)
7023 return 0;
7024
7025 // If any instructions in the clearance range are reading Reg, insert a
7026 // dependency breaking instruction, which is inexpensive and is likely to
7027 // be hidden in other instruction's cycles.
7028 return PartialRegUpdateClearance;
7029}
7030
7031// Return true for any instruction the copies the high bits of the first source
7032// operand into the unused high bits of the destination operand.
7033// Also returns true for instructions that have two inputs where one may
7034// be undef and we want it to use the same register as the other input.
7035static bool hasUndefRegUpdate(unsigned Opcode, unsigned OpNum,
7036 bool ForLoadFold = false) {
7037 // Set the OpNum parameter to the first source operand.
7038 switch (Opcode) {
7039 case X86::MMX_PUNPCKHBWrr:
7040 case X86::MMX_PUNPCKHWDrr:
7041 case X86::MMX_PUNPCKHDQrr:
7042 case X86::MMX_PUNPCKLBWrr:
7043 case X86::MMX_PUNPCKLWDrr:
7044 case X86::MMX_PUNPCKLDQrr:
7045 case X86::MOVHLPSrr:
7046 case X86::PACKSSWBrr:
7047 case X86::PACKUSWBrr:
7048 case X86::PACKSSDWrr:
7049 case X86::PACKUSDWrr:
7050 case X86::PUNPCKHBWrr:
7051 case X86::PUNPCKLBWrr:
7052 case X86::PUNPCKHWDrr:
7053 case X86::PUNPCKLWDrr:
7054 case X86::PUNPCKHDQrr:
7055 case X86::PUNPCKLDQrr:
7056 case X86::PUNPCKHQDQrr:
7057 case X86::PUNPCKLQDQrr:
7058 case X86::SHUFPDrri:
7059 case X86::SHUFPSrri:
7060 // These instructions are sometimes used with an undef first or second
7061 // source. Return true here so BreakFalseDeps will assign this source to the
7062 // same register as the first source to avoid a false dependency.
7063 // Operand 1 of these instructions is tied so they're separate from their
7064 // VEX counterparts.
7065 return OpNum == 2 && !ForLoadFold;
7066
7067 case X86::VMOVLHPSrr:
7068 case X86::VMOVLHPSZrr:
7069 case X86::VPACKSSWBrr:
7070 case X86::VPACKUSWBrr:
7071 case X86::VPACKSSDWrr:
7072 case X86::VPACKUSDWrr:
7073 case X86::VPACKSSWBZ128rr:
7074 case X86::VPACKUSWBZ128rr:
7075 case X86::VPACKSSDWZ128rr:
7076 case X86::VPACKUSDWZ128rr:
7077 case X86::VPERM2F128rri:
7078 case X86::VPERM2I128rri:
7079 case X86::VSHUFF32X4Z256rri:
7080 case X86::VSHUFF32X4Zrri:
7081 case X86::VSHUFF64X2Z256rri:
7082 case X86::VSHUFF64X2Zrri:
7083 case X86::VSHUFI32X4Z256rri:
7084 case X86::VSHUFI32X4Zrri:
7085 case X86::VSHUFI64X2Z256rri:
7086 case X86::VSHUFI64X2Zrri:
7087 case X86::VPUNPCKHBWrr:
7088 case X86::VPUNPCKLBWrr:
7089 case X86::VPUNPCKHBWYrr:
7090 case X86::VPUNPCKLBWYrr:
7091 case X86::VPUNPCKHBWZ128rr:
7092 case X86::VPUNPCKLBWZ128rr:
7093 case X86::VPUNPCKHBWZ256rr:
7094 case X86::VPUNPCKLBWZ256rr:
7095 case X86::VPUNPCKHBWZrr:
7096 case X86::VPUNPCKLBWZrr:
7097 case X86::VPUNPCKHWDrr:
7098 case X86::VPUNPCKLWDrr:
7099 case X86::VPUNPCKHWDYrr:
7100 case X86::VPUNPCKLWDYrr:
7101 case X86::VPUNPCKHWDZ128rr:
7102 case X86::VPUNPCKLWDZ128rr:
7103 case X86::VPUNPCKHWDZ256rr:
7104 case X86::VPUNPCKLWDZ256rr:
7105 case X86::VPUNPCKHWDZrr:
7106 case X86::VPUNPCKLWDZrr:
7107 case X86::VPUNPCKHDQrr:
7108 case X86::VPUNPCKLDQrr:
7109 case X86::VPUNPCKHDQYrr:
7110 case X86::VPUNPCKLDQYrr:
7111 case X86::VPUNPCKHDQZ128rr:
7112 case X86::VPUNPCKLDQZ128rr:
7113 case X86::VPUNPCKHDQZ256rr:
7114 case X86::VPUNPCKLDQZ256rr:
7115 case X86::VPUNPCKHDQZrr:
7116 case X86::VPUNPCKLDQZrr:
7117 case X86::VPUNPCKHQDQrr:
7118 case X86::VPUNPCKLQDQrr:
7119 case X86::VPUNPCKHQDQYrr:
7120 case X86::VPUNPCKLQDQYrr:
7121 case X86::VPUNPCKHQDQZ128rr:
7122 case X86::VPUNPCKLQDQZ128rr:
7123 case X86::VPUNPCKHQDQZ256rr:
7124 case X86::VPUNPCKLQDQZ256rr:
7125 case X86::VPUNPCKHQDQZrr:
7126 case X86::VPUNPCKLQDQZrr:
7127 // These instructions are sometimes used with an undef first or second
7128 // source. Return true here so BreakFalseDeps will assign this source to the
7129 // same register as the first source to avoid a false dependency.
7130 return (OpNum == 1 || OpNum == 2) && !ForLoadFold;
7131
7132 case X86::VCVTSI2SSrr:
7133 case X86::VCVTSI2SSrm:
7134 case X86::VCVTSI2SSrr_Int:
7135 case X86::VCVTSI2SSrm_Int:
7136 case X86::VCVTSI642SSrr:
7137 case X86::VCVTSI642SSrm:
7138 case X86::VCVTSI642SSrr_Int:
7139 case X86::VCVTSI642SSrm_Int:
7140 case X86::VCVTSI2SDrr:
7141 case X86::VCVTSI2SDrm:
7142 case X86::VCVTSI2SDrr_Int:
7143 case X86::VCVTSI2SDrm_Int:
7144 case X86::VCVTSI642SDrr:
7145 case X86::VCVTSI642SDrm:
7146 case X86::VCVTSI642SDrr_Int:
7147 case X86::VCVTSI642SDrm_Int:
7148 // AVX-512
7149 case X86::VCVTSI2SSZrr:
7150 case X86::VCVTSI2SSZrm:
7151 case X86::VCVTSI2SSZrr_Int:
7152 case X86::VCVTSI2SSZrrb_Int:
7153 case X86::VCVTSI2SSZrm_Int:
7154 case X86::VCVTSI642SSZrr:
7155 case X86::VCVTSI642SSZrm:
7156 case X86::VCVTSI642SSZrr_Int:
7157 case X86::VCVTSI642SSZrrb_Int:
7158 case X86::VCVTSI642SSZrm_Int:
7159 case X86::VCVTSI2SDZrr:
7160 case X86::VCVTSI2SDZrm:
7161 case X86::VCVTSI2SDZrr_Int:
7162 case X86::VCVTSI2SDZrm_Int:
7163 case X86::VCVTSI642SDZrr:
7164 case X86::VCVTSI642SDZrm:
7165 case X86::VCVTSI642SDZrr_Int:
7166 case X86::VCVTSI642SDZrrb_Int:
7167 case X86::VCVTSI642SDZrm_Int:
7168 case X86::VCVTUSI2SSZrr:
7169 case X86::VCVTUSI2SSZrm:
7170 case X86::VCVTUSI2SSZrr_Int:
7171 case X86::VCVTUSI2SSZrrb_Int:
7172 case X86::VCVTUSI2SSZrm_Int:
7173 case X86::VCVTUSI642SSZrr:
7174 case X86::VCVTUSI642SSZrm:
7175 case X86::VCVTUSI642SSZrr_Int:
7176 case X86::VCVTUSI642SSZrrb_Int:
7177 case X86::VCVTUSI642SSZrm_Int:
7178 case X86::VCVTUSI2SDZrr:
7179 case X86::VCVTUSI2SDZrm:
7180 case X86::VCVTUSI2SDZrr_Int:
7181 case X86::VCVTUSI2SDZrm_Int:
7182 case X86::VCVTUSI642SDZrr:
7183 case X86::VCVTUSI642SDZrm:
7184 case X86::VCVTUSI642SDZrr_Int:
7185 case X86::VCVTUSI642SDZrrb_Int:
7186 case X86::VCVTUSI642SDZrm_Int:
7187 case X86::VCVTSI2SHZrr:
7188 case X86::VCVTSI2SHZrm:
7189 case X86::VCVTSI2SHZrr_Int:
7190 case X86::VCVTSI2SHZrrb_Int:
7191 case X86::VCVTSI2SHZrm_Int:
7192 case X86::VCVTSI642SHZrr:
7193 case X86::VCVTSI642SHZrm:
7194 case X86::VCVTSI642SHZrr_Int:
7195 case X86::VCVTSI642SHZrrb_Int:
7196 case X86::VCVTSI642SHZrm_Int:
7197 case X86::VCVTUSI2SHZrr:
7198 case X86::VCVTUSI2SHZrm:
7199 case X86::VCVTUSI2SHZrr_Int:
7200 case X86::VCVTUSI2SHZrrb_Int:
7201 case X86::VCVTUSI2SHZrm_Int:
7202 case X86::VCVTUSI642SHZrr:
7203 case X86::VCVTUSI642SHZrm:
7204 case X86::VCVTUSI642SHZrr_Int:
7205 case X86::VCVTUSI642SHZrrb_Int:
7206 case X86::VCVTUSI642SHZrm_Int:
7207 // Load folding won't effect the undef register update since the input is
7208 // a GPR.
7209 return OpNum == 1 && !ForLoadFold;
7210 case X86::VCVTSD2SSrr:
7211 case X86::VCVTSD2SSrm:
7212 case X86::VCVTSD2SSrr_Int:
7213 case X86::VCVTSD2SSrm_Int:
7214 case X86::VCVTSS2SDrr:
7215 case X86::VCVTSS2SDrm:
7216 case X86::VCVTSS2SDrr_Int:
7217 case X86::VCVTSS2SDrm_Int:
7218 case X86::VRCPSSr:
7219 case X86::VRCPSSr_Int:
7220 case X86::VRCPSSm:
7221 case X86::VRCPSSm_Int:
7222 case X86::VROUNDSDri:
7223 case X86::VROUNDSDmi:
7224 case X86::VROUNDSDri_Int:
7225 case X86::VROUNDSDmi_Int:
7226 case X86::VROUNDSSri:
7227 case X86::VROUNDSSmi:
7228 case X86::VROUNDSSri_Int:
7229 case X86::VROUNDSSmi_Int:
7230 case X86::VRSQRTSSr:
7231 case X86::VRSQRTSSr_Int:
7232 case X86::VRSQRTSSm:
7233 case X86::VRSQRTSSm_Int:
7234 case X86::VSQRTSSr:
7235 case X86::VSQRTSSr_Int:
7236 case X86::VSQRTSSm:
7237 case X86::VSQRTSSm_Int:
7238 case X86::VSQRTSDr:
7239 case X86::VSQRTSDr_Int:
7240 case X86::VSQRTSDm:
7241 case X86::VSQRTSDm_Int:
7242 // AVX-512
7243 case X86::VCVTSD2SSZrr:
7244 case X86::VCVTSD2SSZrr_Int:
7245 case X86::VCVTSD2SSZrrb_Int:
7246 case X86::VCVTSD2SSZrm:
7247 case X86::VCVTSD2SSZrm_Int:
7248 case X86::VCVTSS2SDZrr:
7249 case X86::VCVTSS2SDZrr_Int:
7250 case X86::VCVTSS2SDZrrb_Int:
7251 case X86::VCVTSS2SDZrm:
7252 case X86::VCVTSS2SDZrm_Int:
7253 case X86::VGETEXPSDZr:
7254 case X86::VGETEXPSDZrb:
7255 case X86::VGETEXPSDZm:
7256 case X86::VGETEXPSSZr:
7257 case X86::VGETEXPSSZrb:
7258 case X86::VGETEXPSSZm:
7259 case X86::VGETMANTSDZrri:
7260 case X86::VGETMANTSDZrrib:
7261 case X86::VGETMANTSDZrmi:
7262 case X86::VGETMANTSSZrri:
7263 case X86::VGETMANTSSZrrib:
7264 case X86::VGETMANTSSZrmi:
7265 case X86::VRNDSCALESDZrri:
7266 case X86::VRNDSCALESDZrri_Int:
7267 case X86::VRNDSCALESDZrrib_Int:
7268 case X86::VRNDSCALESDZrmi:
7269 case X86::VRNDSCALESDZrmi_Int:
7270 case X86::VRNDSCALESSZrri:
7271 case X86::VRNDSCALESSZrri_Int:
7272 case X86::VRNDSCALESSZrrib_Int:
7273 case X86::VRNDSCALESSZrmi:
7274 case X86::VRNDSCALESSZrmi_Int:
7275 case X86::VRCP14SDZrr:
7276 case X86::VRCP14SDZrm:
7277 case X86::VRCP14SSZrr:
7278 case X86::VRCP14SSZrm:
7279 case X86::VRCPSHZrr:
7280 case X86::VRCPSHZrm:
7281 case X86::VRSQRTSHZrr:
7282 case X86::VRSQRTSHZrm:
7283 case X86::VREDUCESHZrmi:
7284 case X86::VREDUCESHZrri:
7285 case X86::VREDUCESHZrrib:
7286 case X86::VGETEXPSHZr:
7287 case X86::VGETEXPSHZrb:
7288 case X86::VGETEXPSHZm:
7289 case X86::VGETMANTSHZrri:
7290 case X86::VGETMANTSHZrrib:
7291 case X86::VGETMANTSHZrmi:
7292 case X86::VRNDSCALESHZrri:
7293 case X86::VRNDSCALESHZrri_Int:
7294 case X86::VRNDSCALESHZrrib_Int:
7295 case X86::VRNDSCALESHZrmi:
7296 case X86::VRNDSCALESHZrmi_Int:
7297 case X86::VSQRTSHZr:
7298 case X86::VSQRTSHZr_Int:
7299 case X86::VSQRTSHZrb_Int:
7300 case X86::VSQRTSHZm:
7301 case X86::VSQRTSHZm_Int:
7302 case X86::VRCP28SDZr:
7303 case X86::VRCP28SDZrb:
7304 case X86::VRCP28SDZm:
7305 case X86::VRCP28SSZr:
7306 case X86::VRCP28SSZrb:
7307 case X86::VRCP28SSZm:
7308 case X86::VREDUCESSZrmi:
7309 case X86::VREDUCESSZrri:
7310 case X86::VREDUCESSZrrib:
7311 case X86::VRSQRT14SDZrr:
7312 case X86::VRSQRT14SDZrm:
7313 case X86::VRSQRT14SSZrr:
7314 case X86::VRSQRT14SSZrm:
7315 case X86::VRSQRT28SDZr:
7316 case X86::VRSQRT28SDZrb:
7317 case X86::VRSQRT28SDZm:
7318 case X86::VRSQRT28SSZr:
7319 case X86::VRSQRT28SSZrb:
7320 case X86::VRSQRT28SSZm:
7321 case X86::VSQRTSSZr:
7322 case X86::VSQRTSSZr_Int:
7323 case X86::VSQRTSSZrb_Int:
7324 case X86::VSQRTSSZm:
7325 case X86::VSQRTSSZm_Int:
7326 case X86::VSQRTSDZr:
7327 case X86::VSQRTSDZr_Int:
7328 case X86::VSQRTSDZrb_Int:
7329 case X86::VSQRTSDZm:
7330 case X86::VSQRTSDZm_Int:
7331 case X86::VCVTSD2SHZrr:
7332 case X86::VCVTSD2SHZrr_Int:
7333 case X86::VCVTSD2SHZrrb_Int:
7334 case X86::VCVTSD2SHZrm:
7335 case X86::VCVTSD2SHZrm_Int:
7336 case X86::VCVTSS2SHZrr:
7337 case X86::VCVTSS2SHZrr_Int:
7338 case X86::VCVTSS2SHZrrb_Int:
7339 case X86::VCVTSS2SHZrm:
7340 case X86::VCVTSS2SHZrm_Int:
7341 case X86::VCVTSH2SDZrr:
7342 case X86::VCVTSH2SDZrr_Int:
7343 case X86::VCVTSH2SDZrrb_Int:
7344 case X86::VCVTSH2SDZrm:
7345 case X86::VCVTSH2SDZrm_Int:
7346 case X86::VCVTSH2SSZrr:
7347 case X86::VCVTSH2SSZrr_Int:
7348 case X86::VCVTSH2SSZrrb_Int:
7349 case X86::VCVTSH2SSZrm:
7350 case X86::VCVTSH2SSZrm_Int:
7351 return OpNum == 1;
7352 case X86::VMOVSSZrrk:
7353 case X86::VMOVSDZrrk:
7354 return OpNum == 3 && !ForLoadFold;
7355 case X86::VMOVSSZrrkz:
7356 case X86::VMOVSDZrrkz:
7357 return OpNum == 2 && !ForLoadFold;
7358 }
7359
7360 return false;
7361}
7362
7363/// Inform the BreakFalseDeps pass how many idle instructions we would like
7364/// before certain undef register reads.
7365///
7366/// This catches the VCVTSI2SD family of instructions:
7367///
7368/// vcvtsi2sdq %rax, undef %xmm0, %xmm14
7369///
7370/// We should to be careful *not* to catch VXOR idioms which are presumably
7371/// handled specially in the pipeline:
7372///
7373/// vxorps undef %xmm1, undef %xmm1, %xmm1
7374///
7375/// Like getPartialRegUpdateClearance, this makes a strong assumption that the
7376/// high bits that are passed-through are not live.
7377unsigned
7378X86InstrInfo::getUndefRegClearance(const MachineInstr &MI, unsigned OpNum,
7379 const TargetRegisterInfo *TRI) const {
7380 const MachineOperand &MO = MI.getOperand(i: OpNum);
7381 if (MO.getReg().isPhysical() && hasUndefRegUpdate(Opcode: MI.getOpcode(), OpNum))
7382 return UndefRegClearance;
7383
7384 return 0;
7385}
7386
7387void X86InstrInfo::breakPartialRegDependency(
7388 MachineInstr &MI, unsigned OpNum, const TargetRegisterInfo *TRI) const {
7389 Register Reg = MI.getOperand(i: OpNum).getReg();
7390 // If MI kills this register, the false dependence is already broken.
7391 if (MI.killsRegister(Reg, TRI))
7392 return;
7393
7394 if (X86::VR128RegClass.contains(Reg)) {
7395 // These instructions are all floating point domain, so xorps is the best
7396 // choice.
7397 unsigned Opc = Subtarget.hasAVX() ? X86::VXORPSrr : X86::XORPSrr;
7398 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Opc), DestReg: Reg)
7399 .addReg(RegNo: Reg, Flags: RegState::Undef)
7400 .addReg(RegNo: Reg, Flags: RegState::Undef);
7401 MI.addRegisterKilled(IncomingReg: Reg, RegInfo: TRI, AddIfNotFound: true);
7402 } else if (X86::VR256RegClass.contains(Reg)) {
7403 // Use vxorps to clear the full ymm register.
7404 // It wants to read and write the xmm sub-register.
7405 Register XReg = TRI->getSubReg(Reg, Idx: X86::sub_xmm);
7406 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: X86::VXORPSrr), DestReg: XReg)
7407 .addReg(RegNo: XReg, Flags: RegState::Undef)
7408 .addReg(RegNo: XReg, Flags: RegState::Undef)
7409 .addReg(RegNo: Reg, Flags: RegState::ImplicitDefine);
7410 MI.addRegisterKilled(IncomingReg: Reg, RegInfo: TRI, AddIfNotFound: true);
7411 } else if (X86::VR128XRegClass.contains(Reg)) {
7412 // Only handle VLX targets.
7413 if (!Subtarget.hasVLX())
7414 return;
7415 // Since vxorps requires AVX512DQ, vpxord should be the best choice.
7416 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: X86::VPXORDZ128rr), DestReg: Reg)
7417 .addReg(RegNo: Reg, Flags: RegState::Undef)
7418 .addReg(RegNo: Reg, Flags: RegState::Undef);
7419 MI.addRegisterKilled(IncomingReg: Reg, RegInfo: TRI, AddIfNotFound: true);
7420 } else if (X86::VR256XRegClass.contains(Reg) ||
7421 X86::VR512RegClass.contains(Reg)) {
7422 // Only handle VLX targets.
7423 if (!Subtarget.hasVLX())
7424 return;
7425 // Use vpxord to clear the full ymm/zmm register.
7426 // It wants to read and write the xmm sub-register.
7427 Register XReg = TRI->getSubReg(Reg, Idx: X86::sub_xmm);
7428 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: X86::VPXORDZ128rr), DestReg: XReg)
7429 .addReg(RegNo: XReg, Flags: RegState::Undef)
7430 .addReg(RegNo: XReg, Flags: RegState::Undef)
7431 .addReg(RegNo: Reg, Flags: RegState::ImplicitDefine);
7432 MI.addRegisterKilled(IncomingReg: Reg, RegInfo: TRI, AddIfNotFound: true);
7433 } else if (X86::GR64RegClass.contains(Reg)) {
7434 // Using XOR32rr because it has shorter encoding and zeros up the upper bits
7435 // as well.
7436 Register XReg = TRI->getSubReg(Reg, Idx: X86::sub_32bit);
7437 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: X86::XOR32rr), DestReg: XReg)
7438 .addReg(RegNo: XReg, Flags: RegState::Undef)
7439 .addReg(RegNo: XReg, Flags: RegState::Undef)
7440 .addReg(RegNo: Reg, Flags: RegState::ImplicitDefine);
7441 MI.addRegisterKilled(IncomingReg: Reg, RegInfo: TRI, AddIfNotFound: true);
7442 } else if (X86::GR32RegClass.contains(Reg)) {
7443 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: X86::XOR32rr), DestReg: Reg)
7444 .addReg(RegNo: Reg, Flags: RegState::Undef)
7445 .addReg(RegNo: Reg, Flags: RegState::Undef);
7446 MI.addRegisterKilled(IncomingReg: Reg, RegInfo: TRI, AddIfNotFound: true);
7447 } else if ((X86::GR16RegClass.contains(Reg) ||
7448 X86::GR8RegClass.contains(Reg)) &&
7449 X86II::hasNewDataDest(TSFlags: MI.getDesc().TSFlags)) {
7450 // This case is only expected for NDD ops which appear to be partial
7451 // writes, but are not due to the zeroing of the upper part. Here
7452 // we add an implicit def of the superegister, which prevents
7453 // CompressEVEX from converting this to a legacy form.
7454 Register SuperReg = getX86SubSuperRegister(Reg, Size: 64);
7455 MachineInstrBuilder BuildMI(*MI.getParent()->getParent(), &MI);
7456 if (!MI.definesRegister(Reg: SuperReg, /*TRI=*/nullptr))
7457 BuildMI.addReg(RegNo: SuperReg, Flags: RegState::ImplicitDefine);
7458 }
7459}
7460
7461static void addOperands(MachineInstrBuilder &MIB, ArrayRef<MachineOperand> MOs,
7462 int PtrOffset = 0) {
7463 unsigned NumAddrOps = MOs.size();
7464
7465 if (NumAddrOps < 4) {
7466 // FrameIndex only - add an immediate offset (whether its zero or not).
7467 for (unsigned i = 0; i != NumAddrOps; ++i)
7468 MIB.add(MO: MOs[i]);
7469 addOffset(MIB, Offset: PtrOffset);
7470 } else {
7471 // General Memory Addressing - we need to add any offset to an existing
7472 // offset.
7473 assert(MOs.size() == 5 && "Unexpected memory operand list length");
7474 for (unsigned i = 0; i != NumAddrOps; ++i) {
7475 const MachineOperand &MO = MOs[i];
7476 if (i == 3 && PtrOffset != 0) {
7477 MIB.addDisp(Disp: MO, off: PtrOffset);
7478 } else {
7479 MIB.add(MO);
7480 }
7481 }
7482 }
7483}
7484
7485static void updateOperandRegConstraints(MachineFunction &MF,
7486 MachineInstr &NewMI,
7487 const TargetInstrInfo &TII) {
7488 MachineRegisterInfo &MRI = MF.getRegInfo();
7489
7490 for (int Idx : llvm::seq<int>(Begin: 0, End: NewMI.getNumOperands())) {
7491 MachineOperand &MO = NewMI.getOperand(i: Idx);
7492 // We only need to update constraints on virtual register operands.
7493 if (!MO.isReg())
7494 continue;
7495 Register Reg = MO.getReg();
7496 if (!Reg.isVirtual())
7497 continue;
7498
7499 auto *NewRC =
7500 MRI.constrainRegClass(Reg, RC: TII.getRegClass(MCID: NewMI.getDesc(), OpNum: Idx));
7501 if (!NewRC) {
7502 LLVM_DEBUG(
7503 dbgs() << "WARNING: Unable to update register constraint for operand "
7504 << Idx << " of instruction:\n";
7505 NewMI.dump(); dbgs() << "\n");
7506 }
7507 }
7508}
7509
7510static MachineInstr *fuseTwoAddrInst(MachineFunction &MF, unsigned Opcode,
7511 ArrayRef<MachineOperand> MOs,
7512 MachineBasicBlock::iterator InsertPt,
7513 MachineInstr &MI,
7514 const TargetInstrInfo &TII) {
7515 // Create the base instruction with the memory operand as the first part.
7516 // Omit the implicit operands, something BuildMI can't do.
7517 MachineInstr *NewMI =
7518 MF.CreateMachineInstr(MCID: TII.get(Opcode), DL: MI.getDebugLoc(), NoImplicit: true);
7519 MachineInstrBuilder MIB(MF, NewMI);
7520 addOperands(MIB, MOs);
7521
7522 // Loop over the rest of the ri operands, converting them over.
7523 unsigned NumOps = MI.getDesc().getNumOperands() - 2;
7524 for (unsigned i = 0; i != NumOps; ++i) {
7525 MachineOperand &MO = MI.getOperand(i: i + 2);
7526 MIB.add(MO);
7527 }
7528 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.operands(), N: NumOps + 2))
7529 MIB.add(MO);
7530
7531 updateOperandRegConstraints(MF, NewMI&: *NewMI, TII);
7532
7533 MachineBasicBlock *MBB = InsertPt->getParent();
7534 MBB->insert(I: InsertPt, MI: NewMI);
7535
7536 return MIB;
7537}
7538
7539static MachineInstr *fuseInst(MachineFunction &MF, unsigned Opcode,
7540 unsigned OpNo, ArrayRef<MachineOperand> MOs,
7541 MachineBasicBlock::iterator InsertPt,
7542 MachineInstr &MI, const TargetInstrInfo &TII,
7543 int PtrOffset = 0) {
7544 // Omit the implicit operands, something BuildMI can't do.
7545 MachineInstr *NewMI =
7546 MF.CreateMachineInstr(MCID: TII.get(Opcode), DL: MI.getDebugLoc(), NoImplicit: true);
7547 MachineInstrBuilder MIB(MF, NewMI);
7548
7549 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
7550 MachineOperand &MO = MI.getOperand(i);
7551 if (i == OpNo) {
7552 assert(MO.isReg() && "Expected to fold into reg operand!");
7553 addOperands(MIB, MOs, PtrOffset);
7554 } else {
7555 MIB.add(MO);
7556 }
7557 }
7558
7559 updateOperandRegConstraints(MF, NewMI&: *NewMI, TII);
7560
7561 // Copy the NoFPExcept flag from the instruction we're fusing.
7562 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
7563 NewMI->setFlag(MachineInstr::MIFlag::NoFPExcept);
7564
7565 MachineBasicBlock *MBB = InsertPt->getParent();
7566 MBB->insert(I: InsertPt, MI: NewMI);
7567
7568 return MIB;
7569}
7570
7571static MachineInstr *makeM0Inst(const TargetInstrInfo &TII, unsigned Opcode,
7572 ArrayRef<MachineOperand> MOs,
7573 MachineBasicBlock::iterator InsertPt,
7574 MachineInstr &MI) {
7575 MachineInstrBuilder MIB = BuildMI(BB&: *InsertPt->getParent(), I: InsertPt,
7576 MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode));
7577 addOperands(MIB, MOs);
7578 return MIB.addImm(Val: 0);
7579}
7580
7581MachineInstr *X86InstrInfo::foldMemoryOperandCustom(
7582 MachineFunction &MF, MachineInstr &MI, unsigned OpNum,
7583 ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt,
7584 unsigned Size, Align Alignment) const {
7585 switch (MI.getOpcode()) {
7586 case X86::INSERTPSrri:
7587 case X86::VINSERTPSrri:
7588 case X86::VINSERTPSZrri:
7589 // Attempt to convert the load of inserted vector into a fold load
7590 // of a single float.
7591 if (OpNum == 2) {
7592 unsigned Imm = MI.getOperand(i: MI.getNumOperands() - 1).getImm();
7593 unsigned ZMask = Imm & 15;
7594 unsigned DstIdx = (Imm >> 4) & 3;
7595 unsigned SrcIdx = (Imm >> 6) & 3;
7596
7597 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7598 const TargetRegisterClass *RC = getRegClass(MCID: MI.getDesc(), OpNum);
7599 unsigned RCSize = TRI.getRegSizeInBits(RC: *RC) / 8;
7600 if ((Size == 0 || Size >= 16) && RCSize >= 16 &&
7601 (MI.getOpcode() != X86::INSERTPSrri || Alignment >= Align(4))) {
7602 int PtrOffset = SrcIdx * 4;
7603 unsigned NewImm = (DstIdx << 4) | ZMask;
7604 unsigned NewOpCode =
7605 (MI.getOpcode() == X86::VINSERTPSZrri) ? X86::VINSERTPSZrmi
7606 : (MI.getOpcode() == X86::VINSERTPSrri) ? X86::VINSERTPSrmi
7607 : X86::INSERTPSrmi;
7608 MachineInstr *NewMI =
7609 fuseInst(MF, Opcode: NewOpCode, OpNo: OpNum, MOs, InsertPt, MI, TII: *this, PtrOffset);
7610 NewMI->getOperand(i: NewMI->getNumOperands() - 1).setImm(NewImm);
7611 return NewMI;
7612 }
7613 }
7614 break;
7615 case X86::MOVHLPSrr:
7616 case X86::VMOVHLPSrr:
7617 case X86::VMOVHLPSZrr:
7618 // Move the upper 64-bits of the second operand to the lower 64-bits.
7619 // To fold the load, adjust the pointer to the upper and use (V)MOVLPS.
7620 // TODO: In most cases AVX doesn't have a 8-byte alignment requirement.
7621 if (OpNum == 2) {
7622 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7623 const TargetRegisterClass *RC = getRegClass(MCID: MI.getDesc(), OpNum);
7624 unsigned RCSize = TRI.getRegSizeInBits(RC: *RC) / 8;
7625 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment >= Align(8)) {
7626 unsigned NewOpCode =
7627 (MI.getOpcode() == X86::VMOVHLPSZrr) ? X86::VMOVLPSZ128rm
7628 : (MI.getOpcode() == X86::VMOVHLPSrr) ? X86::VMOVLPSrm
7629 : X86::MOVLPSrm;
7630 MachineInstr *NewMI =
7631 fuseInst(MF, Opcode: NewOpCode, OpNo: OpNum, MOs, InsertPt, MI, TII: *this, PtrOffset: 8);
7632 return NewMI;
7633 }
7634 }
7635 break;
7636 case X86::UNPCKLPDrr:
7637 // If we won't be able to fold this to the memory form of UNPCKL, use
7638 // MOVHPD instead. Done as custom because we can't have this in the load
7639 // table twice.
7640 if (OpNum == 2) {
7641 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7642 const TargetRegisterClass *RC = getRegClass(MCID: MI.getDesc(), OpNum);
7643 unsigned RCSize = TRI.getRegSizeInBits(RC: *RC) / 8;
7644 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment < Align(16)) {
7645 MachineInstr *NewMI =
7646 fuseInst(MF, Opcode: X86::MOVHPDrm, OpNo: OpNum, MOs, InsertPt, MI, TII: *this);
7647 return NewMI;
7648 }
7649 }
7650 break;
7651 case X86::MOV32r0:
7652 if (auto *NewMI =
7653 makeM0Inst(TII: *this, Opcode: (Size == 4) ? X86::MOV32mi : X86::MOV64mi32, MOs,
7654 InsertPt, MI))
7655 return NewMI;
7656 break;
7657 }
7658
7659 return nullptr;
7660}
7661
7662static bool shouldPreventUndefRegUpdateMemFold(MachineFunction &MF,
7663 MachineInstr &MI) {
7664 if (!hasUndefRegUpdate(Opcode: MI.getOpcode(), OpNum: 1, /*ForLoadFold*/ true) ||
7665 !MI.getOperand(i: 1).isReg())
7666 return false;
7667
7668 // The are two cases we need to handle depending on where in the pipeline
7669 // the folding attempt is being made.
7670 // -Register has the undef flag set.
7671 // -Register is produced by the IMPLICIT_DEF instruction.
7672
7673 if (MI.getOperand(i: 1).isUndef())
7674 return true;
7675
7676 MachineRegisterInfo &RegInfo = MF.getRegInfo();
7677 MachineInstr *VRegDef = RegInfo.getUniqueVRegDef(Reg: MI.getOperand(i: 1).getReg());
7678 return VRegDef && VRegDef->isImplicitDef();
7679}
7680
7681unsigned X86InstrInfo::commuteOperandsForFold(MachineInstr &MI,
7682 unsigned Idx1) const {
7683 unsigned Idx2 = CommuteAnyOperandIndex;
7684 if (!findCommutedOpIndices(MI, SrcOpIdx1&: Idx1, SrcOpIdx2&: Idx2))
7685 return Idx1;
7686
7687 bool HasDef = MI.getDesc().getNumDefs();
7688 Register Reg0 = HasDef ? MI.getOperand(i: 0).getReg() : Register();
7689 Register Reg1 = MI.getOperand(i: Idx1).getReg();
7690 Register Reg2 = MI.getOperand(i: Idx2).getReg();
7691 bool Tied1 = 0 == MI.getDesc().getOperandConstraint(OpNum: Idx1, Constraint: MCOI::TIED_TO);
7692 bool Tied2 = 0 == MI.getDesc().getOperandConstraint(OpNum: Idx2, Constraint: MCOI::TIED_TO);
7693
7694 // If either of the commutable operands are tied to the destination
7695 // then we can not commute + fold.
7696 if ((HasDef && Reg0 == Reg1 && Tied1) || (HasDef && Reg0 == Reg2 && Tied2))
7697 return Idx1;
7698
7699 return commuteInstruction(MI, NewMI: false, OpIdx1: Idx1, OpIdx2: Idx2) ? Idx2 : Idx1;
7700}
7701
7702static void printFailMsgforFold(const MachineInstr &MI, unsigned Idx) {
7703 if (PrintFailedFusing && !MI.isCopy())
7704 dbgs() << "We failed to fuse operand " << Idx << " in " << MI;
7705}
7706
7707MachineInstr *X86InstrInfo::foldMemoryOperandImpl(
7708 MachineFunction &MF, MachineInstr &MI, unsigned OpNum,
7709 ArrayRef<MachineOperand> MOs, MachineBasicBlock::iterator InsertPt,
7710 unsigned Size, Align Alignment, bool AllowCommute, MachineInstr *&CopyMI,
7711 VirtRegMap *VRM) const {
7712 bool isSlowTwoMemOps = Subtarget.slowTwoMemOps();
7713 bool isSlowIndirectCall = Subtarget.slowIndirectCall();
7714 unsigned Opc = MI.getOpcode();
7715
7716 // For CPUs that favor the register form of a call,
7717 // do not fold loads into calls, unless optimizing for size aggressively.
7718 if ((isSlowTwoMemOps || isSlowIndirectCall) &&
7719 !MF.getFunction().hasMinSize() &&
7720 (Opc == X86::CALL32r || Opc == X86::CALL64r ||
7721 Opc == X86::CALL64r_ImpCall))
7722 return nullptr;
7723
7724 // For CPUs that favor the register form of a push,
7725 // do not fold loads into pushes, unless optimizing for size aggressively.
7726 if (isSlowTwoMemOps && !MF.getFunction().hasMinSize() &&
7727 (Opc == X86::PUSH16r || Opc == X86::PUSH32r || Opc == X86::PUSH64r))
7728 return nullptr;
7729
7730 // Avoid partial and undef register update stalls unless optimizing for size.
7731 if (!MF.getFunction().hasOptSize() &&
7732 (hasPartialRegUpdate(Opcode: Opc, Subtarget, /*ForLoadFold*/ true) ||
7733 shouldPreventUndefRegUpdateMemFold(MF, MI)))
7734 return nullptr;
7735
7736 unsigned NumOps = MI.getDesc().getNumOperands();
7737 bool IsTwoAddr = NumOps > 1 && OpNum < 2 && MI.getOperand(i: 0).isReg() &&
7738 MI.getOperand(i: 1).isReg() &&
7739 MI.getOperand(i: 0).getReg() == MI.getOperand(i: 1).getReg();
7740
7741 // FIXME: AsmPrinter doesn't know how to handle
7742 // X86II::MO_GOT_ABSOLUTE_ADDRESS after folding.
7743 if (Opc == X86::ADD32ri &&
7744 MI.getOperand(i: 2).getTargetFlags() == X86II::MO_GOT_ABSOLUTE_ADDRESS)
7745 return nullptr;
7746
7747 // GOTTPOFF relocation loads can only be folded into add instructions.
7748 // FIXME: Need to exclude other relocations that only support specific
7749 // instructions.
7750 if (MOs.size() == X86::AddrNumOperands &&
7751 MOs[X86::AddrDisp].getTargetFlags() == X86II::MO_GOTTPOFF &&
7752 Opc != X86::ADD64rr)
7753 return nullptr;
7754
7755 // Don't fold loads into indirect calls that need a KCFI check as we'll
7756 // have to unfold these in X86TargetLowering::EmitKCFICheck anyway.
7757 if (MI.isCall() && MI.getCFIType())
7758 return nullptr;
7759
7760 // Attempt to fold any custom cases we have.
7761 if (auto *CustomMI = foldMemoryOperandCustom(MF, MI, OpNum, MOs, InsertPt,
7762 Size, Alignment))
7763 return CustomMI;
7764
7765 // Folding a memory location into the two-address part of a two-address
7766 // instruction is different than folding it other places. It requires
7767 // replacing the *two* registers with the memory location.
7768 //
7769 // Utilize the mapping NonNDD -> RMW for the NDD variant.
7770 unsigned NonNDOpc = Subtarget.hasNDD() ? X86::getNonNDVariant(Opc) : 0U;
7771 // Utilize the mapping NonNDD if NDD memory variant is not preferred.
7772 bool NoNDDM = NonNDOpc && !Subtarget.hasNDDM();
7773
7774 MachineRegisterInfo &MRI = MF.getRegInfo();
7775 if (NoNDDM && !IsTwoAddr && !MRI.isSSA()) {
7776 // Bail out if dst has subreg. It happens during register-coalescer from
7777 // 704B %19:gr32 = SUB32rr_ND killed %0:gr32, killed %7:gr32, ...
7778 // 752B undef %23.sub_32bit:gr64 = COPY killed %19:gr32
7779 // 768B %25:gr32 = LEA64_32r killed %23:gr64, 1, killed %21:gr64_nosp, ...
7780 // to
7781 // 704B undef %23.sub_32bit:gr64_with_sub_8bit = SUB32rr_ND %0:gr32, ...
7782 // 768B %25:gr32 = LEA64_32r %23:gr64_with_sub_8bit, 1, %21:gr64_nosp, ...
7783 // Machine verifier fails if we try to tie %23 to the source.
7784 if (MI.getOperand(i: 0).getSubReg())
7785 return nullptr;
7786
7787 // Bail out if dst has been assigned a physical register. Otherwise, we
7788 // cannot update LiveRegMatrix properly.
7789 Register Dst = MI.getOperand(i: 0).getReg();
7790 if (VRM && Dst != MI.getOperand(i: 1).getReg() &&
7791 (!Dst.isVirtual() || VRM->getPhys(virtReg: Dst)))
7792 return nullptr;
7793 }
7794
7795 const X86FoldTableEntry *I =
7796 IsTwoAddr ? lookupTwoAddrFoldTable(RegOp: NonNDOpc ? NonNDOpc : Opc)
7797 : lookupFoldTable(RegOp: NoNDDM ? NonNDOpc : Opc, OpNum);
7798
7799 MachineInstr *NewMI = nullptr;
7800 if (I) {
7801 unsigned Opcode = I->DstOp;
7802 if (Alignment <
7803 Align(1ULL << ((I->Flags & TB_ALIGN_MASK) >> TB_ALIGN_SHIFT)))
7804 return nullptr;
7805 bool NarrowToMOV32rm = false;
7806 if (Size) {
7807 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7808 const TargetRegisterClass *RC = getRegClass(MCID: MI.getDesc(), OpNum);
7809 unsigned RCSize = TRI.getRegSizeInBits(RC: *RC) / 8;
7810 // Check if it's safe to fold the load. If the size of the object is
7811 // narrower than the load width, then it's not.
7812 // FIXME: Allow scalar intrinsic instructions like ADDSSrm_Int.
7813 if ((I->Flags & TB_FOLDED_LOAD) && Size < RCSize) {
7814 // If this is a 64-bit load, but the spill slot is 32, then we can do
7815 // a 32-bit load which is implicitly zero-extended. This likely is
7816 // due to live interval analysis remat'ing a load from stack slot.
7817 if (Opcode != X86::MOV64rm || RCSize != 8 || Size != 4)
7818 return nullptr;
7819 if (MI.getOperand(i: 0).getSubReg() || MI.getOperand(i: 1).getSubReg())
7820 return nullptr;
7821 Opcode = X86::MOV32rm;
7822 NarrowToMOV32rm = true;
7823 }
7824 // For stores, make sure the size of the object is equal to the size of
7825 // the store. If the object is larger, the extra bits would be garbage. If
7826 // the object is smaller we might overwrite another object or fault.
7827 if ((I->Flags & TB_FOLDED_STORE) && Size != RCSize)
7828 return nullptr;
7829 }
7830
7831 NewMI = IsTwoAddr ? fuseTwoAddrInst(MF, Opcode, MOs, InsertPt, MI, TII: *this)
7832 : fuseInst(MF, Opcode, OpNo: OpNum, MOs, InsertPt, MI, TII: *this);
7833
7834 if (NarrowToMOV32rm) {
7835 // If this is the special case where we use a MOV32rm to load a 32-bit
7836 // value and zero-extend the top bits. Change the destination register
7837 // to a 32-bit one.
7838 Register DstReg = NewMI->getOperand(i: 0).getReg();
7839 if (DstReg.isPhysical())
7840 NewMI->getOperand(i: 0).setReg(RI.getSubReg(Reg: DstReg, Idx: X86::sub_32bit));
7841 else
7842 NewMI->getOperand(i: 0).setSubReg(X86::sub_32bit);
7843 }
7844
7845 if (NoNDDM && !IsTwoAddr) {
7846 Register SrcReg = MI.getOperand(i: 1).getReg();
7847 unsigned SrcSub = MI.getOperand(i: 1).getSubReg();
7848 if (MI.killsRegister(Reg: SrcReg, /*TRI=*/nullptr) ||
7849 MI.getOperand(i: 0).getReg() == SrcReg)
7850 return NewMI;
7851
7852 Register NewSrc = MI.getOperand(i: 0).getReg();
7853 if (MRI.isSSA())
7854 NewSrc = MRI.createVirtualRegister(RegClass: getRegClass(MCID: NewMI->getDesc(), OpNum: 1));
7855
7856 CopyMI = BuildMI(BB&: *NewMI->getParent(), I&: *NewMI, MIMD: MI.getDebugLoc(),
7857 MCID: get(Opcode: TargetOpcode::COPY))
7858 .addDef(RegNo: NewSrc)
7859 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SrcSub);
7860 NewMI->getOperand(i: 1).setReg(NewSrc);
7861 NewMI->getOperand(i: 1).setSubReg(0);
7862 }
7863 return NewMI;
7864 }
7865
7866 if (AllowCommute) {
7867 // If the instruction and target operand are commutable, commute the
7868 // instruction and try again.
7869 unsigned CommuteOpIdx2 = commuteOperandsForFold(MI, Idx1: OpNum);
7870 if (CommuteOpIdx2 == OpNum) {
7871 printFailMsgforFold(MI, Idx: OpNum);
7872 return nullptr;
7873 }
7874 // Attempt to fold with the commuted version of the instruction.
7875 NewMI = foldMemoryOperandImpl(MF, MI, OpNum: CommuteOpIdx2, MOs, InsertPt, Size,
7876 Alignment, /*AllowCommute=*/false, CopyMI);
7877 if (NewMI)
7878 return NewMI;
7879 // Folding failed again - undo the commute before returning.
7880 commuteInstruction(MI, NewMI: false, OpIdx1: OpNum, OpIdx2: CommuteOpIdx2);
7881 }
7882
7883 printFailMsgforFold(MI, Idx: OpNum);
7884 return nullptr;
7885}
7886
7887MachineInstr *
7888X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
7889 ArrayRef<unsigned> Ops, int FrameIndex,
7890 MachineInstr *&CopyMI, LiveIntervals *LIS,
7891 VirtRegMap *VRM) const {
7892 MachineBasicBlock::iterator InsertPt = MI;
7893 // Check switch flag
7894 if (NoFusing)
7895 return nullptr;
7896
7897 // Avoid partial and undef register update stalls unless optimizing for size.
7898 if (!MF.getFunction().hasOptSize() &&
7899 (hasPartialRegUpdate(Opcode: MI.getOpcode(), Subtarget, /*ForLoadFold*/ true) ||
7900 shouldPreventUndefRegUpdateMemFold(MF, MI)))
7901 return nullptr;
7902
7903 // Don't fold subreg spills, or reloads that use a high subreg.
7904 for (auto Op : Ops) {
7905 MachineOperand &MO = MI.getOperand(i: Op);
7906 auto SubReg = MO.getSubReg();
7907 // MOV32r0 is special b/c it's used to clear a 64-bit register too.
7908 // (See patterns for MOV32r0 in TD files).
7909 if (MI.getOpcode() == X86::MOV32r0 && SubReg == X86::sub_32bit)
7910 continue;
7911 if (SubReg && (MO.isDef() || SubReg == X86::sub_8bit_hi))
7912 return nullptr;
7913 }
7914
7915 const MachineFrameInfo &MFI = MF.getFrameInfo();
7916 unsigned Size = MFI.getObjectSize(ObjectIdx: FrameIndex);
7917 Align Alignment = MFI.getObjectAlign(ObjectIdx: FrameIndex);
7918 // If the function stack isn't realigned we don't want to fold instructions
7919 // that need increased alignment.
7920 if (!RI.hasStackRealignment(MF))
7921 Alignment =
7922 std::min(a: Alignment, b: Subtarget.getFrameLowering()->getStackAlign());
7923
7924 auto Impl = [&]() {
7925 return foldMemoryOperandImpl(
7926 MF, MI, OpNum: Ops[0], MOs: MachineOperand::CreateFI(Idx: FrameIndex), InsertPt, Size,
7927 Alignment, /*AllowCommute=*/true, CopyMI, VRM);
7928 };
7929 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
7930 unsigned NewOpc = 0;
7931 unsigned RCSize = 0;
7932 unsigned Opc = MI.getOpcode();
7933 switch (Opc) {
7934 default:
7935 // NDD can be folded into RMW though its Op0 and Op1 are not tied.
7936 return (Subtarget.hasNDD() ? X86::getNonNDVariant(Opc) : 0U) ? Impl()
7937 : nullptr;
7938 case X86::TEST8rr:
7939 NewOpc = X86::CMP8ri;
7940 RCSize = 1;
7941 break;
7942 case X86::TEST16rr:
7943 NewOpc = X86::CMP16ri;
7944 RCSize = 2;
7945 break;
7946 case X86::TEST32rr:
7947 NewOpc = X86::CMP32ri;
7948 RCSize = 4;
7949 break;
7950 case X86::TEST64rr:
7951 NewOpc = X86::CMP64ri32;
7952 RCSize = 8;
7953 break;
7954 }
7955 // Check if it's safe to fold the load. If the size of the object is
7956 // narrower than the load width, then it's not.
7957 if (Size < RCSize)
7958 return nullptr;
7959 // Change to CMPXXri r, 0 first.
7960 MI.setDesc(get(Opcode: NewOpc));
7961 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: 0);
7962 } else if (Ops.size() != 1)
7963 return nullptr;
7964
7965 return Impl();
7966}
7967
7968/// Check if \p LoadMI is a partial register load that we can't fold into \p MI
7969/// because the latter uses contents that wouldn't be defined in the folded
7970/// version. For instance, this transformation isn't legal:
7971/// movss (%rdi), %xmm0
7972/// addps %xmm0, %xmm0
7973/// ->
7974/// addps (%rdi), %xmm0
7975///
7976/// But this one is:
7977/// movss (%rdi), %xmm0
7978/// addss %xmm0, %xmm0
7979/// ->
7980/// addss (%rdi), %xmm0
7981///
7982static bool isNonFoldablePartialRegisterLoad(const MachineInstr &LoadMI,
7983 const MachineInstr &UserMI,
7984 const MachineFunction &MF) {
7985 unsigned Opc = LoadMI.getOpcode();
7986 unsigned UserOpc = UserMI.getOpcode();
7987 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7988 const TargetRegisterClass *RC =
7989 MF.getRegInfo().getRegClass(Reg: LoadMI.getOperand(i: 0).getReg());
7990 unsigned RegSize = TRI.getRegSizeInBits(RC: *RC);
7991
7992 if ((Opc == X86::MOVSSrm || Opc == X86::VMOVSSrm || Opc == X86::VMOVSSZrm ||
7993 Opc == X86::MOVSSrm_alt || Opc == X86::VMOVSSrm_alt ||
7994 Opc == X86::VMOVSSZrm_alt) &&
7995 RegSize > 32) {
7996 // These instructions only load 32 bits, we can't fold them if the
7997 // destination register is wider than 32 bits (4 bytes), and its user
7998 // instruction isn't scalar (SS).
7999 switch (UserOpc) {
8000 case X86::CVTSS2SDrr_Int:
8001 case X86::VCVTSS2SDrr_Int:
8002 case X86::VCVTSS2SDZrr_Int:
8003 case X86::VCVTSS2SDZrrk_Int:
8004 case X86::VCVTSS2SDZrrkz_Int:
8005 case X86::CVTSS2SIrr_Int:
8006 case X86::CVTSS2SI64rr_Int:
8007 case X86::VCVTSS2SIrr_Int:
8008 case X86::VCVTSS2SI64rr_Int:
8009 case X86::VCVTSS2SIZrr_Int:
8010 case X86::VCVTSS2SI64Zrr_Int:
8011 case X86::CVTTSS2SIrr_Int:
8012 case X86::CVTTSS2SI64rr_Int:
8013 case X86::VCVTTSS2SIrr_Int:
8014 case X86::VCVTTSS2SI64rr_Int:
8015 case X86::VCVTTSS2SIZrr_Int:
8016 case X86::VCVTTSS2SI64Zrr_Int:
8017 case X86::VCVTSS2USIZrr_Int:
8018 case X86::VCVTSS2USI64Zrr_Int:
8019 case X86::VCVTTSS2USIZrr_Int:
8020 case X86::VCVTTSS2USI64Zrr_Int:
8021 case X86::RCPSSr_Int:
8022 case X86::VRCPSSr_Int:
8023 case X86::RSQRTSSr_Int:
8024 case X86::VRSQRTSSr_Int:
8025 case X86::ROUNDSSri_Int:
8026 case X86::VROUNDSSri_Int:
8027 case X86::COMISSrr_Int:
8028 case X86::VCOMISSrr_Int:
8029 case X86::VCOMISSZrr_Int:
8030 case X86::UCOMISSrr_Int:
8031 case X86::VUCOMISSrr_Int:
8032 case X86::VUCOMISSZrr_Int:
8033 case X86::ADDSSrr_Int:
8034 case X86::VADDSSrr_Int:
8035 case X86::VADDSSZrr_Int:
8036 case X86::CMPSSrri_Int:
8037 case X86::VCMPSSrri_Int:
8038 case X86::VCMPSSZrri_Int:
8039 case X86::DIVSSrr_Int:
8040 case X86::VDIVSSrr_Int:
8041 case X86::VDIVSSZrr_Int:
8042 case X86::MAXSSrr_Int:
8043 case X86::VMAXSSrr_Int:
8044 case X86::VMAXSSZrr_Int:
8045 case X86::MINSSrr_Int:
8046 case X86::VMINSSrr_Int:
8047 case X86::VMINSSZrr_Int:
8048 case X86::MULSSrr_Int:
8049 case X86::VMULSSrr_Int:
8050 case X86::VMULSSZrr_Int:
8051 case X86::SQRTSSr_Int:
8052 case X86::VSQRTSSr_Int:
8053 case X86::VSQRTSSZr_Int:
8054 case X86::SUBSSrr_Int:
8055 case X86::VSUBSSrr_Int:
8056 case X86::VSUBSSZrr_Int:
8057 case X86::VADDSSZrrk_Int:
8058 case X86::VADDSSZrrkz_Int:
8059 case X86::VCMPSSZrrik_Int:
8060 case X86::VDIVSSZrrk_Int:
8061 case X86::VDIVSSZrrkz_Int:
8062 case X86::VMAXSSZrrk_Int:
8063 case X86::VMAXSSZrrkz_Int:
8064 case X86::VMINSSZrrk_Int:
8065 case X86::VMINSSZrrkz_Int:
8066 case X86::VMULSSZrrk_Int:
8067 case X86::VMULSSZrrkz_Int:
8068 case X86::VSQRTSSZrk_Int:
8069 case X86::VSQRTSSZrkz_Int:
8070 case X86::VSUBSSZrrk_Int:
8071 case X86::VSUBSSZrrkz_Int:
8072 case X86::VFMADDSS4rr_Int:
8073 case X86::VFNMADDSS4rr_Int:
8074 case X86::VFMSUBSS4rr_Int:
8075 case X86::VFNMSUBSS4rr_Int:
8076 case X86::VFMADD132SSr_Int:
8077 case X86::VFNMADD132SSr_Int:
8078 case X86::VFMADD213SSr_Int:
8079 case X86::VFNMADD213SSr_Int:
8080 case X86::VFMADD231SSr_Int:
8081 case X86::VFNMADD231SSr_Int:
8082 case X86::VFMSUB132SSr_Int:
8083 case X86::VFNMSUB132SSr_Int:
8084 case X86::VFMSUB213SSr_Int:
8085 case X86::VFNMSUB213SSr_Int:
8086 case X86::VFMSUB231SSr_Int:
8087 case X86::VFNMSUB231SSr_Int:
8088 case X86::VFMADD132SSZr_Int:
8089 case X86::VFNMADD132SSZr_Int:
8090 case X86::VFMADD213SSZr_Int:
8091 case X86::VFNMADD213SSZr_Int:
8092 case X86::VFMADD231SSZr_Int:
8093 case X86::VFNMADD231SSZr_Int:
8094 case X86::VFMSUB132SSZr_Int:
8095 case X86::VFNMSUB132SSZr_Int:
8096 case X86::VFMSUB213SSZr_Int:
8097 case X86::VFNMSUB213SSZr_Int:
8098 case X86::VFMSUB231SSZr_Int:
8099 case X86::VFNMSUB231SSZr_Int:
8100 case X86::VFMADD132SSZrk_Int:
8101 case X86::VFNMADD132SSZrk_Int:
8102 case X86::VFMADD213SSZrk_Int:
8103 case X86::VFNMADD213SSZrk_Int:
8104 case X86::VFMADD231SSZrk_Int:
8105 case X86::VFNMADD231SSZrk_Int:
8106 case X86::VFMSUB132SSZrk_Int:
8107 case X86::VFNMSUB132SSZrk_Int:
8108 case X86::VFMSUB213SSZrk_Int:
8109 case X86::VFNMSUB213SSZrk_Int:
8110 case X86::VFMSUB231SSZrk_Int:
8111 case X86::VFNMSUB231SSZrk_Int:
8112 case X86::VFMADD132SSZrkz_Int:
8113 case X86::VFNMADD132SSZrkz_Int:
8114 case X86::VFMADD213SSZrkz_Int:
8115 case X86::VFNMADD213SSZrkz_Int:
8116 case X86::VFMADD231SSZrkz_Int:
8117 case X86::VFNMADD231SSZrkz_Int:
8118 case X86::VFMSUB132SSZrkz_Int:
8119 case X86::VFNMSUB132SSZrkz_Int:
8120 case X86::VFMSUB213SSZrkz_Int:
8121 case X86::VFNMSUB213SSZrkz_Int:
8122 case X86::VFMSUB231SSZrkz_Int:
8123 case X86::VFNMSUB231SSZrkz_Int:
8124 case X86::VFIXUPIMMSSZrri:
8125 case X86::VFIXUPIMMSSZrrik:
8126 case X86::VFIXUPIMMSSZrrikz:
8127 case X86::VFPCLASSSSZri:
8128 case X86::VFPCLASSSSZrik:
8129 case X86::VGETEXPSSZr:
8130 case X86::VGETEXPSSZrk:
8131 case X86::VGETEXPSSZrkz:
8132 case X86::VGETMANTSSZrri:
8133 case X86::VGETMANTSSZrrik:
8134 case X86::VGETMANTSSZrrikz:
8135 case X86::VRANGESSZrri:
8136 case X86::VRANGESSZrrik:
8137 case X86::VRANGESSZrrikz:
8138 case X86::VRCP14SSZrr:
8139 case X86::VRCP14SSZrrk:
8140 case X86::VRCP14SSZrrkz:
8141 case X86::VRCP28SSZr:
8142 case X86::VRCP28SSZrk:
8143 case X86::VRCP28SSZrkz:
8144 case X86::VREDUCESSZrri:
8145 case X86::VREDUCESSZrrik:
8146 case X86::VREDUCESSZrrikz:
8147 case X86::VRNDSCALESSZrri_Int:
8148 case X86::VRNDSCALESSZrrik_Int:
8149 case X86::VRNDSCALESSZrrikz_Int:
8150 case X86::VRSQRT14SSZrr:
8151 case X86::VRSQRT14SSZrrk:
8152 case X86::VRSQRT14SSZrrkz:
8153 case X86::VRSQRT28SSZr:
8154 case X86::VRSQRT28SSZrk:
8155 case X86::VRSQRT28SSZrkz:
8156 case X86::VSCALEFSSZrr:
8157 case X86::VSCALEFSSZrrk:
8158 case X86::VSCALEFSSZrrkz:
8159 return false;
8160 default:
8161 return true;
8162 }
8163 }
8164
8165 if ((Opc == X86::MOVSDrm || Opc == X86::VMOVSDrm || Opc == X86::VMOVSDZrm ||
8166 Opc == X86::MOVSDrm_alt || Opc == X86::VMOVSDrm_alt ||
8167 Opc == X86::VMOVSDZrm_alt) &&
8168 RegSize > 64) {
8169 // These instructions only load 64 bits, we can't fold them if the
8170 // destination register is wider than 64 bits (8 bytes), and its user
8171 // instruction isn't scalar (SD).
8172 switch (UserOpc) {
8173 case X86::CVTSD2SSrr_Int:
8174 case X86::VCVTSD2SSrr_Int:
8175 case X86::VCVTSD2SSZrr_Int:
8176 case X86::VCVTSD2SSZrrk_Int:
8177 case X86::VCVTSD2SSZrrkz_Int:
8178 case X86::CVTSD2SIrr_Int:
8179 case X86::CVTSD2SI64rr_Int:
8180 case X86::VCVTSD2SIrr_Int:
8181 case X86::VCVTSD2SI64rr_Int:
8182 case X86::VCVTSD2SIZrr_Int:
8183 case X86::VCVTSD2SI64Zrr_Int:
8184 case X86::CVTTSD2SIrr_Int:
8185 case X86::CVTTSD2SI64rr_Int:
8186 case X86::VCVTTSD2SIrr_Int:
8187 case X86::VCVTTSD2SI64rr_Int:
8188 case X86::VCVTTSD2SIZrr_Int:
8189 case X86::VCVTTSD2SI64Zrr_Int:
8190 case X86::VCVTSD2USIZrr_Int:
8191 case X86::VCVTSD2USI64Zrr_Int:
8192 case X86::VCVTTSD2USIZrr_Int:
8193 case X86::VCVTTSD2USI64Zrr_Int:
8194 case X86::ROUNDSDri_Int:
8195 case X86::VROUNDSDri_Int:
8196 case X86::COMISDrr_Int:
8197 case X86::VCOMISDrr_Int:
8198 case X86::VCOMISDZrr_Int:
8199 case X86::UCOMISDrr_Int:
8200 case X86::VUCOMISDrr_Int:
8201 case X86::VUCOMISDZrr_Int:
8202 case X86::ADDSDrr_Int:
8203 case X86::VADDSDrr_Int:
8204 case X86::VADDSDZrr_Int:
8205 case X86::CMPSDrri_Int:
8206 case X86::VCMPSDrri_Int:
8207 case X86::VCMPSDZrri_Int:
8208 case X86::DIVSDrr_Int:
8209 case X86::VDIVSDrr_Int:
8210 case X86::VDIVSDZrr_Int:
8211 case X86::MAXSDrr_Int:
8212 case X86::VMAXSDrr_Int:
8213 case X86::VMAXSDZrr_Int:
8214 case X86::MINSDrr_Int:
8215 case X86::VMINSDrr_Int:
8216 case X86::VMINSDZrr_Int:
8217 case X86::MULSDrr_Int:
8218 case X86::VMULSDrr_Int:
8219 case X86::VMULSDZrr_Int:
8220 case X86::SQRTSDr_Int:
8221 case X86::VSQRTSDr_Int:
8222 case X86::VSQRTSDZr_Int:
8223 case X86::SUBSDrr_Int:
8224 case X86::VSUBSDrr_Int:
8225 case X86::VSUBSDZrr_Int:
8226 case X86::VADDSDZrrk_Int:
8227 case X86::VADDSDZrrkz_Int:
8228 case X86::VCMPSDZrrik_Int:
8229 case X86::VDIVSDZrrk_Int:
8230 case X86::VDIVSDZrrkz_Int:
8231 case X86::VMAXSDZrrk_Int:
8232 case X86::VMAXSDZrrkz_Int:
8233 case X86::VMINSDZrrk_Int:
8234 case X86::VMINSDZrrkz_Int:
8235 case X86::VMULSDZrrk_Int:
8236 case X86::VMULSDZrrkz_Int:
8237 case X86::VSQRTSDZrk_Int:
8238 case X86::VSQRTSDZrkz_Int:
8239 case X86::VSUBSDZrrk_Int:
8240 case X86::VSUBSDZrrkz_Int:
8241 case X86::VFMADDSD4rr_Int:
8242 case X86::VFNMADDSD4rr_Int:
8243 case X86::VFMSUBSD4rr_Int:
8244 case X86::VFNMSUBSD4rr_Int:
8245 case X86::VFMADD132SDr_Int:
8246 case X86::VFNMADD132SDr_Int:
8247 case X86::VFMADD213SDr_Int:
8248 case X86::VFNMADD213SDr_Int:
8249 case X86::VFMADD231SDr_Int:
8250 case X86::VFNMADD231SDr_Int:
8251 case X86::VFMSUB132SDr_Int:
8252 case X86::VFNMSUB132SDr_Int:
8253 case X86::VFMSUB213SDr_Int:
8254 case X86::VFNMSUB213SDr_Int:
8255 case X86::VFMSUB231SDr_Int:
8256 case X86::VFNMSUB231SDr_Int:
8257 case X86::VFMADD132SDZr_Int:
8258 case X86::VFNMADD132SDZr_Int:
8259 case X86::VFMADD213SDZr_Int:
8260 case X86::VFNMADD213SDZr_Int:
8261 case X86::VFMADD231SDZr_Int:
8262 case X86::VFNMADD231SDZr_Int:
8263 case X86::VFMSUB132SDZr_Int:
8264 case X86::VFNMSUB132SDZr_Int:
8265 case X86::VFMSUB213SDZr_Int:
8266 case X86::VFNMSUB213SDZr_Int:
8267 case X86::VFMSUB231SDZr_Int:
8268 case X86::VFNMSUB231SDZr_Int:
8269 case X86::VFMADD132SDZrk_Int:
8270 case X86::VFNMADD132SDZrk_Int:
8271 case X86::VFMADD213SDZrk_Int:
8272 case X86::VFNMADD213SDZrk_Int:
8273 case X86::VFMADD231SDZrk_Int:
8274 case X86::VFNMADD231SDZrk_Int:
8275 case X86::VFMSUB132SDZrk_Int:
8276 case X86::VFNMSUB132SDZrk_Int:
8277 case X86::VFMSUB213SDZrk_Int:
8278 case X86::VFNMSUB213SDZrk_Int:
8279 case X86::VFMSUB231SDZrk_Int:
8280 case X86::VFNMSUB231SDZrk_Int:
8281 case X86::VFMADD132SDZrkz_Int:
8282 case X86::VFNMADD132SDZrkz_Int:
8283 case X86::VFMADD213SDZrkz_Int:
8284 case X86::VFNMADD213SDZrkz_Int:
8285 case X86::VFMADD231SDZrkz_Int:
8286 case X86::VFNMADD231SDZrkz_Int:
8287 case X86::VFMSUB132SDZrkz_Int:
8288 case X86::VFNMSUB132SDZrkz_Int:
8289 case X86::VFMSUB213SDZrkz_Int:
8290 case X86::VFNMSUB213SDZrkz_Int:
8291 case X86::VFMSUB231SDZrkz_Int:
8292 case X86::VFNMSUB231SDZrkz_Int:
8293 case X86::VFIXUPIMMSDZrri:
8294 case X86::VFIXUPIMMSDZrrik:
8295 case X86::VFIXUPIMMSDZrrikz:
8296 case X86::VFPCLASSSDZri:
8297 case X86::VFPCLASSSDZrik:
8298 case X86::VGETEXPSDZr:
8299 case X86::VGETEXPSDZrk:
8300 case X86::VGETEXPSDZrkz:
8301 case X86::VGETMANTSDZrri:
8302 case X86::VGETMANTSDZrrik:
8303 case X86::VGETMANTSDZrrikz:
8304 case X86::VRANGESDZrri:
8305 case X86::VRANGESDZrrik:
8306 case X86::VRANGESDZrrikz:
8307 case X86::VRCP14SDZrr:
8308 case X86::VRCP14SDZrrk:
8309 case X86::VRCP14SDZrrkz:
8310 case X86::VRCP28SDZr:
8311 case X86::VRCP28SDZrk:
8312 case X86::VRCP28SDZrkz:
8313 case X86::VREDUCESDZrri:
8314 case X86::VREDUCESDZrrik:
8315 case X86::VREDUCESDZrrikz:
8316 case X86::VRNDSCALESDZrri_Int:
8317 case X86::VRNDSCALESDZrrik_Int:
8318 case X86::VRNDSCALESDZrrikz_Int:
8319 case X86::VRSQRT14SDZrr:
8320 case X86::VRSQRT14SDZrrk:
8321 case X86::VRSQRT14SDZrrkz:
8322 case X86::VRSQRT28SDZr:
8323 case X86::VRSQRT28SDZrk:
8324 case X86::VRSQRT28SDZrkz:
8325 case X86::VSCALEFSDZrr:
8326 case X86::VSCALEFSDZrrk:
8327 case X86::VSCALEFSDZrrkz:
8328 return false;
8329 default:
8330 return true;
8331 }
8332 }
8333
8334 if ((Opc == X86::VMOVSHZrm || Opc == X86::VMOVSHZrm_alt) && RegSize > 16) {
8335 // These instructions only load 16 bits, we can't fold them if the
8336 // destination register is wider than 16 bits (2 bytes), and its user
8337 // instruction isn't scalar (SH).
8338 switch (UserOpc) {
8339 case X86::VADDSHZrr_Int:
8340 case X86::VCMPSHZrri_Int:
8341 case X86::VDIVSHZrr_Int:
8342 case X86::VMAXSHZrr_Int:
8343 case X86::VMINSHZrr_Int:
8344 case X86::VMULSHZrr_Int:
8345 case X86::VSUBSHZrr_Int:
8346 case X86::VADDSHZrrk_Int:
8347 case X86::VADDSHZrrkz_Int:
8348 case X86::VCMPSHZrrik_Int:
8349 case X86::VDIVSHZrrk_Int:
8350 case X86::VDIVSHZrrkz_Int:
8351 case X86::VMAXSHZrrk_Int:
8352 case X86::VMAXSHZrrkz_Int:
8353 case X86::VMINSHZrrk_Int:
8354 case X86::VMINSHZrrkz_Int:
8355 case X86::VMULSHZrrk_Int:
8356 case X86::VMULSHZrrkz_Int:
8357 case X86::VSUBSHZrrk_Int:
8358 case X86::VSUBSHZrrkz_Int:
8359 case X86::VFMADD132SHZr_Int:
8360 case X86::VFNMADD132SHZr_Int:
8361 case X86::VFMADD213SHZr_Int:
8362 case X86::VFNMADD213SHZr_Int:
8363 case X86::VFMADD231SHZr_Int:
8364 case X86::VFNMADD231SHZr_Int:
8365 case X86::VFMSUB132SHZr_Int:
8366 case X86::VFNMSUB132SHZr_Int:
8367 case X86::VFMSUB213SHZr_Int:
8368 case X86::VFNMSUB213SHZr_Int:
8369 case X86::VFMSUB231SHZr_Int:
8370 case X86::VFNMSUB231SHZr_Int:
8371 case X86::VFMADD132SHZrk_Int:
8372 case X86::VFNMADD132SHZrk_Int:
8373 case X86::VFMADD213SHZrk_Int:
8374 case X86::VFNMADD213SHZrk_Int:
8375 case X86::VFMADD231SHZrk_Int:
8376 case X86::VFNMADD231SHZrk_Int:
8377 case X86::VFMSUB132SHZrk_Int:
8378 case X86::VFNMSUB132SHZrk_Int:
8379 case X86::VFMSUB213SHZrk_Int:
8380 case X86::VFNMSUB213SHZrk_Int:
8381 case X86::VFMSUB231SHZrk_Int:
8382 case X86::VFNMSUB231SHZrk_Int:
8383 case X86::VFMADD132SHZrkz_Int:
8384 case X86::VFNMADD132SHZrkz_Int:
8385 case X86::VFMADD213SHZrkz_Int:
8386 case X86::VFNMADD213SHZrkz_Int:
8387 case X86::VFMADD231SHZrkz_Int:
8388 case X86::VFNMADD231SHZrkz_Int:
8389 case X86::VFMSUB132SHZrkz_Int:
8390 case X86::VFNMSUB132SHZrkz_Int:
8391 case X86::VFMSUB213SHZrkz_Int:
8392 case X86::VFNMSUB213SHZrkz_Int:
8393 case X86::VFMSUB231SHZrkz_Int:
8394 case X86::VFNMSUB231SHZrkz_Int:
8395 return false;
8396 default:
8397 return true;
8398 }
8399 }
8400
8401 return false;
8402}
8403
8404MachineInstr *
8405X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
8406 ArrayRef<unsigned> Ops,
8407 MachineInstr &LoadMI, MachineInstr *&CopyMI,
8408 LiveIntervals *LIS, VirtRegMap *VRM) const {
8409 MachineBasicBlock::iterator InsertPt = MI;
8410
8411 // If LoadMI is a masked load, check MI having the same mask.
8412 const MCInstrDesc &MCID = get(Opcode: LoadMI.getOpcode());
8413 unsigned NumOps = MCID.getNumOperands();
8414 if (NumOps >= 3) {
8415 Register MaskReg;
8416 const MachineOperand &Op1 = LoadMI.getOperand(i: 1);
8417 const MachineOperand &Op2 = LoadMI.getOperand(i: 2);
8418
8419 auto IsVKWMClass = [](const TargetRegisterClass *RC) {
8420 return RC == &X86::VK2WMRegClass || RC == &X86::VK4WMRegClass ||
8421 RC == &X86::VK8WMRegClass || RC == &X86::VK16WMRegClass ||
8422 RC == &X86::VK32WMRegClass || RC == &X86::VK64WMRegClass;
8423 };
8424
8425 if (Op1.isReg() && IsVKWMClass(getRegClass(MCID, OpNum: 1)))
8426 MaskReg = Op1.getReg();
8427 else if (Op2.isReg() && IsVKWMClass(getRegClass(MCID, OpNum: 2)))
8428 MaskReg = Op2.getReg();
8429
8430 if (MaskReg) {
8431 // Some instructions are invalid to fold into even with the same mask.
8432 // Folding is unsafe if an active destination element may read from a
8433 // source element that is masked off.
8434 if (isNonFoldableWithSameMask(RegOp: MI.getOpcode()))
8435 return nullptr;
8436 bool HasSameMask = false;
8437 for (unsigned I = 1, E = MI.getDesc().getNumOperands(); I < E; ++I) {
8438 const MachineOperand &Op = MI.getOperand(i: I);
8439 if (Op.isReg() && Op.getReg() == MaskReg) {
8440 HasSameMask = true;
8441 break;
8442 }
8443 }
8444 if (!HasSameMask)
8445 return nullptr;
8446 }
8447 }
8448
8449 // TODO: Support the case where LoadMI loads a wide register, but MI
8450 // only uses a subreg.
8451 for (auto Op : Ops) {
8452 if (MI.getOperand(i: Op).getSubReg())
8453 return nullptr;
8454 }
8455
8456 // If loading from a FrameIndex, fold directly from the FrameIndex.
8457 int FrameIndex;
8458 if (isLoadFromStackSlot(MI: LoadMI, FrameIndex)) {
8459 if (isNonFoldablePartialRegisterLoad(LoadMI, UserMI: MI, MF))
8460 return nullptr;
8461 return foldMemoryOperandImpl(MF, MI, Ops, FrameIndex, CopyMI, LIS, VRM);
8462 }
8463
8464 // Check switch flag
8465 if (NoFusing)
8466 return nullptr;
8467
8468 // Avoid partial and undef register update stalls unless optimizing for size.
8469 if (!MF.getFunction().hasOptSize() &&
8470 (hasPartialRegUpdate(Opcode: MI.getOpcode(), Subtarget, /*ForLoadFold*/ true) ||
8471 shouldPreventUndefRegUpdateMemFold(MF, MI)))
8472 return nullptr;
8473
8474 // Do not fold a NDD instruction and a memory instruction with relocation to
8475 // avoid emit APX relocation when the flag is disabled for backward
8476 // compatibility.
8477 uint64_t TSFlags = MI.getDesc().TSFlags;
8478 if (!X86EnableAPXForRelocation && isMemInstrWithGOTPCREL(MI: LoadMI) &&
8479 X86II::hasNewDataDest(TSFlags))
8480 return nullptr;
8481
8482 // Determine the alignment of the load.
8483 Align Alignment;
8484 unsigned LoadOpc = LoadMI.getOpcode();
8485 if (LoadMI.hasOneMemOperand())
8486 Alignment = (*LoadMI.memoperands_begin())->getAlign();
8487 else
8488 switch (LoadOpc) {
8489 case X86::AVX512_512_SET0:
8490 case X86::AVX512_512_SETALLONES:
8491 Alignment = Align(64);
8492 break;
8493 case X86::AVX2_SETALLONES:
8494 case X86::AVX1_SETALLONES:
8495 case X86::AVX_SET0:
8496 case X86::AVX512_256_SET0:
8497 case X86::AVX512_256_SETALLONES:
8498 Alignment = Align(32);
8499 break;
8500 case X86::V_SET0:
8501 case X86::V_SETALLONES:
8502 case X86::AVX512_128_SET0:
8503 case X86::FsFLD0F128:
8504 case X86::AVX512_FsFLD0F128:
8505 case X86::AVX512_128_SETALLONES:
8506 Alignment = Align(16);
8507 break;
8508 case X86::MMX_SET0:
8509 case X86::FsFLD0SD:
8510 case X86::AVX512_FsFLD0SD:
8511 Alignment = Align(8);
8512 break;
8513 case X86::FsFLD0SS:
8514 case X86::AVX512_FsFLD0SS:
8515 Alignment = Align(4);
8516 break;
8517 case X86::FsFLD0SH:
8518 case X86::AVX512_FsFLD0SH:
8519 Alignment = Align(2);
8520 break;
8521 default:
8522 return nullptr;
8523 }
8524 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
8525 unsigned NewOpc = 0;
8526 switch (MI.getOpcode()) {
8527 default:
8528 return nullptr;
8529 case X86::TEST8rr:
8530 NewOpc = X86::CMP8ri;
8531 break;
8532 case X86::TEST16rr:
8533 NewOpc = X86::CMP16ri;
8534 break;
8535 case X86::TEST32rr:
8536 NewOpc = X86::CMP32ri;
8537 break;
8538 case X86::TEST64rr:
8539 NewOpc = X86::CMP64ri32;
8540 break;
8541 }
8542 // Change to CMPXXri r, 0 first.
8543 MI.setDesc(get(Opcode: NewOpc));
8544 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: 0);
8545 } else if (Ops.size() != 1)
8546 return nullptr;
8547
8548 // Make sure the subregisters match.
8549 // Otherwise we risk changing the size of the load.
8550 if (LoadMI.getOperand(i: 0).getSubReg() != MI.getOperand(i: Ops[0]).getSubReg())
8551 return nullptr;
8552
8553 SmallVector<MachineOperand, X86::AddrNumOperands> MOs;
8554 switch (LoadOpc) {
8555 case X86::MMX_SET0:
8556 case X86::V_SET0:
8557 case X86::V_SETALLONES:
8558 case X86::AVX2_SETALLONES:
8559 case X86::AVX1_SETALLONES:
8560 case X86::AVX_SET0:
8561 case X86::AVX512_128_SET0:
8562 case X86::AVX512_256_SET0:
8563 case X86::AVX512_512_SET0:
8564 case X86::AVX512_128_SETALLONES:
8565 case X86::AVX512_256_SETALLONES:
8566 case X86::AVX512_512_SETALLONES:
8567 case X86::FsFLD0SH:
8568 case X86::AVX512_FsFLD0SH:
8569 case X86::FsFLD0SD:
8570 case X86::AVX512_FsFLD0SD:
8571 case X86::FsFLD0SS:
8572 case X86::AVX512_FsFLD0SS:
8573 case X86::FsFLD0F128:
8574 case X86::AVX512_FsFLD0F128: {
8575 // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure.
8576 // Create a constant-pool entry and operands to load from it.
8577
8578 // Large code model can't fold loads this way.
8579 if (MF.getTarget().getCodeModel() == CodeModel::Large)
8580 return nullptr;
8581
8582 // x86-32 PIC requires a PIC base register for constant pools.
8583 unsigned PICBase = 0;
8584 // Since we're using Small or Kernel code model, we can always use
8585 // RIP-relative addressing for a smaller encoding.
8586 if (Subtarget.is64Bit()) {
8587 PICBase = X86::RIP;
8588 } else if (MF.getTarget().isPositionIndependent()) {
8589 // FIXME: PICBase = getGlobalBaseReg(&MF);
8590 // This doesn't work for several reasons.
8591 // 1. GlobalBaseReg may have been spilled.
8592 // 2. It may not be live at MI.
8593 return nullptr;
8594 }
8595
8596 // Create a constant-pool entry.
8597 MachineConstantPool &MCP = *MF.getConstantPool();
8598 Type *Ty;
8599 bool IsAllOnes = false;
8600 switch (LoadOpc) {
8601 case X86::FsFLD0SS:
8602 case X86::AVX512_FsFLD0SS:
8603 Ty = Type::getFloatTy(C&: MF.getFunction().getContext());
8604 break;
8605 case X86::FsFLD0SD:
8606 case X86::AVX512_FsFLD0SD:
8607 Ty = Type::getDoubleTy(C&: MF.getFunction().getContext());
8608 break;
8609 case X86::FsFLD0F128:
8610 case X86::AVX512_FsFLD0F128:
8611 Ty = Type::getFP128Ty(C&: MF.getFunction().getContext());
8612 break;
8613 case X86::FsFLD0SH:
8614 case X86::AVX512_FsFLD0SH:
8615 Ty = Type::getHalfTy(C&: MF.getFunction().getContext());
8616 break;
8617 case X86::AVX512_512_SETALLONES:
8618 IsAllOnes = true;
8619 [[fallthrough]];
8620 case X86::AVX512_512_SET0:
8621 Ty = FixedVectorType::get(ElementType: Type::getInt32Ty(C&: MF.getFunction().getContext()),
8622 NumElts: 16);
8623 break;
8624 case X86::AVX1_SETALLONES:
8625 case X86::AVX2_SETALLONES:
8626 case X86::AVX512_256_SETALLONES:
8627 IsAllOnes = true;
8628 [[fallthrough]];
8629 case X86::AVX512_256_SET0:
8630 case X86::AVX_SET0:
8631 Ty = FixedVectorType::get(ElementType: Type::getInt32Ty(C&: MF.getFunction().getContext()),
8632 NumElts: 8);
8633
8634 break;
8635 case X86::MMX_SET0:
8636 Ty = FixedVectorType::get(ElementType: Type::getInt32Ty(C&: MF.getFunction().getContext()),
8637 NumElts: 2);
8638 break;
8639 case X86::V_SETALLONES:
8640 case X86::AVX512_128_SETALLONES:
8641 IsAllOnes = true;
8642 [[fallthrough]];
8643 case X86::V_SET0:
8644 case X86::AVX512_128_SET0:
8645 Ty = FixedVectorType::get(ElementType: Type::getInt32Ty(C&: MF.getFunction().getContext()),
8646 NumElts: 4);
8647 break;
8648 }
8649
8650 const Constant *C =
8651 IsAllOnes ? Constant::getAllOnesValue(Ty) : Constant::getNullValue(Ty);
8652 unsigned CPI = MCP.getConstantPoolIndex(C, Alignment);
8653
8654 // Create operands to load from the constant pool entry.
8655 MOs.push_back(Elt: MachineOperand::CreateReg(Reg: PICBase, isDef: false));
8656 MOs.push_back(Elt: MachineOperand::CreateImm(Val: 1));
8657 MOs.push_back(Elt: MachineOperand::CreateReg(Reg: 0, isDef: false));
8658 MOs.push_back(Elt: MachineOperand::CreateCPI(Idx: CPI, Offset: 0));
8659 MOs.push_back(Elt: MachineOperand::CreateReg(Reg: 0, isDef: false));
8660 break;
8661 }
8662 case X86::VPBROADCASTBZ128rm:
8663 case X86::VPBROADCASTBZ256rm:
8664 case X86::VPBROADCASTBZrm:
8665 case X86::VBROADCASTF32X2Z256rm:
8666 case X86::VBROADCASTF32X2Zrm:
8667 case X86::VBROADCASTI32X2Z128rm:
8668 case X86::VBROADCASTI32X2Z256rm:
8669 case X86::VBROADCASTI32X2Zrm:
8670 // No instructions currently fuse with 8bits or 32bits x 2.
8671 return nullptr;
8672
8673#define FOLD_BROADCAST(SIZE) \
8674 MOs.append(LoadMI.operands_begin() + NumOps - X86::AddrNumOperands, \
8675 LoadMI.operands_begin() + NumOps); \
8676 return foldMemoryBroadcast(MF, MI, Ops[0], MOs, InsertPt, /*Size=*/SIZE, \
8677 /*AllowCommute=*/true);
8678 case X86::VPBROADCASTWZ128rm:
8679 case X86::VPBROADCASTWZ256rm:
8680 case X86::VPBROADCASTWZrm:
8681 FOLD_BROADCAST(16);
8682 case X86::VPBROADCASTDZ128rm:
8683 case X86::VPBROADCASTDZ256rm:
8684 case X86::VPBROADCASTDZrm:
8685 case X86::VBROADCASTSSZ128rm:
8686 case X86::VBROADCASTSSZ256rm:
8687 case X86::VBROADCASTSSZrm:
8688 FOLD_BROADCAST(32);
8689 case X86::VPBROADCASTQZ128rm:
8690 case X86::VPBROADCASTQZ256rm:
8691 case X86::VPBROADCASTQZrm:
8692 case X86::VBROADCASTSDZ256rm:
8693 case X86::VBROADCASTSDZrm:
8694 FOLD_BROADCAST(64);
8695 default: {
8696 if (isNonFoldablePartialRegisterLoad(LoadMI, UserMI: MI, MF))
8697 return nullptr;
8698
8699 // Folding a normal load. Just copy the load's address operands.
8700 MOs.append(in_start: LoadMI.operands_begin() + NumOps - X86::AddrNumOperands,
8701 in_end: LoadMI.operands_begin() + NumOps);
8702 break;
8703 }
8704 }
8705 return foldMemoryOperandImpl(MF, MI, OpNum: Ops[0], MOs, InsertPt,
8706 /*Size=*/0, Alignment, /*AllowCommute=*/true,
8707 CopyMI);
8708}
8709
8710MachineInstr *
8711X86InstrInfo::foldMemoryBroadcast(MachineFunction &MF, MachineInstr &MI,
8712 unsigned OpNum, ArrayRef<MachineOperand> MOs,
8713 MachineBasicBlock::iterator InsertPt,
8714 unsigned BitsSize, bool AllowCommute) const {
8715
8716 if (auto *I = lookupBroadcastFoldTable(RegOp: MI.getOpcode(), OpNum))
8717 return matchBroadcastSize(Entry: *I, BroadcastBits: BitsSize)
8718 ? fuseInst(MF, Opcode: I->DstOp, OpNo: OpNum, MOs, InsertPt, MI, TII: *this)
8719 : nullptr;
8720
8721 if (AllowCommute) {
8722 // If the instruction and target operand are commutable, commute the
8723 // instruction and try again.
8724 unsigned CommuteOpIdx2 = commuteOperandsForFold(MI, Idx1: OpNum);
8725 if (CommuteOpIdx2 == OpNum) {
8726 printFailMsgforFold(MI, Idx: OpNum);
8727 return nullptr;
8728 }
8729 MachineInstr *NewMI =
8730 foldMemoryBroadcast(MF, MI, OpNum: CommuteOpIdx2, MOs, InsertPt, BitsSize,
8731 /*AllowCommute=*/false);
8732 if (NewMI)
8733 return NewMI;
8734 // Folding failed again - undo the commute before returning.
8735 commuteInstruction(MI, NewMI: false, OpIdx1: OpNum, OpIdx2: CommuteOpIdx2);
8736 }
8737
8738 printFailMsgforFold(MI, Idx: OpNum);
8739 return nullptr;
8740}
8741
8742static SmallVector<MachineMemOperand *, 2>
8743extractLoadMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) {
8744 SmallVector<MachineMemOperand *, 2> LoadMMOs;
8745
8746 for (MachineMemOperand *MMO : MMOs) {
8747 if (!MMO->isLoad())
8748 continue;
8749
8750 if (!MMO->isStore()) {
8751 // Reuse the MMO.
8752 LoadMMOs.push_back(Elt: MMO);
8753 } else {
8754 // Clone the MMO and unset the store flag.
8755 LoadMMOs.push_back(Elt: MF.getMachineMemOperand(
8756 MMO, Flags: MMO->getFlags() & ~MachineMemOperand::MOStore));
8757 }
8758 }
8759
8760 return LoadMMOs;
8761}
8762
8763static SmallVector<MachineMemOperand *, 2>
8764extractStoreMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) {
8765 SmallVector<MachineMemOperand *, 2> StoreMMOs;
8766
8767 for (MachineMemOperand *MMO : MMOs) {
8768 if (!MMO->isStore())
8769 continue;
8770
8771 if (!MMO->isLoad()) {
8772 // Reuse the MMO.
8773 StoreMMOs.push_back(Elt: MMO);
8774 } else {
8775 // Clone the MMO and unset the load flag.
8776 StoreMMOs.push_back(Elt: MF.getMachineMemOperand(
8777 MMO, Flags: MMO->getFlags() & ~MachineMemOperand::MOLoad));
8778 }
8779 }
8780
8781 return StoreMMOs;
8782}
8783
8784static unsigned getBroadcastOpcode(const X86FoldTableEntry *I,
8785 const TargetRegisterClass *RC,
8786 const X86Subtarget &STI) {
8787 assert(STI.hasAVX512() && "Expected at least AVX512!");
8788 unsigned SpillSize = STI.getRegisterInfo()->getSpillSize(RC: *RC);
8789 assert((SpillSize == 64 || STI.hasVLX()) &&
8790 "Can't broadcast less than 64 bytes without AVX512VL!");
8791
8792#define CASE_BCAST_TYPE_OPC(TYPE, OP16, OP32, OP64) \
8793 case TYPE: \
8794 switch (SpillSize) { \
8795 default: \
8796 llvm_unreachable("Unknown spill size"); \
8797 case 16: \
8798 return X86::OP16; \
8799 case 32: \
8800 return X86::OP32; \
8801 case 64: \
8802 return X86::OP64; \
8803 } \
8804 break;
8805
8806 switch (I->Flags & TB_BCAST_MASK) {
8807 default:
8808 llvm_unreachable("Unexpected broadcast type!");
8809 CASE_BCAST_TYPE_OPC(TB_BCAST_W, VPBROADCASTWZ128rm, VPBROADCASTWZ256rm,
8810 VPBROADCASTWZrm)
8811 CASE_BCAST_TYPE_OPC(TB_BCAST_D, VPBROADCASTDZ128rm, VPBROADCASTDZ256rm,
8812 VPBROADCASTDZrm)
8813 CASE_BCAST_TYPE_OPC(TB_BCAST_Q, VPBROADCASTQZ128rm, VPBROADCASTQZ256rm,
8814 VPBROADCASTQZrm)
8815 CASE_BCAST_TYPE_OPC(TB_BCAST_SH, VPBROADCASTWZ128rm, VPBROADCASTWZ256rm,
8816 VPBROADCASTWZrm)
8817 CASE_BCAST_TYPE_OPC(TB_BCAST_SS, VBROADCASTSSZ128rm, VBROADCASTSSZ256rm,
8818 VBROADCASTSSZrm)
8819 CASE_BCAST_TYPE_OPC(TB_BCAST_SD, VMOVDDUPZ128rm, VBROADCASTSDZ256rm,
8820 VBROADCASTSDZrm)
8821 }
8822}
8823
8824bool X86InstrInfo::unfoldMemoryOperand(
8825 MachineFunction &MF, MachineInstr &MI, Register Reg, bool UnfoldLoad,
8826 bool UnfoldStore, SmallVectorImpl<MachineInstr *> &NewMIs) const {
8827 const X86FoldTableEntry *I = lookupUnfoldTable(MemOp: MI.getOpcode());
8828 if (I == nullptr)
8829 return false;
8830 unsigned Opc = I->DstOp;
8831 unsigned Index = I->Flags & TB_INDEX_MASK;
8832 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD;
8833 bool FoldedStore = I->Flags & TB_FOLDED_STORE;
8834 if (UnfoldLoad && !FoldedLoad)
8835 return false;
8836 UnfoldLoad &= FoldedLoad;
8837 if (UnfoldStore && !FoldedStore)
8838 return false;
8839 UnfoldStore &= FoldedStore;
8840
8841 const MCInstrDesc &MCID = get(Opcode: Opc);
8842
8843 const TargetRegisterClass *RC = getRegClass(MCID, OpNum: Index);
8844 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8845 // TODO: Check if 32-byte or greater accesses are slow too?
8846 if (!MI.hasOneMemOperand() && RC == &X86::VR128RegClass &&
8847 Subtarget.isUnalignedMem16Slow())
8848 // Without memoperands, loadRegFromAddr and storeRegToStackSlot will
8849 // conservatively assume the address is unaligned. That's bad for
8850 // performance.
8851 return false;
8852 SmallVector<MachineOperand, X86::AddrNumOperands> AddrOps;
8853 SmallVector<MachineOperand, 2> BeforeOps;
8854 SmallVector<MachineOperand, 2> AfterOps;
8855 SmallVector<MachineOperand, 4> ImpOps;
8856 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
8857 MachineOperand &Op = MI.getOperand(i);
8858 if (i >= Index && i < Index + X86::AddrNumOperands)
8859 AddrOps.push_back(Elt: Op);
8860 else if (Op.isReg() && Op.isImplicit())
8861 ImpOps.push_back(Elt: Op);
8862 else if (i < Index)
8863 BeforeOps.push_back(Elt: Op);
8864 else if (i > Index)
8865 AfterOps.push_back(Elt: Op);
8866 }
8867
8868 // Emit the load or broadcast instruction.
8869 if (UnfoldLoad) {
8870 auto MMOs = extractLoadMMOs(MMOs: MI.memoperands(), MF);
8871
8872 unsigned Opc;
8873 if (I->Flags & TB_BCAST_MASK) {
8874 Opc = getBroadcastOpcode(I, RC, STI: Subtarget);
8875 } else {
8876 unsigned Alignment = std::max<uint32_t>(a: TRI.getSpillSize(RC: *RC), b: 16);
8877 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
8878 Opc = getLoadRegOpcode(DestReg: Reg, RC, IsStackAligned: isAligned, STI: Subtarget);
8879 }
8880
8881 DebugLoc DL;
8882 MachineInstrBuilder MIB = BuildMI(MF, MIMD: DL, MCID: get(Opcode: Opc), DestReg: Reg);
8883 for (const MachineOperand &AddrOp : AddrOps)
8884 MIB.add(MO: AddrOp);
8885 MIB.setMemRefs(MMOs);
8886 NewMIs.push_back(Elt: MIB);
8887
8888 if (UnfoldStore) {
8889 // Address operands cannot be marked isKill.
8890 for (unsigned i = 1; i != 1 + X86::AddrNumOperands; ++i) {
8891 MachineOperand &MO = NewMIs[0]->getOperand(i);
8892 if (MO.isReg())
8893 MO.setIsKill(false);
8894 }
8895 }
8896 }
8897
8898 // Emit the data processing instruction.
8899 MachineInstr *DataMI = MF.CreateMachineInstr(MCID, DL: MI.getDebugLoc(), NoImplicit: true);
8900 MachineInstrBuilder MIB(MF, DataMI);
8901
8902 if (FoldedStore)
8903 MIB.addReg(RegNo: Reg, Flags: RegState::Define);
8904 for (MachineOperand &BeforeOp : BeforeOps)
8905 MIB.add(MO: BeforeOp);
8906 if (FoldedLoad)
8907 MIB.addReg(RegNo: Reg);
8908 for (MachineOperand &AfterOp : AfterOps)
8909 MIB.add(MO: AfterOp);
8910 for (MachineOperand &ImpOp : ImpOps) {
8911 MIB.addReg(RegNo: ImpOp.getReg(), Flags: getDefRegState(B: ImpOp.isDef()) |
8912 RegState::Implicit |
8913 getKillRegState(B: ImpOp.isKill()) |
8914 getDeadRegState(B: ImpOp.isDead()) |
8915 getUndefRegState(B: ImpOp.isUndef()));
8916 }
8917 // Change CMP32ri r, 0 back to TEST32rr r, r, etc.
8918 switch (DataMI->getOpcode()) {
8919 default:
8920 break;
8921 case X86::CMP64ri32:
8922 case X86::CMP32ri:
8923 case X86::CMP16ri:
8924 case X86::CMP8ri: {
8925 MachineOperand &MO0 = DataMI->getOperand(i: 0);
8926 MachineOperand &MO1 = DataMI->getOperand(i: 1);
8927 if (MO1.isImm() && MO1.getImm() == 0) {
8928 unsigned NewOpc;
8929 switch (DataMI->getOpcode()) {
8930 default:
8931 llvm_unreachable("Unreachable!");
8932 case X86::CMP64ri32:
8933 NewOpc = X86::TEST64rr;
8934 break;
8935 case X86::CMP32ri:
8936 NewOpc = X86::TEST32rr;
8937 break;
8938 case X86::CMP16ri:
8939 NewOpc = X86::TEST16rr;
8940 break;
8941 case X86::CMP8ri:
8942 NewOpc = X86::TEST8rr;
8943 break;
8944 }
8945 DataMI->setDesc(get(Opcode: NewOpc));
8946 MO1.ChangeToRegister(Reg: MO0.getReg(), isDef: false);
8947 }
8948 }
8949 }
8950 NewMIs.push_back(Elt: DataMI);
8951
8952 // Emit the store instruction.
8953 if (UnfoldStore) {
8954 const TargetRegisterClass *DstRC = getRegClass(MCID, OpNum: 0);
8955 auto MMOs = extractStoreMMOs(MMOs: MI.memoperands(), MF);
8956 unsigned Alignment = std::max<uint32_t>(a: TRI.getSpillSize(RC: *DstRC), b: 16);
8957 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
8958 unsigned Opc = getStoreRegOpcode(SrcReg: Reg, RC: DstRC, IsStackAligned: isAligned, STI: Subtarget);
8959 DebugLoc DL;
8960 MachineInstrBuilder MIB = BuildMI(MF, MIMD: DL, MCID: get(Opcode: Opc));
8961 for (const MachineOperand &AddrOp : AddrOps)
8962 MIB.add(MO: AddrOp);
8963 MIB.addReg(RegNo: Reg, Flags: RegState::Kill);
8964 MIB.setMemRefs(MMOs);
8965 NewMIs.push_back(Elt: MIB);
8966 }
8967
8968 return true;
8969}
8970
8971bool X86InstrInfo::unfoldMemoryOperand(
8972 SelectionDAG &DAG, SDNode *N, SmallVectorImpl<SDNode *> &NewNodes) const {
8973 if (!N->isMachineOpcode())
8974 return false;
8975
8976 const X86FoldTableEntry *I = lookupUnfoldTable(MemOp: N->getMachineOpcode());
8977 if (I == nullptr)
8978 return false;
8979 unsigned Opc = I->DstOp;
8980 unsigned Index = I->Flags & TB_INDEX_MASK;
8981 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD;
8982 bool FoldedStore = I->Flags & TB_FOLDED_STORE;
8983 const MCInstrDesc &MCID = get(Opcode: Opc);
8984 MachineFunction &MF = DAG.getMachineFunction();
8985 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
8986 const TargetRegisterClass *RC = getRegClass(MCID, OpNum: Index);
8987 unsigned NumDefs = MCID.NumDefs;
8988 std::vector<SDValue> AddrOps;
8989 std::vector<SDValue> BeforeOps;
8990 std::vector<SDValue> AfterOps;
8991 SDLoc dl(N);
8992 unsigned NumOps = N->getNumOperands();
8993 for (unsigned i = 0; i != NumOps - 1; ++i) {
8994 SDValue Op = N->getOperand(Num: i);
8995 if (i >= Index - NumDefs && i < Index - NumDefs + X86::AddrNumOperands)
8996 AddrOps.push_back(x: Op);
8997 else if (i < Index - NumDefs)
8998 BeforeOps.push_back(x: Op);
8999 else if (i > Index - NumDefs)
9000 AfterOps.push_back(x: Op);
9001 }
9002 SDValue Chain = N->getOperand(Num: NumOps - 1);
9003 AddrOps.push_back(x: Chain);
9004
9005 // Emit the load instruction.
9006 SDNode *Load = nullptr;
9007 if (FoldedLoad) {
9008 EVT VT = *TRI.legalclasstypes_begin(RC: *RC);
9009 auto MMOs = extractLoadMMOs(MMOs: cast<MachineSDNode>(Val: N)->memoperands(), MF);
9010 if (MMOs.empty() && RC == &X86::VR128RegClass &&
9011 Subtarget.isUnalignedMem16Slow())
9012 // Do not introduce a slow unaligned load.
9013 return false;
9014 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte
9015 // memory access is slow above.
9016
9017 unsigned Opc;
9018 if (I->Flags & TB_BCAST_MASK) {
9019 Opc = getBroadcastOpcode(I, RC, STI: Subtarget);
9020 } else {
9021 unsigned Alignment = std::max<uint32_t>(a: TRI.getSpillSize(RC: *RC), b: 16);
9022 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
9023 Opc = getLoadRegOpcode(DestReg: 0, RC, IsStackAligned: isAligned, STI: Subtarget);
9024 }
9025
9026 Load = DAG.getMachineNode(Opcode: Opc, dl, VT1: VT, VT2: MVT::Other, Ops: AddrOps);
9027 NewNodes.push_back(Elt: Load);
9028
9029 // Preserve memory reference information.
9030 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load), NewMemRefs: MMOs);
9031 }
9032
9033 // Emit the data processing instruction.
9034 std::vector<EVT> VTs;
9035 const TargetRegisterClass *DstRC = nullptr;
9036 if (MCID.getNumDefs() > 0) {
9037 DstRC = getRegClass(MCID, OpNum: 0);
9038 VTs.push_back(x: *TRI.legalclasstypes_begin(RC: *DstRC));
9039 }
9040 for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
9041 EVT VT = N->getValueType(ResNo: i);
9042 if (VT != MVT::Other && i >= (unsigned)MCID.getNumDefs())
9043 VTs.push_back(x: VT);
9044 }
9045 if (Load)
9046 BeforeOps.push_back(x: SDValue(Load, 0));
9047 llvm::append_range(C&: BeforeOps, R&: AfterOps);
9048 // Change CMP32ri r, 0 back to TEST32rr r, r, etc.
9049 switch (Opc) {
9050 default:
9051 break;
9052 case X86::CMP64ri32:
9053 case X86::CMP32ri:
9054 case X86::CMP16ri:
9055 case X86::CMP8ri:
9056 if (isNullConstant(V: BeforeOps[1])) {
9057 switch (Opc) {
9058 default:
9059 llvm_unreachable("Unreachable!");
9060 case X86::CMP64ri32:
9061 Opc = X86::TEST64rr;
9062 break;
9063 case X86::CMP32ri:
9064 Opc = X86::TEST32rr;
9065 break;
9066 case X86::CMP16ri:
9067 Opc = X86::TEST16rr;
9068 break;
9069 case X86::CMP8ri:
9070 Opc = X86::TEST8rr;
9071 break;
9072 }
9073 BeforeOps[1] = BeforeOps[0];
9074 }
9075 }
9076 SDNode *NewNode = DAG.getMachineNode(Opcode: Opc, dl, ResultTys: VTs, Ops: BeforeOps);
9077 NewNodes.push_back(Elt: NewNode);
9078
9079 // Emit the store instruction.
9080 if (FoldedStore) {
9081 AddrOps.pop_back();
9082 AddrOps.push_back(x: SDValue(NewNode, 0));
9083 AddrOps.push_back(x: Chain);
9084 auto MMOs = extractStoreMMOs(MMOs: cast<MachineSDNode>(Val: N)->memoperands(), MF);
9085 if (MMOs.empty() && RC == &X86::VR128RegClass &&
9086 Subtarget.isUnalignedMem16Slow())
9087 // Do not introduce a slow unaligned store.
9088 return false;
9089 // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte
9090 // memory access is slow above.
9091 unsigned Alignment = std::max<uint32_t>(a: TRI.getSpillSize(RC: *RC), b: 16);
9092 bool isAligned = !MMOs.empty() && MMOs.front()->getAlign() >= Alignment;
9093 SDNode *Store =
9094 DAG.getMachineNode(Opcode: getStoreRegOpcode(SrcReg: 0, RC: DstRC, IsStackAligned: isAligned, STI: Subtarget),
9095 dl, VT: MVT::Other, Ops: AddrOps);
9096 NewNodes.push_back(Elt: Store);
9097
9098 // Preserve memory reference information.
9099 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Store), NewMemRefs: MMOs);
9100 }
9101
9102 return true;
9103}
9104
9105unsigned
9106X86InstrInfo::getOpcodeAfterMemoryUnfold(unsigned Opc, bool UnfoldLoad,
9107 bool UnfoldStore,
9108 unsigned *LoadRegIndex) const {
9109 const X86FoldTableEntry *I = lookupUnfoldTable(MemOp: Opc);
9110 if (I == nullptr)
9111 return 0;
9112 bool FoldedLoad = I->Flags & TB_FOLDED_LOAD;
9113 bool FoldedStore = I->Flags & TB_FOLDED_STORE;
9114 if (UnfoldLoad && !FoldedLoad)
9115 return 0;
9116 if (UnfoldStore && !FoldedStore)
9117 return 0;
9118 if (LoadRegIndex)
9119 *LoadRegIndex = I->Flags & TB_INDEX_MASK;
9120 return I->DstOp;
9121}
9122
9123bool X86InstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
9124 int64_t &Offset1,
9125 int64_t &Offset2) const {
9126 if (!Load1->isMachineOpcode() || !Load2->isMachineOpcode())
9127 return false;
9128
9129 auto IsLoadOpcode = [&](unsigned Opcode) {
9130 switch (Opcode) {
9131 default:
9132 return false;
9133 case X86::MOV8rm:
9134 case X86::MOV16rm:
9135 case X86::MOV32rm:
9136 case X86::MOV64rm:
9137 case X86::LD_Fp32m:
9138 case X86::LD_Fp64m:
9139 case X86::LD_Fp80m:
9140 case X86::MOVSSrm:
9141 case X86::MOVSSrm_alt:
9142 case X86::MOVSDrm:
9143 case X86::MOVSDrm_alt:
9144 case X86::MMX_MOVD64rm:
9145 case X86::MMX_MOVQ64rm:
9146 case X86::MOVAPSrm:
9147 case X86::MOVUPSrm:
9148 case X86::MOVAPDrm:
9149 case X86::MOVUPDrm:
9150 case X86::MOVDQArm:
9151 case X86::MOVDQUrm:
9152 // AVX load instructions
9153 case X86::VMOVSSrm:
9154 case X86::VMOVSSrm_alt:
9155 case X86::VMOVSDrm:
9156 case X86::VMOVSDrm_alt:
9157 case X86::VMOVAPSrm:
9158 case X86::VMOVUPSrm:
9159 case X86::VMOVAPDrm:
9160 case X86::VMOVUPDrm:
9161 case X86::VMOVDQArm:
9162 case X86::VMOVDQUrm:
9163 case X86::VMOVAPSYrm:
9164 case X86::VMOVUPSYrm:
9165 case X86::VMOVAPDYrm:
9166 case X86::VMOVUPDYrm:
9167 case X86::VMOVDQAYrm:
9168 case X86::VMOVDQUYrm:
9169 // AVX512 load instructions
9170 case X86::VMOVSSZrm:
9171 case X86::VMOVSSZrm_alt:
9172 case X86::VMOVSDZrm:
9173 case X86::VMOVSDZrm_alt:
9174 case X86::VMOVAPSZ128rm:
9175 case X86::VMOVUPSZ128rm:
9176 case X86::VMOVAPSZ128rm_NOVLX:
9177 case X86::VMOVUPSZ128rm_NOVLX:
9178 case X86::VMOVAPDZ128rm:
9179 case X86::VMOVUPDZ128rm:
9180 case X86::VMOVDQU8Z128rm:
9181 case X86::VMOVDQU16Z128rm:
9182 case X86::VMOVDQA32Z128rm:
9183 case X86::VMOVDQU32Z128rm:
9184 case X86::VMOVDQA64Z128rm:
9185 case X86::VMOVDQU64Z128rm:
9186 case X86::VMOVAPSZ256rm:
9187 case X86::VMOVUPSZ256rm:
9188 case X86::VMOVAPSZ256rm_NOVLX:
9189 case X86::VMOVUPSZ256rm_NOVLX:
9190 case X86::VMOVAPDZ256rm:
9191 case X86::VMOVUPDZ256rm:
9192 case X86::VMOVDQU8Z256rm:
9193 case X86::VMOVDQU16Z256rm:
9194 case X86::VMOVDQA32Z256rm:
9195 case X86::VMOVDQU32Z256rm:
9196 case X86::VMOVDQA64Z256rm:
9197 case X86::VMOVDQU64Z256rm:
9198 case X86::VMOVAPSZrm:
9199 case X86::VMOVUPSZrm:
9200 case X86::VMOVAPDZrm:
9201 case X86::VMOVUPDZrm:
9202 case X86::VMOVDQU8Zrm:
9203 case X86::VMOVDQU16Zrm:
9204 case X86::VMOVDQA32Zrm:
9205 case X86::VMOVDQU32Zrm:
9206 case X86::VMOVDQA64Zrm:
9207 case X86::VMOVDQU64Zrm:
9208 case X86::KMOVBkm:
9209 case X86::KMOVBkm_EVEX:
9210 case X86::KMOVWkm:
9211 case X86::KMOVWkm_EVEX:
9212 case X86::KMOVDkm:
9213 case X86::KMOVDkm_EVEX:
9214 case X86::KMOVQkm:
9215 case X86::KMOVQkm_EVEX:
9216 return true;
9217 }
9218 };
9219
9220 if (!IsLoadOpcode(Load1->getMachineOpcode()) ||
9221 !IsLoadOpcode(Load2->getMachineOpcode()))
9222 return false;
9223
9224 // Lambda to check if both the loads have the same value for an operand index.
9225 auto HasSameOp = [&](int I) {
9226 return Load1->getOperand(Num: I) == Load2->getOperand(Num: I);
9227 };
9228
9229 // All operands except the displacement should match.
9230 if (!HasSameOp(X86::AddrBaseReg) || !HasSameOp(X86::AddrScaleAmt) ||
9231 !HasSameOp(X86::AddrIndexReg) || !HasSameOp(X86::AddrSegmentReg))
9232 return false;
9233
9234 // Chain Operand must be the same.
9235 if (!HasSameOp(5))
9236 return false;
9237
9238 // Now let's examine if the displacements are constants.
9239 auto Disp1 = dyn_cast<ConstantSDNode>(Val: Load1->getOperand(Num: X86::AddrDisp));
9240 auto Disp2 = dyn_cast<ConstantSDNode>(Val: Load2->getOperand(Num: X86::AddrDisp));
9241 if (!Disp1 || !Disp2)
9242 return false;
9243
9244 Offset1 = Disp1->getSExtValue();
9245 Offset2 = Disp2->getSExtValue();
9246 return true;
9247}
9248
9249bool X86InstrInfo::shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2,
9250 int64_t Offset1, int64_t Offset2,
9251 unsigned NumLoads) const {
9252 assert(Offset2 > Offset1);
9253 if ((Offset2 - Offset1) / 8 > 64)
9254 return false;
9255
9256 unsigned Opc1 = Load1->getMachineOpcode();
9257 unsigned Opc2 = Load2->getMachineOpcode();
9258 if (Opc1 != Opc2)
9259 return false; // FIXME: overly conservative?
9260
9261 switch (Opc1) {
9262 default:
9263 break;
9264 case X86::LD_Fp32m:
9265 case X86::LD_Fp64m:
9266 case X86::LD_Fp80m:
9267 case X86::MMX_MOVD64rm:
9268 case X86::MMX_MOVQ64rm:
9269 return false;
9270 }
9271
9272 EVT VT = Load1->getValueType(ResNo: 0);
9273 switch (VT.getSimpleVT().SimpleTy) {
9274 default:
9275 // XMM registers. In 64-bit mode we can be a bit more aggressive since we
9276 // have 16 of them to play with.
9277 if (Subtarget.is64Bit()) {
9278 if (NumLoads >= 3)
9279 return false;
9280 } else if (NumLoads) {
9281 return false;
9282 }
9283 break;
9284 case MVT::i8:
9285 case MVT::i16:
9286 case MVT::i32:
9287 case MVT::i64:
9288 case MVT::f32:
9289 case MVT::f64:
9290 if (NumLoads)
9291 return false;
9292 break;
9293 }
9294
9295 return true;
9296}
9297
9298bool X86InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
9299 const MachineBasicBlock *MBB,
9300 const MachineFunction &MF) const {
9301
9302 // ENDBR instructions should not be scheduled around.
9303 unsigned Opcode = MI.getOpcode();
9304 if (Opcode == X86::ENDBR64 || Opcode == X86::ENDBR32 ||
9305 Opcode == X86::PLDTILECFGV)
9306 return true;
9307
9308 // Frame setup and destroy can't be scheduled around.
9309 if (MI.getFlag(Flag: MachineInstr::FrameSetup) ||
9310 MI.getFlag(Flag: MachineInstr::FrameDestroy))
9311 return true;
9312
9313 return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF);
9314}
9315
9316bool X86InstrInfo::reverseBranchCondition(
9317 SmallVectorImpl<MachineOperand> &Cond) const {
9318 assert(Cond.size() == 1 && "Invalid X86 branch condition!");
9319 X86::CondCode CC = static_cast<X86::CondCode>(Cond[0].getImm());
9320 Cond[0].setImm(GetOppositeBranchCondition(CC));
9321 return false;
9322}
9323
9324bool X86InstrInfo::isSafeToMoveRegClassDefs(
9325 const TargetRegisterClass *RC) const {
9326 // FIXME: Return false for x87 stack register classes for now. We can't
9327 // allow any loads of these registers before FpGet_ST0_80.
9328 return !(RC == &X86::CCRRegClass || RC == &X86::DFCCRRegClass ||
9329 RC == &X86::RFP32RegClass || RC == &X86::RFP64RegClass ||
9330 RC == &X86::RFP80RegClass);
9331}
9332
9333/// Return a virtual register initialized with the
9334/// the global base register value. Output instructions required to
9335/// initialize the register in the function entry block, if necessary.
9336///
9337/// TODO: Eliminate this and move the code to X86MachineFunctionInfo.
9338///
9339Register X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const {
9340 X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>();
9341 Register GlobalBaseReg = X86FI->getGlobalBaseReg();
9342 if (GlobalBaseReg)
9343 return GlobalBaseReg;
9344
9345 // Create the register. The code to initialize it is inserted
9346 // later, by the CGBR pass (below).
9347 MachineRegisterInfo &RegInfo = MF->getRegInfo();
9348 GlobalBaseReg = RegInfo.createVirtualRegister(
9349 RegClass: Subtarget.is64Bit() ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass);
9350 X86FI->setGlobalBaseReg(GlobalBaseReg);
9351 return GlobalBaseReg;
9352}
9353
9354// FIXME: Some shuffle and unpack instructions have equivalents in different
9355// domains, but they require a bit more work than just switching opcodes.
9356
9357static const uint16_t *lookup(unsigned opcode, unsigned domain,
9358 ArrayRef<uint16_t[3]> Table) {
9359 for (const uint16_t(&Row)[3] : Table)
9360 if (Row[domain - 1] == opcode)
9361 return Row;
9362 return nullptr;
9363}
9364
9365static const uint16_t *lookupAVX512(unsigned opcode, unsigned domain,
9366 ArrayRef<uint16_t[4]> Table) {
9367 // If this is the integer domain make sure to check both integer columns.
9368 for (const uint16_t(&Row)[4] : Table)
9369 if (Row[domain - 1] == opcode || (domain == 3 && Row[3] == opcode))
9370 return Row;
9371 return nullptr;
9372}
9373
9374// Helper to attempt to widen/narrow blend masks.
9375static bool AdjustBlendMask(unsigned OldMask, unsigned OldWidth,
9376 unsigned NewWidth, unsigned *pNewMask = nullptr) {
9377 assert(((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) &&
9378 "Illegal blend mask scale");
9379 unsigned NewMask = 0;
9380
9381 if ((OldWidth % NewWidth) == 0) {
9382 unsigned Scale = OldWidth / NewWidth;
9383 unsigned SubMask = (1u << Scale) - 1;
9384 for (unsigned i = 0; i != NewWidth; ++i) {
9385 unsigned Sub = (OldMask >> (i * Scale)) & SubMask;
9386 if (Sub == SubMask)
9387 NewMask |= (1u << i);
9388 else if (Sub != 0x0)
9389 return false;
9390 }
9391 } else {
9392 unsigned Scale = NewWidth / OldWidth;
9393 unsigned SubMask = (1u << Scale) - 1;
9394 for (unsigned i = 0; i != OldWidth; ++i) {
9395 if (OldMask & (1 << i)) {
9396 NewMask |= (SubMask << (i * Scale));
9397 }
9398 }
9399 }
9400
9401 if (pNewMask)
9402 *pNewMask = NewMask;
9403 return true;
9404}
9405
9406uint16_t X86InstrInfo::getExecutionDomainCustom(const MachineInstr &MI) const {
9407 unsigned Opcode = MI.getOpcode();
9408 unsigned NumOperands = MI.getDesc().getNumOperands();
9409
9410 auto GetBlendDomains = [&](unsigned ImmWidth, bool Is256) {
9411 uint16_t validDomains = 0;
9412 if (MI.getOperand(i: NumOperands - 1).isImm()) {
9413 unsigned Imm = MI.getOperand(i: NumOperands - 1).getImm();
9414 if (AdjustBlendMask(OldMask: Imm, OldWidth: ImmWidth, NewWidth: Is256 ? 8 : 4))
9415 validDomains |= 0x2; // PackedSingle
9416 if (AdjustBlendMask(OldMask: Imm, OldWidth: ImmWidth, NewWidth: Is256 ? 4 : 2))
9417 validDomains |= 0x4; // PackedDouble
9418 if (!Is256 || Subtarget.hasAVX2())
9419 validDomains |= 0x8; // PackedInt
9420 }
9421 return validDomains;
9422 };
9423
9424 switch (Opcode) {
9425 case X86::BLENDPDrmi:
9426 case X86::BLENDPDrri:
9427 case X86::VBLENDPDrmi:
9428 case X86::VBLENDPDrri:
9429 return GetBlendDomains(2, false);
9430 case X86::VBLENDPDYrmi:
9431 case X86::VBLENDPDYrri:
9432 return GetBlendDomains(4, true);
9433 case X86::BLENDPSrmi:
9434 case X86::BLENDPSrri:
9435 case X86::VBLENDPSrmi:
9436 case X86::VBLENDPSrri:
9437 case X86::VPBLENDDrmi:
9438 case X86::VPBLENDDrri:
9439 return GetBlendDomains(4, false);
9440 case X86::VBLENDPSYrmi:
9441 case X86::VBLENDPSYrri:
9442 case X86::VPBLENDDYrmi:
9443 case X86::VPBLENDDYrri:
9444 return GetBlendDomains(8, true);
9445 case X86::PBLENDWrmi:
9446 case X86::PBLENDWrri:
9447 case X86::VPBLENDWrmi:
9448 case X86::VPBLENDWrri:
9449 // Treat VPBLENDWY as a 128-bit vector as it repeats the lo/hi masks.
9450 case X86::VPBLENDWYrmi:
9451 case X86::VPBLENDWYrri:
9452 return GetBlendDomains(8, false);
9453 case X86::VPANDDZ128rr:
9454 case X86::VPANDDZ128rm:
9455 case X86::VPANDDZ256rr:
9456 case X86::VPANDDZ256rm:
9457 case X86::VPANDQZ128rr:
9458 case X86::VPANDQZ128rm:
9459 case X86::VPANDQZ256rr:
9460 case X86::VPANDQZ256rm:
9461 case X86::VPANDNDZ128rr:
9462 case X86::VPANDNDZ128rm:
9463 case X86::VPANDNDZ256rr:
9464 case X86::VPANDNDZ256rm:
9465 case X86::VPANDNQZ128rr:
9466 case X86::VPANDNQZ128rm:
9467 case X86::VPANDNQZ256rr:
9468 case X86::VPANDNQZ256rm:
9469 case X86::VPORDZ128rr:
9470 case X86::VPORDZ128rm:
9471 case X86::VPORDZ256rr:
9472 case X86::VPORDZ256rm:
9473 case X86::VPORQZ128rr:
9474 case X86::VPORQZ128rm:
9475 case X86::VPORQZ256rr:
9476 case X86::VPORQZ256rm:
9477 case X86::VPXORDZ128rr:
9478 case X86::VPXORDZ128rm:
9479 case X86::VPXORDZ256rr:
9480 case X86::VPXORDZ256rm:
9481 case X86::VPXORQZ128rr:
9482 case X86::VPXORQZ128rm:
9483 case X86::VPXORQZ256rr:
9484 case X86::VPXORQZ256rm:
9485 // If we don't have DQI see if we can still switch from an EVEX integer
9486 // instruction to a VEX floating point instruction.
9487 if (Subtarget.hasDQI())
9488 return 0;
9489
9490 if (RI.getEncodingValue(Reg: MI.getOperand(i: 0).getReg()) >= 16)
9491 return 0;
9492 if (RI.getEncodingValue(Reg: MI.getOperand(i: 1).getReg()) >= 16)
9493 return 0;
9494 // Register forms will have 3 operands. Memory form will have more.
9495 if (NumOperands == 3 &&
9496 RI.getEncodingValue(Reg: MI.getOperand(i: 2).getReg()) >= 16)
9497 return 0;
9498
9499 // All domains are valid.
9500 return 0xe;
9501 case X86::MOVHLPSrr:
9502 // We can swap domains when both inputs are the same register.
9503 // FIXME: This doesn't catch all the cases we would like. If the input
9504 // register isn't KILLed by the instruction, the two address instruction
9505 // pass puts a COPY on one input. The other input uses the original
9506 // register. This prevents the same physical register from being used by
9507 // both inputs.
9508 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg() &&
9509 MI.getOperand(i: 0).getSubReg() == 0 &&
9510 MI.getOperand(i: 1).getSubReg() == 0 && MI.getOperand(i: 2).getSubReg() == 0)
9511 return 0x6;
9512 return 0;
9513 case X86::SHUFPDrri:
9514 return 0x6;
9515 }
9516 return 0;
9517}
9518
9519#include "X86ReplaceableInstrs.def"
9520
9521bool X86InstrInfo::setExecutionDomainCustom(MachineInstr &MI,
9522 unsigned Domain) const {
9523 assert(Domain > 0 && Domain < 4 && "Invalid execution domain");
9524 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3;
9525 assert(dom && "Not an SSE instruction");
9526
9527 unsigned Opcode = MI.getOpcode();
9528 unsigned NumOperands = MI.getDesc().getNumOperands();
9529
9530 auto SetBlendDomain = [&](unsigned ImmWidth, bool Is256) {
9531 if (MI.getOperand(i: NumOperands - 1).isImm()) {
9532 unsigned Imm = MI.getOperand(i: NumOperands - 1).getImm() & 255;
9533 Imm = (ImmWidth == 16 ? ((Imm << 8) | Imm) : Imm);
9534 unsigned NewImm = Imm;
9535
9536 const uint16_t *table = lookup(opcode: Opcode, domain: dom, Table: ReplaceableBlendInstrs);
9537 if (!table)
9538 table = lookup(opcode: Opcode, domain: dom, Table: ReplaceableBlendAVX2Instrs);
9539
9540 if (Domain == 1) { // PackedSingle
9541 AdjustBlendMask(OldMask: Imm, OldWidth: ImmWidth, NewWidth: Is256 ? 8 : 4, pNewMask: &NewImm);
9542 } else if (Domain == 2) { // PackedDouble
9543 AdjustBlendMask(OldMask: Imm, OldWidth: ImmWidth, NewWidth: Is256 ? 4 : 2, pNewMask: &NewImm);
9544 } else if (Domain == 3) { // PackedInt
9545 if (Subtarget.hasAVX2()) {
9546 // If we are already VPBLENDW use that, else use VPBLENDD.
9547 if ((ImmWidth / (Is256 ? 2 : 1)) != 8) {
9548 table = lookup(opcode: Opcode, domain: dom, Table: ReplaceableBlendAVX2Instrs);
9549 AdjustBlendMask(OldMask: Imm, OldWidth: ImmWidth, NewWidth: Is256 ? 8 : 4, pNewMask: &NewImm);
9550 }
9551 } else {
9552 assert(!Is256 && "128-bit vector expected");
9553 AdjustBlendMask(OldMask: Imm, OldWidth: ImmWidth, NewWidth: 8, pNewMask: &NewImm);
9554 }
9555 }
9556
9557 assert(table && table[Domain - 1] && "Unknown domain op");
9558 MI.setDesc(get(Opcode: table[Domain - 1]));
9559 MI.getOperand(i: NumOperands - 1).setImm(NewImm & 255);
9560 }
9561 return true;
9562 };
9563
9564 switch (Opcode) {
9565 case X86::BLENDPDrmi:
9566 case X86::BLENDPDrri:
9567 case X86::VBLENDPDrmi:
9568 case X86::VBLENDPDrri:
9569 return SetBlendDomain(2, false);
9570 case X86::VBLENDPDYrmi:
9571 case X86::VBLENDPDYrri:
9572 return SetBlendDomain(4, true);
9573 case X86::BLENDPSrmi:
9574 case X86::BLENDPSrri:
9575 case X86::VBLENDPSrmi:
9576 case X86::VBLENDPSrri:
9577 case X86::VPBLENDDrmi:
9578 case X86::VPBLENDDrri:
9579 return SetBlendDomain(4, false);
9580 case X86::VBLENDPSYrmi:
9581 case X86::VBLENDPSYrri:
9582 case X86::VPBLENDDYrmi:
9583 case X86::VPBLENDDYrri:
9584 return SetBlendDomain(8, true);
9585 case X86::PBLENDWrmi:
9586 case X86::PBLENDWrri:
9587 case X86::VPBLENDWrmi:
9588 case X86::VPBLENDWrri:
9589 return SetBlendDomain(8, false);
9590 case X86::VPBLENDWYrmi:
9591 case X86::VPBLENDWYrri:
9592 return SetBlendDomain(16, true);
9593 case X86::VPANDDZ128rr:
9594 case X86::VPANDDZ128rm:
9595 case X86::VPANDDZ256rr:
9596 case X86::VPANDDZ256rm:
9597 case X86::VPANDQZ128rr:
9598 case X86::VPANDQZ128rm:
9599 case X86::VPANDQZ256rr:
9600 case X86::VPANDQZ256rm:
9601 case X86::VPANDNDZ128rr:
9602 case X86::VPANDNDZ128rm:
9603 case X86::VPANDNDZ256rr:
9604 case X86::VPANDNDZ256rm:
9605 case X86::VPANDNQZ128rr:
9606 case X86::VPANDNQZ128rm:
9607 case X86::VPANDNQZ256rr:
9608 case X86::VPANDNQZ256rm:
9609 case X86::VPORDZ128rr:
9610 case X86::VPORDZ128rm:
9611 case X86::VPORDZ256rr:
9612 case X86::VPORDZ256rm:
9613 case X86::VPORQZ128rr:
9614 case X86::VPORQZ128rm:
9615 case X86::VPORQZ256rr:
9616 case X86::VPORQZ256rm:
9617 case X86::VPXORDZ128rr:
9618 case X86::VPXORDZ128rm:
9619 case X86::VPXORDZ256rr:
9620 case X86::VPXORDZ256rm:
9621 case X86::VPXORQZ128rr:
9622 case X86::VPXORQZ128rm:
9623 case X86::VPXORQZ256rr:
9624 case X86::VPXORQZ256rm: {
9625 // Without DQI, convert EVEX instructions to VEX instructions.
9626 if (Subtarget.hasDQI())
9627 return false;
9628
9629 const uint16_t *table =
9630 lookupAVX512(opcode: MI.getOpcode(), domain: dom, Table: ReplaceableCustomAVX512LogicInstrs);
9631 assert(table && "Instruction not found in table?");
9632 // Don't change integer Q instructions to D instructions and
9633 // use D intructions if we started with a PS instruction.
9634 if (Domain == 3 && (dom == 1 || table[3] == MI.getOpcode()))
9635 Domain = 4;
9636 MI.setDesc(get(Opcode: table[Domain - 1]));
9637 return true;
9638 }
9639 case X86::UNPCKHPDrr:
9640 case X86::MOVHLPSrr:
9641 // We just need to commute the instruction which will switch the domains.
9642 if (Domain != dom && Domain != 3 &&
9643 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg() &&
9644 MI.getOperand(i: 0).getSubReg() == 0 &&
9645 MI.getOperand(i: 1).getSubReg() == 0 &&
9646 MI.getOperand(i: 2).getSubReg() == 0) {
9647 commuteInstruction(MI, NewMI: false);
9648 return true;
9649 }
9650 // We must always return true for MOVHLPSrr.
9651 if (Opcode == X86::MOVHLPSrr)
9652 return true;
9653 break;
9654 case X86::SHUFPDrri: {
9655 if (Domain == 1) {
9656 unsigned Imm = MI.getOperand(i: 3).getImm();
9657 unsigned NewImm = 0x44;
9658 if (Imm & 1)
9659 NewImm |= 0x0a;
9660 if (Imm & 2)
9661 NewImm |= 0xa0;
9662 MI.getOperand(i: 3).setImm(NewImm);
9663 MI.setDesc(get(Opcode: X86::SHUFPSrri));
9664 }
9665 return true;
9666 }
9667 }
9668 return false;
9669}
9670
9671std::pair<uint16_t, uint16_t>
9672X86InstrInfo::getExecutionDomain(const MachineInstr &MI) const {
9673 uint16_t domain = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3;
9674 unsigned opcode = MI.getOpcode();
9675 uint16_t validDomains = 0;
9676 if (domain) {
9677 // Attempt to match for custom instructions.
9678 validDomains = getExecutionDomainCustom(MI);
9679 if (validDomains)
9680 return std::make_pair(x&: domain, y&: validDomains);
9681
9682 if (lookup(opcode, domain, Table: ReplaceableInstrs)) {
9683 validDomains = 0xe;
9684 } else if (lookup(opcode, domain, Table: ReplaceableInstrsAVX2)) {
9685 validDomains = Subtarget.hasAVX2() ? 0xe : 0x6;
9686 } else if (lookup(opcode, domain, Table: ReplaceableInstrsFP)) {
9687 validDomains = 0x6;
9688 } else if (lookup(opcode, domain, Table: ReplaceableInstrsAVX2InsertExtract)) {
9689 // Insert/extract instructions should only effect domain if AVX2
9690 // is enabled.
9691 if (!Subtarget.hasAVX2())
9692 return std::make_pair(x: 0, y: 0);
9693 validDomains = 0xe;
9694 } else if (lookupAVX512(opcode, domain, Table: ReplaceableInstrsAVX512)) {
9695 validDomains = 0xe;
9696 } else if (Subtarget.hasDQI() &&
9697 lookupAVX512(opcode, domain, Table: ReplaceableInstrsAVX512DQ)) {
9698 validDomains = 0xe;
9699 } else if (Subtarget.hasDQI()) {
9700 if (const uint16_t *table =
9701 lookupAVX512(opcode, domain, Table: ReplaceableInstrsAVX512DQMasked)) {
9702 if (domain == 1 || (domain == 3 && table[3] == opcode))
9703 validDomains = 0xa;
9704 else
9705 validDomains = 0xc;
9706 }
9707 }
9708 }
9709 return std::make_pair(x&: domain, y&: validDomains);
9710}
9711
9712void X86InstrInfo::setExecutionDomain(MachineInstr &MI, unsigned Domain) const {
9713 assert(Domain > 0 && Domain < 4 && "Invalid execution domain");
9714 uint16_t dom = (MI.getDesc().TSFlags >> X86II::SSEDomainShift) & 3;
9715 assert(dom && "Not an SSE instruction");
9716
9717 // Attempt to match for custom instructions.
9718 if (setExecutionDomainCustom(MI, Domain))
9719 return;
9720
9721 const uint16_t *table = lookup(opcode: MI.getOpcode(), domain: dom, Table: ReplaceableInstrs);
9722 if (!table) { // try the other table
9723 assert((Subtarget.hasAVX2() || Domain < 3) &&
9724 "256-bit vector operations only available in AVX2");
9725 table = lookup(opcode: MI.getOpcode(), domain: dom, Table: ReplaceableInstrsAVX2);
9726 }
9727 if (!table) { // try the FP table
9728 table = lookup(opcode: MI.getOpcode(), domain: dom, Table: ReplaceableInstrsFP);
9729 assert((!table || Domain < 3) &&
9730 "Can only select PackedSingle or PackedDouble");
9731 }
9732 if (!table) { // try the other table
9733 assert(Subtarget.hasAVX2() &&
9734 "256-bit insert/extract only available in AVX2");
9735 table = lookup(opcode: MI.getOpcode(), domain: dom, Table: ReplaceableInstrsAVX2InsertExtract);
9736 }
9737 if (!table) { // try the AVX512 table
9738 assert(Subtarget.hasAVX512() && "Requires AVX-512");
9739 table = lookupAVX512(opcode: MI.getOpcode(), domain: dom, Table: ReplaceableInstrsAVX512);
9740 // Don't change integer Q instructions to D instructions.
9741 if (table && Domain == 3 && table[3] == MI.getOpcode())
9742 Domain = 4;
9743 }
9744 if (!table) { // try the AVX512DQ table
9745 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ");
9746 table = lookupAVX512(opcode: MI.getOpcode(), domain: dom, Table: ReplaceableInstrsAVX512DQ);
9747 // Don't change integer Q instructions to D instructions and
9748 // use D instructions if we started with a PS instruction.
9749 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode()))
9750 Domain = 4;
9751 }
9752 if (!table) { // try the AVX512DQMasked table
9753 assert((Subtarget.hasDQI() || Domain >= 3) && "Requires AVX-512DQ");
9754 table = lookupAVX512(opcode: MI.getOpcode(), domain: dom, Table: ReplaceableInstrsAVX512DQMasked);
9755 if (table && Domain == 3 && (dom == 1 || table[3] == MI.getOpcode()))
9756 Domain = 4;
9757 }
9758 assert(table && "Cannot change domain");
9759 MI.setDesc(get(Opcode: table[Domain - 1]));
9760}
9761
9762void X86InstrInfo::insertNoop(MachineBasicBlock &MBB,
9763 MachineBasicBlock::iterator MI) const {
9764 DebugLoc DL;
9765 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: X86::NOOP));
9766}
9767
9768/// Return the noop instruction to use for a noop.
9769MCInst X86InstrInfo::getNop() const {
9770 MCInst Nop;
9771 Nop.setOpcode(X86::NOOP);
9772 return Nop;
9773}
9774
9775bool X86InstrInfo::isHighLatencyDef(int opc) const {
9776 switch (opc) {
9777 default:
9778 return false;
9779 case X86::DIVPDrm:
9780 case X86::DIVPDrr:
9781 case X86::DIVPSrm:
9782 case X86::DIVPSrr:
9783 case X86::DIVSDrm:
9784 case X86::DIVSDrm_Int:
9785 case X86::DIVSDrr:
9786 case X86::DIVSDrr_Int:
9787 case X86::DIVSSrm:
9788 case X86::DIVSSrm_Int:
9789 case X86::DIVSSrr:
9790 case X86::DIVSSrr_Int:
9791 case X86::SQRTPDm:
9792 case X86::SQRTPDr:
9793 case X86::SQRTPSm:
9794 case X86::SQRTPSr:
9795 case X86::SQRTSDm:
9796 case X86::SQRTSDm_Int:
9797 case X86::SQRTSDr:
9798 case X86::SQRTSDr_Int:
9799 case X86::SQRTSSm:
9800 case X86::SQRTSSm_Int:
9801 case X86::SQRTSSr:
9802 case X86::SQRTSSr_Int:
9803 // AVX instructions with high latency
9804 case X86::VDIVPDrm:
9805 case X86::VDIVPDrr:
9806 case X86::VDIVPDYrm:
9807 case X86::VDIVPDYrr:
9808 case X86::VDIVPSrm:
9809 case X86::VDIVPSrr:
9810 case X86::VDIVPSYrm:
9811 case X86::VDIVPSYrr:
9812 case X86::VDIVSDrm:
9813 case X86::VDIVSDrm_Int:
9814 case X86::VDIVSDrr:
9815 case X86::VDIVSDrr_Int:
9816 case X86::VDIVSSrm:
9817 case X86::VDIVSSrm_Int:
9818 case X86::VDIVSSrr:
9819 case X86::VDIVSSrr_Int:
9820 case X86::VSQRTPDm:
9821 case X86::VSQRTPDr:
9822 case X86::VSQRTPDYm:
9823 case X86::VSQRTPDYr:
9824 case X86::VSQRTPSm:
9825 case X86::VSQRTPSr:
9826 case X86::VSQRTPSYm:
9827 case X86::VSQRTPSYr:
9828 case X86::VSQRTSDm:
9829 case X86::VSQRTSDm_Int:
9830 case X86::VSQRTSDr:
9831 case X86::VSQRTSDr_Int:
9832 case X86::VSQRTSSm:
9833 case X86::VSQRTSSm_Int:
9834 case X86::VSQRTSSr:
9835 case X86::VSQRTSSr_Int:
9836 // AVX512 instructions with high latency
9837 case X86::VDIVPDZ128rm:
9838 case X86::VDIVPDZ128rmb:
9839 case X86::VDIVPDZ128rmbk:
9840 case X86::VDIVPDZ128rmbkz:
9841 case X86::VDIVPDZ128rmk:
9842 case X86::VDIVPDZ128rmkz:
9843 case X86::VDIVPDZ128rr:
9844 case X86::VDIVPDZ128rrk:
9845 case X86::VDIVPDZ128rrkz:
9846 case X86::VDIVPDZ256rm:
9847 case X86::VDIVPDZ256rmb:
9848 case X86::VDIVPDZ256rmbk:
9849 case X86::VDIVPDZ256rmbkz:
9850 case X86::VDIVPDZ256rmk:
9851 case X86::VDIVPDZ256rmkz:
9852 case X86::VDIVPDZ256rr:
9853 case X86::VDIVPDZ256rrk:
9854 case X86::VDIVPDZ256rrkz:
9855 case X86::VDIVPDZrrb:
9856 case X86::VDIVPDZrrbk:
9857 case X86::VDIVPDZrrbkz:
9858 case X86::VDIVPDZrm:
9859 case X86::VDIVPDZrmb:
9860 case X86::VDIVPDZrmbk:
9861 case X86::VDIVPDZrmbkz:
9862 case X86::VDIVPDZrmk:
9863 case X86::VDIVPDZrmkz:
9864 case X86::VDIVPDZrr:
9865 case X86::VDIVPDZrrk:
9866 case X86::VDIVPDZrrkz:
9867 case X86::VDIVPSZ128rm:
9868 case X86::VDIVPSZ128rmb:
9869 case X86::VDIVPSZ128rmbk:
9870 case X86::VDIVPSZ128rmbkz:
9871 case X86::VDIVPSZ128rmk:
9872 case X86::VDIVPSZ128rmkz:
9873 case X86::VDIVPSZ128rr:
9874 case X86::VDIVPSZ128rrk:
9875 case X86::VDIVPSZ128rrkz:
9876 case X86::VDIVPSZ256rm:
9877 case X86::VDIVPSZ256rmb:
9878 case X86::VDIVPSZ256rmbk:
9879 case X86::VDIVPSZ256rmbkz:
9880 case X86::VDIVPSZ256rmk:
9881 case X86::VDIVPSZ256rmkz:
9882 case X86::VDIVPSZ256rr:
9883 case X86::VDIVPSZ256rrk:
9884 case X86::VDIVPSZ256rrkz:
9885 case X86::VDIVPSZrrb:
9886 case X86::VDIVPSZrrbk:
9887 case X86::VDIVPSZrrbkz:
9888 case X86::VDIVPSZrm:
9889 case X86::VDIVPSZrmb:
9890 case X86::VDIVPSZrmbk:
9891 case X86::VDIVPSZrmbkz:
9892 case X86::VDIVPSZrmk:
9893 case X86::VDIVPSZrmkz:
9894 case X86::VDIVPSZrr:
9895 case X86::VDIVPSZrrk:
9896 case X86::VDIVPSZrrkz:
9897 case X86::VDIVSDZrm:
9898 case X86::VDIVSDZrr:
9899 case X86::VDIVSDZrm_Int:
9900 case X86::VDIVSDZrmk_Int:
9901 case X86::VDIVSDZrmkz_Int:
9902 case X86::VDIVSDZrr_Int:
9903 case X86::VDIVSDZrrk_Int:
9904 case X86::VDIVSDZrrkz_Int:
9905 case X86::VDIVSDZrrb_Int:
9906 case X86::VDIVSDZrrbk_Int:
9907 case X86::VDIVSDZrrbkz_Int:
9908 case X86::VDIVSSZrm:
9909 case X86::VDIVSSZrr:
9910 case X86::VDIVSSZrm_Int:
9911 case X86::VDIVSSZrmk_Int:
9912 case X86::VDIVSSZrmkz_Int:
9913 case X86::VDIVSSZrr_Int:
9914 case X86::VDIVSSZrrk_Int:
9915 case X86::VDIVSSZrrkz_Int:
9916 case X86::VDIVSSZrrb_Int:
9917 case X86::VDIVSSZrrbk_Int:
9918 case X86::VDIVSSZrrbkz_Int:
9919 case X86::VSQRTPDZ128m:
9920 case X86::VSQRTPDZ128mb:
9921 case X86::VSQRTPDZ128mbk:
9922 case X86::VSQRTPDZ128mbkz:
9923 case X86::VSQRTPDZ128mk:
9924 case X86::VSQRTPDZ128mkz:
9925 case X86::VSQRTPDZ128r:
9926 case X86::VSQRTPDZ128rk:
9927 case X86::VSQRTPDZ128rkz:
9928 case X86::VSQRTPDZ256m:
9929 case X86::VSQRTPDZ256mb:
9930 case X86::VSQRTPDZ256mbk:
9931 case X86::VSQRTPDZ256mbkz:
9932 case X86::VSQRTPDZ256mk:
9933 case X86::VSQRTPDZ256mkz:
9934 case X86::VSQRTPDZ256r:
9935 case X86::VSQRTPDZ256rk:
9936 case X86::VSQRTPDZ256rkz:
9937 case X86::VSQRTPDZm:
9938 case X86::VSQRTPDZmb:
9939 case X86::VSQRTPDZmbk:
9940 case X86::VSQRTPDZmbkz:
9941 case X86::VSQRTPDZmk:
9942 case X86::VSQRTPDZmkz:
9943 case X86::VSQRTPDZr:
9944 case X86::VSQRTPDZrb:
9945 case X86::VSQRTPDZrbk:
9946 case X86::VSQRTPDZrbkz:
9947 case X86::VSQRTPDZrk:
9948 case X86::VSQRTPDZrkz:
9949 case X86::VSQRTPSZ128m:
9950 case X86::VSQRTPSZ128mb:
9951 case X86::VSQRTPSZ128mbk:
9952 case X86::VSQRTPSZ128mbkz:
9953 case X86::VSQRTPSZ128mk:
9954 case X86::VSQRTPSZ128mkz:
9955 case X86::VSQRTPSZ128r:
9956 case X86::VSQRTPSZ128rk:
9957 case X86::VSQRTPSZ128rkz:
9958 case X86::VSQRTPSZ256m:
9959 case X86::VSQRTPSZ256mb:
9960 case X86::VSQRTPSZ256mbk:
9961 case X86::VSQRTPSZ256mbkz:
9962 case X86::VSQRTPSZ256mk:
9963 case X86::VSQRTPSZ256mkz:
9964 case X86::VSQRTPSZ256r:
9965 case X86::VSQRTPSZ256rk:
9966 case X86::VSQRTPSZ256rkz:
9967 case X86::VSQRTPSZm:
9968 case X86::VSQRTPSZmb:
9969 case X86::VSQRTPSZmbk:
9970 case X86::VSQRTPSZmbkz:
9971 case X86::VSQRTPSZmk:
9972 case X86::VSQRTPSZmkz:
9973 case X86::VSQRTPSZr:
9974 case X86::VSQRTPSZrb:
9975 case X86::VSQRTPSZrbk:
9976 case X86::VSQRTPSZrbkz:
9977 case X86::VSQRTPSZrk:
9978 case X86::VSQRTPSZrkz:
9979 case X86::VSQRTSDZm:
9980 case X86::VSQRTSDZm_Int:
9981 case X86::VSQRTSDZmk_Int:
9982 case X86::VSQRTSDZmkz_Int:
9983 case X86::VSQRTSDZr:
9984 case X86::VSQRTSDZr_Int:
9985 case X86::VSQRTSDZrk_Int:
9986 case X86::VSQRTSDZrkz_Int:
9987 case X86::VSQRTSDZrb_Int:
9988 case X86::VSQRTSDZrbk_Int:
9989 case X86::VSQRTSDZrbkz_Int:
9990 case X86::VSQRTSSZm:
9991 case X86::VSQRTSSZm_Int:
9992 case X86::VSQRTSSZmk_Int:
9993 case X86::VSQRTSSZmkz_Int:
9994 case X86::VSQRTSSZr:
9995 case X86::VSQRTSSZr_Int:
9996 case X86::VSQRTSSZrk_Int:
9997 case X86::VSQRTSSZrkz_Int:
9998 case X86::VSQRTSSZrb_Int:
9999 case X86::VSQRTSSZrbk_Int:
10000 case X86::VSQRTSSZrbkz_Int:
10001
10002 case X86::VGATHERDPDYrm:
10003 case X86::VGATHERDPDZ128rm:
10004 case X86::VGATHERDPDZ256rm:
10005 case X86::VGATHERDPDZrm:
10006 case X86::VGATHERDPDrm:
10007 case X86::VGATHERDPSYrm:
10008 case X86::VGATHERDPSZ128rm:
10009 case X86::VGATHERDPSZ256rm:
10010 case X86::VGATHERDPSZrm:
10011 case X86::VGATHERDPSrm:
10012 case X86::VGATHERPF0DPDm:
10013 case X86::VGATHERPF0DPSm:
10014 case X86::VGATHERPF0QPDm:
10015 case X86::VGATHERPF0QPSm:
10016 case X86::VGATHERPF1DPDm:
10017 case X86::VGATHERPF1DPSm:
10018 case X86::VGATHERPF1QPDm:
10019 case X86::VGATHERPF1QPSm:
10020 case X86::VGATHERQPDYrm:
10021 case X86::VGATHERQPDZ128rm:
10022 case X86::VGATHERQPDZ256rm:
10023 case X86::VGATHERQPDZrm:
10024 case X86::VGATHERQPDrm:
10025 case X86::VGATHERQPSYrm:
10026 case X86::VGATHERQPSZ128rm:
10027 case X86::VGATHERQPSZ256rm:
10028 case X86::VGATHERQPSZrm:
10029 case X86::VGATHERQPSrm:
10030 case X86::VPGATHERDDYrm:
10031 case X86::VPGATHERDDZ128rm:
10032 case X86::VPGATHERDDZ256rm:
10033 case X86::VPGATHERDDZrm:
10034 case X86::VPGATHERDDrm:
10035 case X86::VPGATHERDQYrm:
10036 case X86::VPGATHERDQZ128rm:
10037 case X86::VPGATHERDQZ256rm:
10038 case X86::VPGATHERDQZrm:
10039 case X86::VPGATHERDQrm:
10040 case X86::VPGATHERQDYrm:
10041 case X86::VPGATHERQDZ128rm:
10042 case X86::VPGATHERQDZ256rm:
10043 case X86::VPGATHERQDZrm:
10044 case X86::VPGATHERQDrm:
10045 case X86::VPGATHERQQYrm:
10046 case X86::VPGATHERQQZ128rm:
10047 case X86::VPGATHERQQZ256rm:
10048 case X86::VPGATHERQQZrm:
10049 case X86::VPGATHERQQrm:
10050 case X86::VSCATTERDPDZ128mr:
10051 case X86::VSCATTERDPDZ256mr:
10052 case X86::VSCATTERDPDZmr:
10053 case X86::VSCATTERDPSZ128mr:
10054 case X86::VSCATTERDPSZ256mr:
10055 case X86::VSCATTERDPSZmr:
10056 case X86::VSCATTERPF0DPDm:
10057 case X86::VSCATTERPF0DPSm:
10058 case X86::VSCATTERPF0QPDm:
10059 case X86::VSCATTERPF0QPSm:
10060 case X86::VSCATTERPF1DPDm:
10061 case X86::VSCATTERPF1DPSm:
10062 case X86::VSCATTERPF1QPDm:
10063 case X86::VSCATTERPF1QPSm:
10064 case X86::VSCATTERQPDZ128mr:
10065 case X86::VSCATTERQPDZ256mr:
10066 case X86::VSCATTERQPDZmr:
10067 case X86::VSCATTERQPSZ128mr:
10068 case X86::VSCATTERQPSZ256mr:
10069 case X86::VSCATTERQPSZmr:
10070 case X86::VPSCATTERDDZ128mr:
10071 case X86::VPSCATTERDDZ256mr:
10072 case X86::VPSCATTERDDZmr:
10073 case X86::VPSCATTERDQZ128mr:
10074 case X86::VPSCATTERDQZ256mr:
10075 case X86::VPSCATTERDQZmr:
10076 case X86::VPSCATTERQDZ128mr:
10077 case X86::VPSCATTERQDZ256mr:
10078 case X86::VPSCATTERQDZmr:
10079 case X86::VPSCATTERQQZ128mr:
10080 case X86::VPSCATTERQQZ256mr:
10081 case X86::VPSCATTERQQZmr:
10082 return true;
10083 }
10084}
10085
10086bool X86InstrInfo::hasHighOperandLatency(const TargetSchedModel &SchedModel,
10087 const MachineRegisterInfo *MRI,
10088 const MachineInstr &DefMI,
10089 unsigned DefIdx,
10090 const MachineInstr &UseMI,
10091 unsigned UseIdx) const {
10092 return isHighLatencyDef(opc: DefMI.getOpcode());
10093}
10094
10095bool X86InstrInfo::hasReassociableOperands(const MachineInstr &Inst,
10096 const MachineBasicBlock *MBB) const {
10097 assert(Inst.getNumExplicitOperands() == 3 && Inst.getNumExplicitDefs() == 1 &&
10098 Inst.getNumDefs() <= 2 && "Reassociation needs binary operators");
10099
10100 // Integer binary math/logic instructions have a third source operand:
10101 // the EFLAGS register. That operand must be both defined here and never
10102 // used; ie, it must be dead. If the EFLAGS operand is live, then we can
10103 // not change anything because rearranging the operands could affect other
10104 // instructions that depend on the exact status flags (zero, sign, etc.)
10105 // that are set by using these particular operands with this operation.
10106 const MachineOperand *FlagDef =
10107 Inst.findRegisterDefOperand(Reg: X86::EFLAGS, /*TRI=*/nullptr);
10108 assert((Inst.getNumDefs() == 1 || FlagDef) && "Implicit def isn't flags?");
10109 if (FlagDef && !FlagDef->isDead())
10110 return false;
10111
10112 return TargetInstrInfo::hasReassociableOperands(Inst, MBB);
10113}
10114
10115// TODO: There are many more machine instruction opcodes to match:
10116// 1. Other data types (integer, vectors)
10117// 2. Other math / logic operations (xor, or)
10118// 3. Other forms of the same operation (intrinsics and other variants)
10119bool X86InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
10120 bool Invert) const {
10121 if (Invert)
10122 return false;
10123 switch (Inst.getOpcode()) {
10124 CASE_ND(ADD8rr)
10125 CASE_ND(ADD16rr)
10126 CASE_ND(ADD32rr)
10127 CASE_ND(ADD64rr)
10128 CASE_ND(AND8rr)
10129 CASE_ND(AND16rr)
10130 CASE_ND(AND32rr)
10131 CASE_ND(AND64rr)
10132 CASE_ND(OR8rr)
10133 CASE_ND(OR16rr)
10134 CASE_ND(OR32rr)
10135 CASE_ND(OR64rr)
10136 CASE_ND(XOR8rr)
10137 CASE_ND(XOR16rr)
10138 CASE_ND(XOR32rr)
10139 CASE_ND(XOR64rr)
10140 CASE_ND(IMUL16rr)
10141 CASE_ND(IMUL32rr)
10142 CASE_ND(IMUL64rr)
10143 case X86::PANDrr:
10144 case X86::PORrr:
10145 case X86::PXORrr:
10146 case X86::ANDPDrr:
10147 case X86::ANDPSrr:
10148 case X86::ORPDrr:
10149 case X86::ORPSrr:
10150 case X86::XORPDrr:
10151 case X86::XORPSrr:
10152 case X86::PADDBrr:
10153 case X86::PADDWrr:
10154 case X86::PADDDrr:
10155 case X86::PADDQrr:
10156 case X86::PMULLWrr:
10157 case X86::PMULLDrr:
10158 case X86::PMAXSBrr:
10159 case X86::PMAXSDrr:
10160 case X86::PMAXSWrr:
10161 case X86::PMAXUBrr:
10162 case X86::PMAXUDrr:
10163 case X86::PMAXUWrr:
10164 case X86::PMINSBrr:
10165 case X86::PMINSDrr:
10166 case X86::PMINSWrr:
10167 case X86::PMINUBrr:
10168 case X86::PMINUDrr:
10169 case X86::PMINUWrr:
10170 case X86::VPANDrr:
10171 case X86::VPANDYrr:
10172 case X86::VPANDDZ128rr:
10173 case X86::VPANDDZ256rr:
10174 case X86::VPANDDZrr:
10175 case X86::VPANDQZ128rr:
10176 case X86::VPANDQZ256rr:
10177 case X86::VPANDQZrr:
10178 case X86::VPORrr:
10179 case X86::VPORYrr:
10180 case X86::VPORDZ128rr:
10181 case X86::VPORDZ256rr:
10182 case X86::VPORDZrr:
10183 case X86::VPORQZ128rr:
10184 case X86::VPORQZ256rr:
10185 case X86::VPORQZrr:
10186 case X86::VPXORrr:
10187 case X86::VPXORYrr:
10188 case X86::VPXORDZ128rr:
10189 case X86::VPXORDZ256rr:
10190 case X86::VPXORDZrr:
10191 case X86::VPXORQZ128rr:
10192 case X86::VPXORQZ256rr:
10193 case X86::VPXORQZrr:
10194 case X86::VANDPDrr:
10195 case X86::VANDPSrr:
10196 case X86::VANDPDYrr:
10197 case X86::VANDPSYrr:
10198 case X86::VANDPDZ128rr:
10199 case X86::VANDPSZ128rr:
10200 case X86::VANDPDZ256rr:
10201 case X86::VANDPSZ256rr:
10202 case X86::VANDPDZrr:
10203 case X86::VANDPSZrr:
10204 case X86::VORPDrr:
10205 case X86::VORPSrr:
10206 case X86::VORPDYrr:
10207 case X86::VORPSYrr:
10208 case X86::VORPDZ128rr:
10209 case X86::VORPSZ128rr:
10210 case X86::VORPDZ256rr:
10211 case X86::VORPSZ256rr:
10212 case X86::VORPDZrr:
10213 case X86::VORPSZrr:
10214 case X86::VXORPDrr:
10215 case X86::VXORPSrr:
10216 case X86::VXORPDYrr:
10217 case X86::VXORPSYrr:
10218 case X86::VXORPDZ128rr:
10219 case X86::VXORPSZ128rr:
10220 case X86::VXORPDZ256rr:
10221 case X86::VXORPSZ256rr:
10222 case X86::VXORPDZrr:
10223 case X86::VXORPSZrr:
10224 case X86::KADDBkk:
10225 case X86::KADDWkk:
10226 case X86::KADDDkk:
10227 case X86::KADDQkk:
10228 case X86::KANDBkk:
10229 case X86::KANDWkk:
10230 case X86::KANDDkk:
10231 case X86::KANDQkk:
10232 case X86::KORBkk:
10233 case X86::KORWkk:
10234 case X86::KORDkk:
10235 case X86::KORQkk:
10236 case X86::KXORBkk:
10237 case X86::KXORWkk:
10238 case X86::KXORDkk:
10239 case X86::KXORQkk:
10240 case X86::VPADDBrr:
10241 case X86::VPADDWrr:
10242 case X86::VPADDDrr:
10243 case X86::VPADDQrr:
10244 case X86::VPADDBYrr:
10245 case X86::VPADDWYrr:
10246 case X86::VPADDDYrr:
10247 case X86::VPADDQYrr:
10248 case X86::VPADDBZ128rr:
10249 case X86::VPADDWZ128rr:
10250 case X86::VPADDDZ128rr:
10251 case X86::VPADDQZ128rr:
10252 case X86::VPADDBZ256rr:
10253 case X86::VPADDWZ256rr:
10254 case X86::VPADDDZ256rr:
10255 case X86::VPADDQZ256rr:
10256 case X86::VPADDBZrr:
10257 case X86::VPADDWZrr:
10258 case X86::VPADDDZrr:
10259 case X86::VPADDQZrr:
10260 case X86::VPMULLWrr:
10261 case X86::VPMULLWYrr:
10262 case X86::VPMULLWZ128rr:
10263 case X86::VPMULLWZ256rr:
10264 case X86::VPMULLWZrr:
10265 case X86::VPMULLDrr:
10266 case X86::VPMULLDYrr:
10267 case X86::VPMULLDZ128rr:
10268 case X86::VPMULLDZ256rr:
10269 case X86::VPMULLDZrr:
10270 case X86::VPMULLQZ128rr:
10271 case X86::VPMULLQZ256rr:
10272 case X86::VPMULLQZrr:
10273 case X86::VPMAXSBrr:
10274 case X86::VPMAXSBYrr:
10275 case X86::VPMAXSBZ128rr:
10276 case X86::VPMAXSBZ256rr:
10277 case X86::VPMAXSBZrr:
10278 case X86::VPMAXSDrr:
10279 case X86::VPMAXSDYrr:
10280 case X86::VPMAXSDZ128rr:
10281 case X86::VPMAXSDZ256rr:
10282 case X86::VPMAXSDZrr:
10283 case X86::VPMAXSQZ128rr:
10284 case X86::VPMAXSQZ256rr:
10285 case X86::VPMAXSQZrr:
10286 case X86::VPMAXSWrr:
10287 case X86::VPMAXSWYrr:
10288 case X86::VPMAXSWZ128rr:
10289 case X86::VPMAXSWZ256rr:
10290 case X86::VPMAXSWZrr:
10291 case X86::VPMAXUBrr:
10292 case X86::VPMAXUBYrr:
10293 case X86::VPMAXUBZ128rr:
10294 case X86::VPMAXUBZ256rr:
10295 case X86::VPMAXUBZrr:
10296 case X86::VPMAXUDrr:
10297 case X86::VPMAXUDYrr:
10298 case X86::VPMAXUDZ128rr:
10299 case X86::VPMAXUDZ256rr:
10300 case X86::VPMAXUDZrr:
10301 case X86::VPMAXUQZ128rr:
10302 case X86::VPMAXUQZ256rr:
10303 case X86::VPMAXUQZrr:
10304 case X86::VPMAXUWrr:
10305 case X86::VPMAXUWYrr:
10306 case X86::VPMAXUWZ128rr:
10307 case X86::VPMAXUWZ256rr:
10308 case X86::VPMAXUWZrr:
10309 case X86::VPMINSBrr:
10310 case X86::VPMINSBYrr:
10311 case X86::VPMINSBZ128rr:
10312 case X86::VPMINSBZ256rr:
10313 case X86::VPMINSBZrr:
10314 case X86::VPMINSDrr:
10315 case X86::VPMINSDYrr:
10316 case X86::VPMINSDZ128rr:
10317 case X86::VPMINSDZ256rr:
10318 case X86::VPMINSDZrr:
10319 case X86::VPMINSQZ128rr:
10320 case X86::VPMINSQZ256rr:
10321 case X86::VPMINSQZrr:
10322 case X86::VPMINSWrr:
10323 case X86::VPMINSWYrr:
10324 case X86::VPMINSWZ128rr:
10325 case X86::VPMINSWZ256rr:
10326 case X86::VPMINSWZrr:
10327 case X86::VPMINUBrr:
10328 case X86::VPMINUBYrr:
10329 case X86::VPMINUBZ128rr:
10330 case X86::VPMINUBZ256rr:
10331 case X86::VPMINUBZrr:
10332 case X86::VPMINUDrr:
10333 case X86::VPMINUDYrr:
10334 case X86::VPMINUDZ128rr:
10335 case X86::VPMINUDZ256rr:
10336 case X86::VPMINUDZrr:
10337 case X86::VPMINUQZ128rr:
10338 case X86::VPMINUQZ256rr:
10339 case X86::VPMINUQZrr:
10340 case X86::VPMINUWrr:
10341 case X86::VPMINUWYrr:
10342 case X86::VPMINUWZ128rr:
10343 case X86::VPMINUWZ256rr:
10344 case X86::VPMINUWZrr:
10345 // Normal min/max instructions are not commutative because of NaN and signed
10346 // zero semantics, but these are. Thus, there's no need to check for global
10347 // relaxed math; the instructions themselves have the properties we need.
10348 case X86::MAXCPDrr:
10349 case X86::MAXCPSrr:
10350 case X86::MAXCSDrr:
10351 case X86::MAXCSSrr:
10352 case X86::MINCPDrr:
10353 case X86::MINCPSrr:
10354 case X86::MINCSDrr:
10355 case X86::MINCSSrr:
10356 case X86::VMAXCPDrr:
10357 case X86::VMAXCPSrr:
10358 case X86::VMAXCPDYrr:
10359 case X86::VMAXCPSYrr:
10360 case X86::VMAXCPDZ128rr:
10361 case X86::VMAXCPSZ128rr:
10362 case X86::VMAXCPDZ256rr:
10363 case X86::VMAXCPSZ256rr:
10364 case X86::VMAXCPDZrr:
10365 case X86::VMAXCPSZrr:
10366 case X86::VMAXCSDrr:
10367 case X86::VMAXCSSrr:
10368 case X86::VMAXCSDZrr:
10369 case X86::VMAXCSSZrr:
10370 case X86::VMINCPDrr:
10371 case X86::VMINCPSrr:
10372 case X86::VMINCPDYrr:
10373 case X86::VMINCPSYrr:
10374 case X86::VMINCPDZ128rr:
10375 case X86::VMINCPSZ128rr:
10376 case X86::VMINCPDZ256rr:
10377 case X86::VMINCPSZ256rr:
10378 case X86::VMINCPDZrr:
10379 case X86::VMINCPSZrr:
10380 case X86::VMINCSDrr:
10381 case X86::VMINCSSrr:
10382 case X86::VMINCSDZrr:
10383 case X86::VMINCSSZrr:
10384 case X86::VMAXCPHZ128rr:
10385 case X86::VMAXCPHZ256rr:
10386 case X86::VMAXCPHZrr:
10387 case X86::VMAXCSHZrr:
10388 case X86::VMINCPHZ128rr:
10389 case X86::VMINCPHZ256rr:
10390 case X86::VMINCPHZrr:
10391 case X86::VMINCSHZrr:
10392 return true;
10393 case X86::ADDPDrr:
10394 case X86::ADDPSrr:
10395 case X86::ADDSDrr:
10396 case X86::ADDSSrr:
10397 case X86::MULPDrr:
10398 case X86::MULPSrr:
10399 case X86::MULSDrr:
10400 case X86::MULSSrr:
10401 case X86::VADDPDrr:
10402 case X86::VADDPSrr:
10403 case X86::VADDPDYrr:
10404 case X86::VADDPSYrr:
10405 case X86::VADDPDZ128rr:
10406 case X86::VADDPSZ128rr:
10407 case X86::VADDPDZ256rr:
10408 case X86::VADDPSZ256rr:
10409 case X86::VADDPDZrr:
10410 case X86::VADDPSZrr:
10411 case X86::VADDSDrr:
10412 case X86::VADDSSrr:
10413 case X86::VADDSDZrr:
10414 case X86::VADDSSZrr:
10415 case X86::VMULPDrr:
10416 case X86::VMULPSrr:
10417 case X86::VMULPDYrr:
10418 case X86::VMULPSYrr:
10419 case X86::VMULPDZ128rr:
10420 case X86::VMULPSZ128rr:
10421 case X86::VMULPDZ256rr:
10422 case X86::VMULPSZ256rr:
10423 case X86::VMULPDZrr:
10424 case X86::VMULPSZrr:
10425 case X86::VMULSDrr:
10426 case X86::VMULSSrr:
10427 case X86::VMULSDZrr:
10428 case X86::VMULSSZrr:
10429 case X86::VADDPHZ128rr:
10430 case X86::VADDPHZ256rr:
10431 case X86::VADDPHZrr:
10432 case X86::VADDSHZrr:
10433 case X86::VMULPHZ128rr:
10434 case X86::VMULPHZ256rr:
10435 case X86::VMULPHZrr:
10436 case X86::VMULSHZrr:
10437 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
10438 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
10439 default:
10440 return false;
10441 }
10442}
10443
10444/// If \p DescribedReg overlaps with the MOVrr instruction's destination
10445/// register then, if possible, describe the value in terms of the source
10446/// register.
10447static std::optional<ParamLoadedValue>
10448describeMOVrrLoadedValue(const MachineInstr &MI, Register DescribedReg,
10449 const TargetRegisterInfo *TRI) {
10450 Register DestReg = MI.getOperand(i: 0).getReg();
10451 Register SrcReg = MI.getOperand(i: 1).getReg();
10452
10453 auto Expr = DIExpression::get(Context&: MI.getMF()->getFunction().getContext(), Elements: {});
10454
10455 // If the described register is the destination, just return the source.
10456 if (DestReg == DescribedReg)
10457 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
10458
10459 // If the described register is a sub-register of the destination register,
10460 // then pick out the source register's corresponding sub-register.
10461 if (unsigned SubRegIdx = TRI->getSubRegIndex(RegNo: DestReg, SubRegNo: DescribedReg)) {
10462 Register SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: SubRegIdx);
10463 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcSubReg, isDef: false), Expr);
10464 }
10465
10466 // The remaining case to consider is when the described register is a
10467 // super-register of the destination register. MOV8rr and MOV16rr does not
10468 // write to any of the other bytes in the register, meaning that we'd have to
10469 // describe the value using a combination of the source register and the
10470 // non-overlapping bits in the described register, which is not currently
10471 // possible.
10472 if (MI.getOpcode() == X86::MOV8rr || MI.getOpcode() == X86::MOV16rr ||
10473 !TRI->isSuperRegister(RegA: DestReg, RegB: DescribedReg))
10474 return std::nullopt;
10475
10476 assert(MI.getOpcode() == X86::MOV32rr && "Unexpected super-register case");
10477 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
10478}
10479
10480std::optional<ParamLoadedValue>
10481X86InstrInfo::describeLoadedValue(const MachineInstr &MI, Register Reg) const {
10482 const MachineOperand *Op = nullptr;
10483 DIExpression *Expr = nullptr;
10484
10485 const TargetRegisterInfo *TRI = &getRegisterInfo();
10486
10487 switch (MI.getOpcode()) {
10488 case X86::LEA32r:
10489 case X86::LEA64r:
10490 case X86::LEA64_32r: {
10491 // We may need to describe a 64-bit parameter with a 32-bit LEA.
10492 if (!TRI->isSuperRegisterEq(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
10493 return std::nullopt;
10494
10495 // Operand 4 could be global address. For now we do not support
10496 // such situation.
10497 if (!MI.getOperand(i: 4).isImm() || !MI.getOperand(i: 2).isImm())
10498 return std::nullopt;
10499
10500 const MachineOperand &Op1 = MI.getOperand(i: 1);
10501 const MachineOperand &Op2 = MI.getOperand(i: 3);
10502 assert(Op2.isReg() &&
10503 (Op2.getReg() == X86::NoRegister || Op2.getReg().isPhysical()));
10504
10505 // Omit situations like:
10506 // %rsi = lea %rsi, 4, ...
10507 if ((Op1.isReg() && Op1.getReg() == MI.getOperand(i: 0).getReg()) ||
10508 Op2.getReg() == MI.getOperand(i: 0).getReg())
10509 return std::nullopt;
10510 else if ((Op1.isReg() && Op1.getReg() != X86::NoRegister &&
10511 TRI->regsOverlap(RegA: Op1.getReg(), RegB: MI.getOperand(i: 0).getReg())) ||
10512 (Op2.getReg() != X86::NoRegister &&
10513 TRI->regsOverlap(RegA: Op2.getReg(), RegB: MI.getOperand(i: 0).getReg())))
10514 return std::nullopt;
10515
10516 int64_t Coef = MI.getOperand(i: 2).getImm();
10517 int64_t Offset = MI.getOperand(i: 4).getImm();
10518 SmallVector<uint64_t, 8> Ops;
10519
10520 if ((Op1.isReg() && Op1.getReg() != X86::NoRegister)) {
10521 Op = &Op1;
10522 } else if (Op1.isFI())
10523 Op = &Op1;
10524
10525 if (Op && Op->isReg() && Op->getReg() == Op2.getReg() && Coef > 0) {
10526 Ops.push_back(Elt: dwarf::DW_OP_constu);
10527 Ops.push_back(Elt: Coef + 1);
10528 Ops.push_back(Elt: dwarf::DW_OP_mul);
10529 } else {
10530 if (Op && Op2.getReg() != X86::NoRegister) {
10531 int dwarfReg = TRI->getDwarfRegNum(Reg: Op2.getReg(), isEH: false);
10532 if (dwarfReg < 0)
10533 return std::nullopt;
10534 else if (dwarfReg < 32) {
10535 Ops.push_back(Elt: dwarf::DW_OP_breg0 + dwarfReg);
10536 Ops.push_back(Elt: 0);
10537 } else {
10538 Ops.push_back(Elt: dwarf::DW_OP_bregx);
10539 Ops.push_back(Elt: dwarfReg);
10540 Ops.push_back(Elt: 0);
10541 }
10542 } else if (!Op) {
10543 assert(Op2.getReg() != X86::NoRegister);
10544 Op = &Op2;
10545 }
10546
10547 if (Coef > 1) {
10548 assert(Op2.getReg() != X86::NoRegister);
10549 Ops.push_back(Elt: dwarf::DW_OP_constu);
10550 Ops.push_back(Elt: Coef);
10551 Ops.push_back(Elt: dwarf::DW_OP_mul);
10552 }
10553
10554 if (((Op1.isReg() && Op1.getReg() != X86::NoRegister) || Op1.isFI()) &&
10555 Op2.getReg() != X86::NoRegister) {
10556 Ops.push_back(Elt: dwarf::DW_OP_plus);
10557 }
10558 }
10559
10560 DIExpression::appendOffset(Ops, Offset);
10561 Expr = DIExpression::get(Context&: MI.getMF()->getFunction().getContext(), Elements: Ops);
10562
10563 return ParamLoadedValue(*Op, Expr);
10564 }
10565 case X86::MOV8ri:
10566 case X86::MOV16ri:
10567 // TODO: Handle MOV8ri and MOV16ri.
10568 return std::nullopt;
10569 case X86::MOV32ri:
10570 case X86::MOV64ri:
10571 case X86::MOV64ri32:
10572 // MOV32ri may be used for producing zero-extended 32-bit immediates in
10573 // 64-bit parameters, so we need to consider super-registers.
10574 if (!TRI->isSuperRegisterEq(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
10575 return std::nullopt;
10576 return ParamLoadedValue(MI.getOperand(i: 1), Expr);
10577 case X86::MOV8rr:
10578 case X86::MOV16rr:
10579 case X86::MOV32rr:
10580 case X86::MOV64rr:
10581 return describeMOVrrLoadedValue(MI, DescribedReg: Reg, TRI);
10582 case X86::XOR32rr: {
10583 // 64-bit parameters are zero-materialized using XOR32rr, so also consider
10584 // super-registers.
10585 if (!TRI->isSuperRegisterEq(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
10586 return std::nullopt;
10587 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg())
10588 return ParamLoadedValue(MachineOperand::CreateImm(Val: 0), Expr);
10589 return std::nullopt;
10590 }
10591 case X86::MOVSX64rr32: {
10592 // We may need to describe the lower 32 bits of the MOVSX; for example, in
10593 // cases like this:
10594 //
10595 // $ebx = [...]
10596 // $rdi = MOVSX64rr32 $ebx
10597 // $esi = MOV32rr $edi
10598 if (!TRI->isSubRegisterEq(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
10599 return std::nullopt;
10600
10601 Expr = DIExpression::get(Context&: MI.getMF()->getFunction().getContext(), Elements: {});
10602
10603 // If the described register is the destination register we need to
10604 // sign-extend the source register from 32 bits. The other case we handle
10605 // is when the described register is the 32-bit sub-register of the
10606 // destination register, in case we just need to return the source
10607 // register.
10608 if (Reg == MI.getOperand(i: 0).getReg())
10609 Expr = DIExpression::appendExt(Expr, FromSize: 32, ToSize: 64, Signed: true);
10610 else
10611 assert(getX86MCRegisterClass(X86::GR32RegClassID).contains(Reg) &&
10612 "Unhandled sub-register case for MOVSX64rr32");
10613
10614 return ParamLoadedValue(MI.getOperand(i: 1), Expr);
10615 }
10616 default:
10617 assert(!MI.isMoveImmediate() && "Unexpected MoveImm instruction");
10618 return TargetInstrInfo::describeLoadedValue(MI, Reg);
10619 }
10620}
10621
10622/// This is an architecture-specific helper function of reassociateOps.
10623/// Set special operand attributes for new instructions after reassociation.
10624void X86InstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1,
10625 MachineInstr &OldMI2,
10626 MachineInstr &NewMI1,
10627 MachineInstr &NewMI2) const {
10628 // Integer instructions may define an implicit EFLAGS dest register operand.
10629 MachineOperand *OldFlagDef1 =
10630 OldMI1.findRegisterDefOperand(Reg: X86::EFLAGS, /*TRI=*/nullptr);
10631 MachineOperand *OldFlagDef2 =
10632 OldMI2.findRegisterDefOperand(Reg: X86::EFLAGS, /*TRI=*/nullptr);
10633
10634 assert(!OldFlagDef1 == !OldFlagDef2 &&
10635 "Unexpected instruction type for reassociation");
10636
10637 if (!OldFlagDef1 || !OldFlagDef2)
10638 return;
10639
10640 assert(OldFlagDef1->isDead() && OldFlagDef2->isDead() &&
10641 "Must have dead EFLAGS operand in reassociable instruction");
10642
10643 MachineOperand *NewFlagDef1 =
10644 NewMI1.findRegisterDefOperand(Reg: X86::EFLAGS, /*TRI=*/nullptr);
10645 MachineOperand *NewFlagDef2 =
10646 NewMI2.findRegisterDefOperand(Reg: X86::EFLAGS, /*TRI=*/nullptr);
10647
10648 assert(NewFlagDef1 && NewFlagDef2 &&
10649 "Unexpected operand in reassociable instruction");
10650
10651 // Mark the new EFLAGS operands as dead to be helpful to subsequent iterations
10652 // of this pass or other passes. The EFLAGS operands must be dead in these new
10653 // instructions because the EFLAGS operands in the original instructions must
10654 // be dead in order for reassociation to occur.
10655 NewFlagDef1->setIsDead();
10656 NewFlagDef2->setIsDead();
10657}
10658
10659std::pair<unsigned, unsigned>
10660X86InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10661 return std::make_pair(x&: TF, y: 0u);
10662}
10663
10664ArrayRef<std::pair<unsigned, const char *>>
10665X86InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10666 using namespace X86II;
10667 static const std::pair<unsigned, const char *> TargetFlags[] = {
10668 {MO_GOT_ABSOLUTE_ADDRESS, "x86-got-absolute-address"},
10669 {MO_PIC_BASE_OFFSET, "x86-pic-base-offset"},
10670 {MO_GOT, "x86-got"},
10671 {MO_GOTOFF, "x86-gotoff"},
10672 {MO_GOTPCREL, "x86-gotpcrel"},
10673 {MO_GOTPCREL_NORELAX, "x86-gotpcrel-norelax"},
10674 {MO_PLT, "x86-plt"},
10675 {MO_TLSGD, "x86-tlsgd"},
10676 {MO_TLSLD, "x86-tlsld"},
10677 {MO_TLSLDM, "x86-tlsldm"},
10678 {MO_GOTTPOFF, "x86-gottpoff"},
10679 {MO_INDNTPOFF, "x86-indntpoff"},
10680 {MO_TPOFF, "x86-tpoff"},
10681 {MO_DTPOFF, "x86-dtpoff"},
10682 {MO_NTPOFF, "x86-ntpoff"},
10683 {MO_GOTNTPOFF, "x86-gotntpoff"},
10684 {MO_DLLIMPORT, "x86-dllimport"},
10685 {MO_DARWIN_NONLAZY, "x86-darwin-nonlazy"},
10686 {MO_DARWIN_NONLAZY_PIC_BASE, "x86-darwin-nonlazy-pic-base"},
10687 {MO_TLVP, "x86-tlvp"},
10688 {MO_TLVP_PIC_BASE, "x86-tlvp-pic-base"},
10689 {MO_SECREL, "x86-secrel"},
10690 {MO_COFFSTUB, "x86-coffstub"}};
10691 return ArrayRef(TargetFlags);
10692}
10693
10694/// Constants defining how certain sequences should be outlined.
10695///
10696/// \p MachineOutlinerDefault implies that the function is called with a call
10697/// instruction, and a return must be emitted for the outlined function frame.
10698///
10699/// That is,
10700///
10701/// I1 OUTLINED_FUNCTION:
10702/// I2 --> call OUTLINED_FUNCTION I1
10703/// I3 I2
10704/// I3
10705/// ret
10706///
10707/// * Call construction overhead: 1 (call instruction)
10708/// * Frame construction overhead: 1 (return instruction)
10709///
10710/// \p MachineOutlinerTailCall implies that the function is being tail called.
10711/// A jump is emitted instead of a call, and the return is already present in
10712/// the outlined sequence. That is,
10713///
10714/// I1 OUTLINED_FUNCTION:
10715/// I2 --> jmp OUTLINED_FUNCTION I1
10716/// ret I2
10717/// ret
10718///
10719/// * Call construction overhead: 1 (jump instruction)
10720/// * Frame construction overhead: 0 (don't need to return)
10721///
10722enum MachineOutlinerClass { MachineOutlinerDefault, MachineOutlinerTailCall };
10723
10724std::optional<std::unique_ptr<outliner::OutlinedFunction>>
10725X86InstrInfo::getOutliningCandidateInfo(
10726 const MachineModuleInfo &MMI,
10727 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10728 unsigned MinRepeats) const {
10729 unsigned SequenceSize = 0;
10730 for (auto &MI : RepeatedSequenceLocs[0]) {
10731 // FIXME: x86 doesn't implement getInstSizeInBytes, so
10732 // we can't tell the cost. Just assume each instruction
10733 // is one byte.
10734 if (MI.isDebugInstr() || MI.isKill())
10735 continue;
10736 SequenceSize += 1;
10737 }
10738
10739 // We check to see if CFI Instructions are present, and if they are
10740 // we find the number of CFI Instructions in the candidates.
10741 unsigned CFICount = 0;
10742 for (auto &I : RepeatedSequenceLocs[0]) {
10743 if (I.isCFIInstruction())
10744 CFICount++;
10745 }
10746
10747 // We compare the number of found CFI Instructions to the number of CFI
10748 // instructions in the parent function for each candidate. We must check this
10749 // since if we outline one of the CFI instructions in a function, we have to
10750 // outline them all for correctness. If we do not, the address offsets will be
10751 // incorrect between the two sections of the program.
10752 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10753 std::vector<MCCFIInstruction> CFIInstructions =
10754 C.getMF()->getFrameInstructions();
10755
10756 if (CFICount > 0 && CFICount != CFIInstructions.size())
10757 return std::nullopt;
10758 }
10759
10760 // FIXME: Use real size in bytes for call and ret instructions.
10761 if (RepeatedSequenceLocs[0].back().isTerminator()) {
10762 for (outliner::Candidate &C : RepeatedSequenceLocs)
10763 C.setCallInfo(CID: MachineOutlinerTailCall, CO: 1);
10764
10765 return std::make_unique<outliner::OutlinedFunction>(
10766 args&: RepeatedSequenceLocs, args&: SequenceSize,
10767 args: 0, // Number of bytes to emit frame.
10768 args: MachineOutlinerTailCall // Type of frame.
10769 );
10770 }
10771
10772 if (CFICount > 0)
10773 return std::nullopt;
10774
10775 for (outliner::Candidate &C : RepeatedSequenceLocs)
10776 C.setCallInfo(CID: MachineOutlinerDefault, CO: 1);
10777
10778 return std::make_unique<outliner::OutlinedFunction>(
10779 args&: RepeatedSequenceLocs, args&: SequenceSize, args: 1, args: MachineOutlinerDefault);
10780}
10781
10782bool X86InstrInfo::isFunctionSafeToOutlineFrom(
10783 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
10784 const Function &F = MF.getFunction();
10785
10786 // Does the function use a red zone? If it does, then we can't risk messing
10787 // with the stack.
10788 if (Subtarget.getFrameLowering()->has128ByteRedZone(MF)) {
10789 // It could have a red zone. If it does, then we don't want to touch it.
10790 const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
10791 if (!X86FI || X86FI->getUsesRedZone())
10792 return false;
10793 }
10794
10795 // If we *don't* want to outline from things that could potentially be deduped
10796 // then return false.
10797 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
10798 return false;
10799
10800 // This function is viable for outlining, so return true.
10801 return true;
10802}
10803
10804outliner::InstrType
10805X86InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
10806 MachineBasicBlock::iterator &MIT,
10807 unsigned Flags) const {
10808 MachineInstr &MI = *MIT;
10809
10810 // Is this a terminator for a basic block?
10811 if (MI.isTerminator())
10812 // TargetInstrInfo::getOutliningType has already filtered out anything
10813 // that would break this, so we can allow it here.
10814 return outliner::InstrType::Legal;
10815
10816 // Don't outline anything that modifies or reads from the stack pointer.
10817 //
10818 // FIXME: There are instructions which are being manually built without
10819 // explicit uses/defs so we also have to check the MCInstrDesc. We should be
10820 // able to remove the extra checks once those are fixed up. For example,
10821 // sometimes we might get something like %rax = POP64r 1. This won't be
10822 // caught by modifiesRegister or readsRegister even though the instruction
10823 // really ought to be formed so that modifiesRegister/readsRegister would
10824 // catch it.
10825 if (MI.modifiesRegister(Reg: X86::RSP, TRI: &RI) || MI.readsRegister(Reg: X86::RSP, TRI: &RI) ||
10826 MI.getDesc().hasImplicitUseOfPhysReg(Reg: X86::RSP) ||
10827 MI.getDesc().hasImplicitDefOfPhysReg(Reg: X86::RSP))
10828 return outliner::InstrType::Illegal;
10829
10830 // Outlined calls change the instruction pointer, so don't read from it.
10831 if (MI.readsRegister(Reg: X86::RIP, TRI: &RI) ||
10832 MI.getDesc().hasImplicitUseOfPhysReg(Reg: X86::RIP) ||
10833 MI.getDesc().hasImplicitDefOfPhysReg(Reg: X86::RIP))
10834 return outliner::InstrType::Illegal;
10835
10836 // Don't outline CFI instructions.
10837 if (MI.isCFIInstruction())
10838 return outliner::InstrType::Illegal;
10839
10840 return outliner::InstrType::Legal;
10841}
10842
10843void X86InstrInfo::buildOutlinedFrame(
10844 MachineBasicBlock &MBB, MachineFunction &MF,
10845 const outliner::OutlinedFunction &OF) const {
10846 // If we're a tail call, we already have a return, so don't do anything.
10847 if (OF.FrameConstructionID == MachineOutlinerTailCall)
10848 return;
10849
10850 // We're a normal call, so our sequence doesn't have a return instruction.
10851 // Add it in.
10852 MachineInstr *retq = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: X86::RET64));
10853 MBB.insert(I: MBB.end(), MI: retq);
10854}
10855
10856MachineBasicBlock::iterator X86InstrInfo::insertOutlinedCall(
10857 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
10858 MachineFunction &MF, outliner::Candidate &C) const {
10859 // Is it a tail call?
10860 if (C.CallConstructionID == MachineOutlinerTailCall) {
10861 // Yes, just insert a JMP.
10862 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: X86::TAILJMPd64))
10863 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
10864 } else {
10865 // No, insert a call.
10866 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: X86::CALL64pcrel32))
10867 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
10868 }
10869
10870 return It;
10871}
10872
10873void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
10874 MachineBasicBlock::iterator Iter,
10875 DebugLoc &DL,
10876 bool AllowSideEffects) const {
10877 const MachineFunction &MF = *MBB.getParent();
10878 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
10879 const TargetRegisterInfo &TRI = getRegisterInfo();
10880
10881 if (ST.hasMMX() && X86::VR64RegClass.contains(Reg))
10882 // FIXME: Should we ignore MMX registers?
10883 return;
10884
10885 if (TRI.isGeneralPurposeRegister(MF, PhysReg: Reg)) {
10886 // Convert register to the 32-bit version. Both 'movl' and 'xorl' clear the
10887 // upper bits of a 64-bit register automagically.
10888 Reg = getX86SubSuperRegister(Reg, Size: 32);
10889
10890 if (!AllowSideEffects)
10891 // XOR affects flags, so use a MOV instead.
10892 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: X86::MOV32ri), DestReg: Reg).addImm(Val: 0);
10893 else
10894 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: X86::XOR32rr), DestReg: Reg)
10895 .addReg(RegNo: Reg, Flags: RegState::Undef)
10896 .addReg(RegNo: Reg, Flags: RegState::Undef);
10897 } else if (X86::VR128RegClass.contains(Reg)) {
10898 // XMM#
10899 if (!ST.hasSSE1())
10900 return;
10901
10902 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: X86::V_SET0), DestReg: Reg);
10903 } else if (X86::VR256RegClass.contains(Reg)) {
10904 // YMM#
10905 if (!ST.hasAVX())
10906 return;
10907
10908 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: X86::AVX_SET0), DestReg: Reg);
10909 } else if (X86::VR512RegClass.contains(Reg)) {
10910 // ZMM#
10911 if (!ST.hasAVX512())
10912 return;
10913
10914 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: X86::AVX512_512_SET0), DestReg: Reg);
10915 } else if (X86::VK1RegClass.contains(Reg) || X86::VK2RegClass.contains(Reg) ||
10916 X86::VK4RegClass.contains(Reg) || X86::VK8RegClass.contains(Reg) ||
10917 X86::VK16RegClass.contains(Reg)) {
10918 if (!ST.hasVLX())
10919 return;
10920
10921 unsigned Op = ST.hasBWI() ? X86::KSET0Q : X86::KSET0W;
10922 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: Op), DestReg: Reg);
10923 }
10924}
10925
10926bool X86InstrInfo::getMachineCombinerPatterns(
10927 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
10928 bool DoRegPressureReduce) const {
10929 unsigned Opc = Root.getOpcode();
10930 switch (Opc) {
10931 case X86::VPDPWSSDrr:
10932 case X86::VPDPWSSDrm:
10933 case X86::VPDPWSSDYrr:
10934 case X86::VPDPWSSDYrm: {
10935 if (!Subtarget.hasFastDPWSSD()) {
10936 Patterns.push_back(Elt: X86MachineCombinerPattern::DPWSSD);
10937 return true;
10938 }
10939 break;
10940 }
10941 case X86::VPDPWSSDZ128rr:
10942 case X86::VPDPWSSDZ128rm:
10943 case X86::VPDPWSSDZ256rr:
10944 case X86::VPDPWSSDZ256rm:
10945 case X86::VPDPWSSDZrr:
10946 case X86::VPDPWSSDZrm: {
10947 if (Subtarget.hasBWI() && !Subtarget.hasFastDPWSSD()) {
10948 Patterns.push_back(Elt: X86MachineCombinerPattern::DPWSSD);
10949 return true;
10950 }
10951 break;
10952 }
10953 }
10954 return TargetInstrInfo::getMachineCombinerPatterns(Root,
10955 Patterns, DoRegPressureReduce);
10956}
10957
10958static void
10959genAlternativeDpCodeSequence(MachineInstr &Root, const TargetInstrInfo &TII,
10960 SmallVectorImpl<MachineInstr *> &InsInstrs,
10961 SmallVectorImpl<MachineInstr *> &DelInstrs,
10962 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
10963 MachineFunction *MF = Root.getMF();
10964 MachineRegisterInfo &RegInfo = MF->getRegInfo();
10965
10966 unsigned Opc = Root.getOpcode();
10967 unsigned AddOpc = 0;
10968 unsigned MaddOpc = 0;
10969 switch (Opc) {
10970 default:
10971 assert(false && "It should not reach here");
10972 break;
10973 // vpdpwssd xmm2,xmm3,xmm1
10974 // -->
10975 // vpmaddwd xmm3,xmm3,xmm1
10976 // vpaddd xmm2,xmm2,xmm3
10977 case X86::VPDPWSSDrr:
10978 MaddOpc = X86::VPMADDWDrr;
10979 AddOpc = X86::VPADDDrr;
10980 break;
10981 case X86::VPDPWSSDrm:
10982 MaddOpc = X86::VPMADDWDrm;
10983 AddOpc = X86::VPADDDrr;
10984 break;
10985 case X86::VPDPWSSDZ128rr:
10986 MaddOpc = X86::VPMADDWDZ128rr;
10987 AddOpc = X86::VPADDDZ128rr;
10988 break;
10989 case X86::VPDPWSSDZ128rm:
10990 MaddOpc = X86::VPMADDWDZ128rm;
10991 AddOpc = X86::VPADDDZ128rr;
10992 break;
10993 // vpdpwssd ymm2,ymm3,ymm1
10994 // -->
10995 // vpmaddwd ymm3,ymm3,ymm1
10996 // vpaddd ymm2,ymm2,ymm3
10997 case X86::VPDPWSSDYrr:
10998 MaddOpc = X86::VPMADDWDYrr;
10999 AddOpc = X86::VPADDDYrr;
11000 break;
11001 case X86::VPDPWSSDYrm:
11002 MaddOpc = X86::VPMADDWDYrm;
11003 AddOpc = X86::VPADDDYrr;
11004 break;
11005 case X86::VPDPWSSDZ256rr:
11006 MaddOpc = X86::VPMADDWDZ256rr;
11007 AddOpc = X86::VPADDDZ256rr;
11008 break;
11009 case X86::VPDPWSSDZ256rm:
11010 MaddOpc = X86::VPMADDWDZ256rm;
11011 AddOpc = X86::VPADDDZ256rr;
11012 break;
11013 // vpdpwssd zmm2,zmm3,zmm1
11014 // -->
11015 // vpmaddwd zmm3,zmm3,zmm1
11016 // vpaddd zmm2,zmm2,zmm3
11017 case X86::VPDPWSSDZrr:
11018 MaddOpc = X86::VPMADDWDZrr;
11019 AddOpc = X86::VPADDDZrr;
11020 break;
11021 case X86::VPDPWSSDZrm:
11022 MaddOpc = X86::VPMADDWDZrm;
11023 AddOpc = X86::VPADDDZrr;
11024 break;
11025 }
11026 // Create vpmaddwd.
11027 const TargetRegisterClass *RC =
11028 RegInfo.getRegClass(Reg: Root.getOperand(i: 0).getReg());
11029 Register NewReg = RegInfo.createVirtualRegister(RegClass: RC);
11030 MachineInstr *Madd = Root.getMF()->CloneMachineInstr(Orig: &Root);
11031 Madd->setDesc(TII.get(Opcode: MaddOpc));
11032 Madd->untieRegOperand(OpIdx: 1);
11033 Madd->removeOperand(OpNo: 1);
11034 Madd->getOperand(i: 0).setReg(NewReg);
11035 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewReg, y: 0));
11036 // Create vpaddd.
11037 Register DstReg = Root.getOperand(i: 0).getReg();
11038 bool IsKill = Root.getOperand(i: 1).isKill();
11039 MachineInstr *Add =
11040 BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII.get(Opcode: AddOpc), DestReg: DstReg)
11041 .addReg(RegNo: Root.getOperand(i: 1).getReg(), Flags: getKillRegState(B: IsKill))
11042 .addReg(RegNo: Madd->getOperand(i: 0).getReg(), Flags: getKillRegState(B: true));
11043 InsInstrs.push_back(Elt: Madd);
11044 InsInstrs.push_back(Elt: Add);
11045 DelInstrs.push_back(Elt: &Root);
11046}
11047
11048void X86InstrInfo::genAlternativeCodeSequence(
11049 MachineInstr &Root, unsigned Pattern,
11050 SmallVectorImpl<MachineInstr *> &InsInstrs,
11051 SmallVectorImpl<MachineInstr *> &DelInstrs,
11052 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
11053 switch (Pattern) {
11054 default:
11055 // Reassociate instructions.
11056 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
11057 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
11058 return;
11059 case X86MachineCombinerPattern::DPWSSD:
11060 genAlternativeDpCodeSequence(Root, TII: *this, InsInstrs, DelInstrs,
11061 InstrIdxForVirtReg);
11062 return;
11063 }
11064}
11065
11066// See also: X86DAGToDAGISel::SelectInlineAsmMemoryOperand().
11067void X86InstrInfo::getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops,
11068 int FI) const {
11069 X86AddressMode M;
11070 M.BaseType = X86AddressMode::FrameIndexBase;
11071 M.Base.FrameIndex = FI;
11072 M.getFullAddress(MO&: Ops);
11073}
11074
11075MachineInstr *
11076X86InstrInfo::insertCodePrefetchInstr(MachineBasicBlock &MBB,
11077 MachineBasicBlock::iterator InsertBefore,
11078 const GlobalValue *GV) const {
11079 MachineFunction &MF = *MBB.getParent();
11080 MachineInstr *PrefetchInstr = MF.CreateMachineInstr(
11081 MCID: get(Opcode: X86::PREFETCHIT1),
11082 DL: InsertBefore == MBB.instr_end() ? MBB.findPrevDebugLoc(MBBI: InsertBefore)
11083 : InsertBefore->getDebugLoc(),
11084 NoImplicit: true);
11085 MachineInstrBuilder MIB(MF, PrefetchInstr);
11086 MIB.addMemOperand(MMO: MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(GV),
11087 F: MachineMemOperand::MOLoad, /*s=*/Size: 8,
11088 /*base_alignment=*/BaseAlignment: llvm::Align(1)));
11089 MIB.addReg(RegNo: X86::RIP).addImm(Val: 1).addReg(RegNo: X86::NoRegister);
11090 MIB.addGlobalAddress(GV);
11091 MIB.addReg(RegNo: X86::NoRegister);
11092 MBB.insert(I: InsertBefore, MI: PrefetchInstr);
11093 return PrefetchInstr;
11094}
11095
11096#define GET_INSTRINFO_HELPERS
11097#include "X86GenInstrInfo.inc"
11098