1//===-- X86InstrInfo.h - X86 Instruction Information ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the X86 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_X86_X86INSTRINFO_H
14#define LLVM_LIB_TARGET_X86_X86INSTRINFO_H
15
16#include "MCTargetDesc/X86BaseInfo.h"
17#include "X86InstrFMA3Info.h"
18#include "X86RegisterInfo.h"
19#include "llvm/CodeGen/ISDOpcodes.h"
20#include "llvm/CodeGen/TargetInstrInfo.h"
21#include <vector>
22
23#define GET_INSTRINFO_HEADER
24#include "X86GenInstrInfo.inc"
25
26namespace llvm {
27class X86Subtarget;
28
29// X86 MachineCombiner patterns
30enum X86MachineCombinerPattern : unsigned {
31 // X86 VNNI
32 DPWSSD = MachineCombinerPattern::TARGET_PATTERN_START,
33};
34
35namespace X86 {
36
37enum AsmComments : MachineInstr::AsmPrinterFlagTy {
38 // For instr that was compressed from EVEX to LEGACY.
39 AC_EVEX_2_LEGACY = MachineInstr::TAsmComments,
40 // For instr that was compressed from EVEX to VEX.
41 AC_EVEX_2_VEX = AC_EVEX_2_LEGACY << 1,
42 // For instr that was compressed from EVEX to EVEX.
43 AC_EVEX_2_EVEX = AC_EVEX_2_VEX << 1
44};
45
46/// Return a pair of condition code for the given predicate and whether
47/// the instruction operands should be swaped to match the condition code.
48std::pair<CondCode, bool> getX86ConditionCode(CmpInst::Predicate Predicate);
49
50/// Return a cmov opcode for the given register size in bytes, and operand type.
51unsigned getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand = false,
52 bool HasNDD = false);
53
54/// Return a MOVri opcode for materializing \p Imm into a 32- or 64-bit GPR.
55unsigned getMOVriOpcode(bool Use64BitReg, int64_t Imm);
56
57/// Return the source operand # for condition code by \p MCID. If the
58/// instruction doesn't have a condition code, return -1.
59int getCondSrcNoFromDesc(const MCInstrDesc &MCID);
60
61/// Return the condition code of the instruction. If the instruction doesn't
62/// have a condition code, return X86::COND_INVALID.
63CondCode getCondFromMI(const MachineInstr &MI);
64
65// Turn JCC instruction into condition code.
66CondCode getCondFromBranch(const MachineInstr &MI);
67
68// Turn SETCC instruction into condition code.
69CondCode getCondFromSETCC(const MachineInstr &MI);
70
71// Turn CMOV instruction into condition code.
72CondCode getCondFromCMov(const MachineInstr &MI);
73
74// Turn CFCMOV instruction into condition code.
75CondCode getCondFromCFCMov(const MachineInstr &MI);
76
77// Turn CCMP instruction into condition code.
78CondCode getCondFromCCMP(const MachineInstr &MI);
79
80// Turn condition code into condition flags for CCMP/CTEST.
81int getCCMPCondFlagsFromCondCode(CondCode CC);
82
83// Get the opcode of corresponding NF variant.
84unsigned getNFVariant(unsigned Opc);
85
86// If \p MI clobbers EFLAGS and that clobber can be removed by rewriting the
87// instruction to its NF (no-flags) variant, return that NF opcode; otherwise
88// return 0. The clobber is removable only if the EFLAGS def is dead, an NF
89// variant exists, and (for linker backward-compat) it is not an ADDrm/ADDmr
90// with relocation.
91unsigned
92getNFVariantIfClobberRemovable(const MachineInstr &MI,
93 const TargetRegisterInfo *TRI = nullptr);
94
95// Get the opcode of corresponding NonND variant.
96unsigned getNonNDVariant(unsigned Opc);
97
98/// GetOppositeBranchCondition - Return the inverse of the specified cond,
99/// e.g. turning COND_E to COND_NE.
100CondCode GetOppositeBranchCondition(CondCode CC);
101
102/// Get the VPCMP immediate for the given condition.
103unsigned getVPCMPImmForCond(ISD::CondCode CC);
104
105/// Get the VPCMP immediate if the opcodes are swapped.
106unsigned getSwappedVPCMPImm(unsigned Imm);
107
108/// Get the VPCOM immediate if the opcodes are swapped.
109unsigned getSwappedVPCOMImm(unsigned Imm);
110
111/// Get the VCMP immediate if the opcodes are swapped.
112unsigned getSwappedVCMPImm(unsigned Imm);
113
114/// Get the width of the vector register operand.
115unsigned getVectorRegisterWidth(const MCOperandInfo &Info);
116
117/// Check if the instruction is X87 instruction.
118bool isX87Instruction(MachineInstr &MI);
119
120/// Return the index of the instruction's first address operand, if it has a
121/// memory reference, or -1 if it has none. Unlike X86II::getMemoryOperandNo(),
122/// this also works for both pseudo instructions (e.g., TCRETURNmi) as well as
123/// real instructions (e.g., JMP64m).
124int getFirstAddrOperandIdx(const MachineInstr &MI);
125
126/// Find any constant pool entry associated with a specific instruction operand.
127const Constant *getConstantFromPool(const MachineInstr &MI, unsigned OpNo);
128
129} // namespace X86
130
131/// isGlobalStubReference - Return true if the specified TargetFlag operand is
132/// a reference to a stub for a global, not the global itself.
133inline static bool isGlobalStubReference(unsigned char TargetFlag) {
134 switch (TargetFlag) {
135 case X86II::MO_DLLIMPORT: // dllimport stub.
136 case X86II::MO_GOTPCREL: // rip-relative GOT reference.
137 case X86II::MO_GOTPCREL_NORELAX: // rip-relative GOT reference.
138 case X86II::MO_GOT: // normal GOT reference.
139 case X86II::MO_DARWIN_NONLAZY_PIC_BASE: // Normal $non_lazy_ptr ref.
140 case X86II::MO_DARWIN_NONLAZY: // Normal $non_lazy_ptr ref.
141 case X86II::MO_COFFSTUB: // COFF .refptr stub.
142 return true;
143 default:
144 return false;
145 }
146}
147
148/// isGlobalRelativeToPICBase - Return true if the specified global value
149/// reference is relative to a 32-bit PIC base (X86ISD::GlobalBaseReg). If this
150/// is true, the addressing mode has the PIC base register added in (e.g. EBX).
151inline static bool isGlobalRelativeToPICBase(unsigned char TargetFlag) {
152 switch (TargetFlag) {
153 case X86II::MO_GOTOFF: // isPICStyleGOT: local global.
154 case X86II::MO_GOT: // isPICStyleGOT: other global.
155 case X86II::MO_PIC_BASE_OFFSET: // Darwin local global.
156 case X86II::MO_DARWIN_NONLAZY_PIC_BASE: // Darwin/32 external global.
157 case X86II::MO_TLVP: // ??? Pretty sure..
158 return true;
159 default:
160 return false;
161 }
162}
163
164inline static bool isScale(const MachineOperand &MO) {
165 return MO.isImm() && (MO.getImm() == 1 || MO.getImm() == 2 ||
166 MO.getImm() == 4 || MO.getImm() == 8);
167}
168
169inline static bool isLeaMem(const MachineInstr &MI, unsigned Op) {
170 if (MI.getOperand(i: Op).isFI())
171 return true;
172 return Op + X86::AddrSegmentReg <= MI.getNumOperands() &&
173 MI.getOperand(i: Op + X86::AddrBaseReg).isReg() &&
174 isScale(MO: MI.getOperand(i: Op + X86::AddrScaleAmt)) &&
175 MI.getOperand(i: Op + X86::AddrIndexReg).isReg() &&
176 (MI.getOperand(i: Op + X86::AddrDisp).isImm() ||
177 MI.getOperand(i: Op + X86::AddrDisp).isGlobal() ||
178 MI.getOperand(i: Op + X86::AddrDisp).isCPI() ||
179 MI.getOperand(i: Op + X86::AddrDisp).isJTI());
180}
181
182inline static bool isMem(const MachineInstr &MI, unsigned Op) {
183 if (MI.getOperand(i: Op).isFI())
184 return true;
185 return Op + X86::AddrNumOperands <= MI.getNumOperands() &&
186 MI.getOperand(i: Op + X86::AddrSegmentReg).isReg() && isLeaMem(MI, Op);
187}
188
189inline static bool isAddMemInstrWithRelocation(const MachineInstr &MI) {
190 unsigned Op = MI.getOpcode();
191 if (Op == X86::ADD64rm || Op == X86::ADD64mr_ND || Op == X86::ADD64rm_ND) {
192 int MemOpNo = X86II::getMemoryOperandNo(TSFlags: MI.getDesc().TSFlags) +
193 X86II::getOperandBias(Desc: MI.getDesc());
194 const MachineOperand &MO = MI.getOperand(i: X86::AddrDisp + MemOpNo);
195 if (MO.getTargetFlags() == X86II::MO_GOTTPOFF)
196 return true;
197 }
198
199 return false;
200}
201
202inline static bool isMemInstrWithGOTPCREL(const MachineInstr &MI) {
203 unsigned Op = MI.getOpcode();
204 switch (Op) {
205 case X86::TEST32mr:
206 case X86::TEST64mr:
207 case X86::CMP32rm:
208 case X86::CMP64rm:
209 case X86::MOV32rm:
210 case X86::MOV64rm:
211 case X86::ADC32rm:
212 case X86::ADD32rm:
213 case X86::AND32rm:
214 case X86::OR32rm:
215 case X86::SBB32rm:
216 case X86::SUB32rm:
217 case X86::XOR32rm:
218 case X86::ADC64rm:
219 case X86::ADD64rm:
220 case X86::AND64rm:
221 case X86::OR64rm:
222 case X86::SBB64rm:
223 case X86::SUB64rm:
224 case X86::XOR64rm: {
225 int MemOpNo = X86II::getMemoryOperandNo(TSFlags: MI.getDesc().TSFlags) +
226 X86II::getOperandBias(Desc: MI.getDesc());
227 const MachineOperand &MO = MI.getOperand(i: X86::AddrDisp + MemOpNo);
228 if (MO.getTargetFlags() == X86II::MO_GOTPCREL)
229 return true;
230 break;
231 }
232 }
233 return false;
234}
235
236class X86InstrInfo final : public X86GenInstrInfo {
237 const X86Subtarget &Subtarget;
238 const X86RegisterInfo RI;
239
240 LLVM_DECLARE_VIRTUAL_ANCHOR_FUNCTION();
241
242 bool analyzeBranchImpl(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
243 MachineBasicBlock *&FBB,
244 SmallVectorImpl<MachineOperand> &Cond,
245 SmallVectorImpl<MachineInstr *> &CondBranches,
246 bool AllowModify) const;
247
248 bool foldImmediateImpl(MachineInstr &UseMI, MachineInstr *DefMI, Register Reg,
249 int64_t ImmVal, MachineRegisterInfo *MRI,
250 bool MakeChange) const;
251
252public:
253 explicit X86InstrInfo(const X86Subtarget &STI);
254
255 /// Given a machine instruction descriptor, returns the register
256 /// class constraint for OpNum, or NULL. Returned register class
257 /// may be different from the definition in the TD file, e.g.
258 /// GR*RegClass (definition in TD file)
259 /// ->
260 /// GR*_NOREX2RegClass (Returned register class)
261 const TargetRegisterClass *getRegClass(const MCInstrDesc &MCID,
262 unsigned OpNum) const override;
263
264 /// getRegisterInfo - TargetInstrInfo is a superset of MRegister info. As
265 /// such, whenever a client has an instance of instruction info, it should
266 /// always be able to get register info as well (through this method).
267 ///
268 const X86RegisterInfo &getRegisterInfo() const { return RI; }
269
270 /// Returns the stack pointer adjustment that happens inside the frame
271 /// setup..destroy sequence (e.g. by pushes, or inside the callee).
272 int64_t getFrameAdjustment(const MachineInstr &I) const {
273 assert(isFrameInstr(I));
274 if (isFrameSetup(I))
275 return I.getOperand(i: 2).getImm();
276 return I.getOperand(i: 1).getImm();
277 }
278
279 /// Sets the stack pointer adjustment made inside the frame made up by this
280 /// instruction.
281 void setFrameAdjustment(MachineInstr &I, int64_t V) const {
282 assert(isFrameInstr(I));
283 if (isFrameSetup(I))
284 I.getOperand(i: 2).setImm(V);
285 else
286 I.getOperand(i: 1).setImm(V);
287 }
288
289 /// getSPAdjust - This returns the stack pointer adjustment made by
290 /// this instruction. For x86, we need to handle more complex call
291 /// sequences involving PUSHes.
292 int getSPAdjust(const MachineInstr &MI) const override;
293
294 /// isCoalescableExtInstr - Return true if the instruction is a "coalescable"
295 /// extension instruction. That is, it's like a copy where it's legal for the
296 /// source to overlap the destination. e.g. X86::MOVSX64rr32. If this returns
297 /// true, then it's expected the pre-extension value is available as a subreg
298 /// of the result register. This also returns the sub-register index in
299 /// SubIdx.
300 bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg,
301 Register &DstReg, unsigned &SubIdx) const override;
302
303 /// Returns true if the instruction has no behavior (specified or otherwise)
304 /// that is based on the value of any of its register operands
305 ///
306 /// Instructions are considered data invariant even if they set EFLAGS.
307 ///
308 /// A classical example of something that is inherently not data invariant is
309 /// an indirect jump -- the destination is loaded into icache based on the
310 /// bits set in the jump destination register.
311 ///
312 /// FIXME: This should become part of our instruction tables.
313 static bool isDataInvariant(MachineInstr &MI);
314
315 /// Returns true if the instruction has no behavior (specified or otherwise)
316 /// that is based on the value loaded from memory or the value of any
317 /// non-address register operands.
318 ///
319 /// For example, if the latency of the instruction is dependent on the
320 /// particular bits set in any of the registers *or* any of the bits loaded
321 /// from memory.
322 ///
323 /// Instructions are considered data invariant even if they set EFLAGS.
324 ///
325 /// A classical example of something that is inherently not data invariant is
326 /// an indirect jump -- the destination is loaded into icache based on the
327 /// bits set in the jump destination register.
328 ///
329 /// FIXME: This should become part of our instruction tables.
330 static bool isDataInvariantLoad(MachineInstr &MI);
331
332 Register isLoadFromStackSlot(const MachineInstr &MI,
333 int &FrameIndex) const override;
334 Register isLoadFromStackSlot(const MachineInstr &MI,
335 int &FrameIndex,
336 TypeSize &MemBytes) const override;
337 /// isLoadFromStackSlotPostFE - Check for post-frame ptr elimination
338 /// stack locations as well. This uses a heuristic so it isn't
339 /// reliable for correctness.
340 Register isLoadFromStackSlotPostFE(const MachineInstr &MI,
341 int &FrameIndex) const override;
342
343 Register isStoreToStackSlot(const MachineInstr &MI,
344 int &FrameIndex) const override;
345 Register isStoreToStackSlot(const MachineInstr &MI,
346 int &FrameIndex,
347 TypeSize &MemBytes) const override;
348 /// isStoreToStackSlotPostFE - Check for post-frame ptr elimination
349 /// stack locations as well. This uses a heuristic so it isn't
350 /// reliable for correctness.
351 Register isStoreToStackSlotPostFE(const MachineInstr &MI,
352 int &FrameIndex) const override;
353
354 bool isReMaterializableImpl(const MachineInstr &MI) const override;
355 void
356 reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
357 Register DestReg, unsigned SubIdx, const MachineInstr &Orig,
358 LaneBitmask UsedLanes = LaneBitmask::getAll()) const override;
359
360 /// Given an operand within a MachineInstr, insert preceding code to put it
361 /// into the right format for a particular kind of LEA instruction. This may
362 /// involve using an appropriate super-register instead (with an implicit use
363 /// of the original) or creating a new virtual register and inserting COPY
364 /// instructions to get the data into the right class.
365 ///
366 /// Reference parameters are set to indicate how caller should add this
367 /// operand to the LEA instruction.
368 bool classifyLEAReg(MachineInstr &MI, const MachineOperand &Src,
369 unsigned LEAOpcode, bool AllowSP, Register &NewSrc,
370 unsigned &NewSrcSubReg, bool &isKill,
371 MachineOperand &ImplicitOp, LiveVariables *LV,
372 LiveIntervals *LIS) const;
373
374 /// convertToThreeAddress - This method must be implemented by targets that
375 /// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target
376 /// may be able to convert a two-address instruction into a true
377 /// three-address instruction on demand. This allows the X86 target (for
378 /// example) to convert ADD and SHL instructions into LEA instructions if they
379 /// would require register copies due to two-addressness.
380 ///
381 /// This method returns a null pointer if the transformation cannot be
382 /// performed, otherwise it returns the new instruction.
383 ///
384 MachineInstr *convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,
385 LiveIntervals *LIS) const override;
386
387 /// Returns true iff the routine could find two commutable operands in the
388 /// given machine instruction.
389 /// The 'SrcOpIdx1' and 'SrcOpIdx2' are INPUT and OUTPUT arguments. Their
390 /// input values can be re-defined in this method only if the input values
391 /// are not pre-defined, which is designated by the special value
392 /// 'CommuteAnyOperandIndex' assigned to it.
393 /// If both of indices are pre-defined and refer to some operands, then the
394 /// method simply returns true if the corresponding operands are commutable
395 /// and returns false otherwise.
396 ///
397 /// For example, calling this method this way:
398 /// unsigned Op1 = 1, Op2 = CommuteAnyOperandIndex;
399 /// findCommutedOpIndices(MI, Op1, Op2);
400 /// can be interpreted as a query asking to find an operand that would be
401 /// commutable with the operand#1.
402 bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1,
403 unsigned &SrcOpIdx2) const override;
404
405 /// Returns true if we have preference on the operands order in MI, the
406 /// commute decision is returned in Commute.
407 bool hasCommutePreference(MachineInstr &MI, bool &Commute) const override;
408
409 /// Returns an adjusted FMA opcode that must be used in FMA instruction that
410 /// performs the same computations as the given \p MI but which has the
411 /// operands \p SrcOpIdx1 and \p SrcOpIdx2 commuted.
412 /// It may return 0 if it is unsafe to commute the operands.
413 /// Note that a machine instruction (instead of its opcode) is passed as the
414 /// first parameter to make it possible to analyze the instruction's uses and
415 /// commute the first operand of FMA even when it seems unsafe when you look
416 /// at the opcode. For example, it is Ok to commute the first operand of
417 /// VFMADD*SD_Int, if ONLY the lowest 64-bit element of the result is used.
418 ///
419 /// The returned FMA opcode may differ from the opcode in the given \p MI.
420 /// For example, commuting the operands #1 and #3 in the following FMA
421 /// FMA213 #1, #2, #3
422 /// results into instruction with adjusted opcode:
423 /// FMA231 #3, #2, #1
424 unsigned
425 getFMA3OpcodeToCommuteOperands(const MachineInstr &MI, unsigned SrcOpIdx1,
426 unsigned SrcOpIdx2,
427 const X86InstrFMA3Group &FMA3Group) const;
428
429 // Branch analysis.
430 bool isUnconditionalTailCall(const MachineInstr &MI) const override;
431 bool canMakeTailCallConditional(SmallVectorImpl<MachineOperand> &Cond,
432 const MachineInstr &TailCall) const override;
433 void replaceBranchWithTailCall(MachineBasicBlock &MBB,
434 SmallVectorImpl<MachineOperand> &Cond,
435 const MachineInstr &TailCall) const override;
436
437 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
438 MachineBasicBlock *&FBB,
439 SmallVectorImpl<MachineOperand> &Cond,
440 bool AllowModify) const override;
441
442 int getJumpTableIndex(const MachineInstr &MI) const override;
443
444 std::optional<ExtAddrMode>
445 getAddrModeFromMemoryOp(const MachineInstr &MemI,
446 const TargetRegisterInfo *TRI) const override;
447
448 bool getConstValDefinedInReg(const MachineInstr &MI, const Register Reg,
449 int64_t &ImmVal) const override;
450
451 bool preservesZeroValueInReg(const MachineInstr *MI,
452 const Register NullValueReg,
453 const TargetRegisterInfo *TRI) const override;
454
455 bool getMemOperandsWithOffsetWidth(
456 const MachineInstr &LdSt,
457 SmallVectorImpl<const MachineOperand *> &BaseOps, int64_t &Offset,
458 bool &OffsetIsScalable, LocationSize &Width,
459 const TargetRegisterInfo *TRI) const override;
460 bool analyzeBranchPredicate(MachineBasicBlock &MBB,
461 TargetInstrInfo::MachineBranchPredicate &MBP,
462 bool AllowModify = false) const override;
463
464 unsigned removeBranch(MachineBasicBlock &MBB,
465 int *BytesRemoved = nullptr) const override;
466 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
467 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
468 const DebugLoc &DL,
469 int *BytesAdded = nullptr) const override;
470 bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
471 Register, Register, Register, int &, int &,
472 int &) const override;
473 void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
474 const DebugLoc &DL, Register DstReg,
475 ArrayRef<MachineOperand> Cond, Register TrueReg,
476 Register FalseReg) const override;
477 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
478 const DebugLoc &DL, Register DestReg, Register SrcReg,
479 bool KillSrc, bool RenamableDest = false,
480 bool RenamableSrc = false) const override;
481 void storeRegToStackSlot(
482 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
483 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
484 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
485
486 void loadRegFromStackSlot(
487 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register DestReg,
488 int FrameIndex, const TargetRegisterClass *RC, Register VReg,
489 unsigned SubReg = 0,
490 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
491
492 void loadStoreTileReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
493 unsigned Opc, Register Reg, int FrameIdx,
494 bool isKill = false) const;
495
496 bool expandPostRAPseudo(MachineInstr &MI) const override;
497
498 /// Check whether the target can fold a load that feeds a subreg operand
499 /// (or a subreg operand that feeds a store).
500 bool isSubregFoldable() const override { return true; }
501
502 /// Fold a load or store of the specified stack slot into the specified
503 /// machine instruction for the specified operand(s). If folding happens, it
504 /// is likely that the referenced instruction has been changed.
505 ///
506 /// \returns true on success.
507 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
508 ArrayRef<unsigned> Ops, int FrameIndex,
509 MachineInstr *&CopyMI,
510 LiveIntervals *LIS = nullptr,
511 VirtRegMap *VRM = nullptr) const override;
512
513 /// Same as the previous version except it allows folding of any load and
514 /// store from / to any address, not just from a specific stack slot.
515 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
516 ArrayRef<unsigned> Ops,
517 MachineInstr &LoadMI,
518 MachineInstr *&CopyMI,
519 LiveIntervals *LIS = nullptr,
520 VirtRegMap *VRM = nullptr) const override;
521
522 bool
523 unfoldMemoryOperand(MachineFunction &MF, MachineInstr &MI, Register Reg,
524 bool UnfoldLoad, bool UnfoldStore,
525 SmallVectorImpl<MachineInstr *> &NewMIs) const override;
526
527 bool unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N,
528 SmallVectorImpl<SDNode *> &NewNodes) const override;
529
530 unsigned
531 getOpcodeAfterMemoryUnfold(unsigned Opc, bool UnfoldLoad, bool UnfoldStore,
532 unsigned *LoadRegIndex = nullptr) const override;
533
534 bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, int64_t &Offset1,
535 int64_t &Offset2) const override;
536
537 /// Overrides the isSchedulingBoundary from Codegen/TargetInstrInfo.cpp to
538 /// make it capable of identifying ENDBR intructions and prevent it from being
539 /// re-scheduled.
540 bool isSchedulingBoundary(const MachineInstr &MI,
541 const MachineBasicBlock *MBB,
542 const MachineFunction &MF) const override;
543
544 /// This is a used by the pre-regalloc scheduler to determine (in conjunction
545 /// with areLoadsFromSameBasePtr) if two loads should be scheduled togther. On
546 /// some targets if two loads are loading from addresses in the same cache
547 /// line, it's better if they are scheduled together. This function takes two
548 /// integers that represent the load offsets from the common base address. It
549 /// returns true if it decides it's desirable to schedule the two loads
550 /// together. "NumLoads" is the number of loads that have already been
551 /// scheduled after Load1.
552 bool shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, int64_t Offset1,
553 int64_t Offset2,
554 unsigned NumLoads) const override;
555
556 void insertNoop(MachineBasicBlock &MBB,
557 MachineBasicBlock::iterator MI) const override;
558
559 MCInst getNop() const override;
560
561 bool
562 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
563
564 bool isSafeToMoveRegClassDefs(const TargetRegisterClass *RC) const override;
565
566 /// True if MI has a condition code def, e.g. EFLAGS, that is
567 /// not marked dead.
568 bool hasLiveCondCodeDef(MachineInstr &MI) const;
569
570 /// getGlobalBaseReg - Return a virtual register initialized with the
571 /// the global base register value. Output instructions required to
572 /// initialize the register in the function entry block, if necessary.
573 ///
574 Register getGlobalBaseReg(MachineFunction *MF) const;
575
576 std::pair<uint16_t, uint16_t>
577 getExecutionDomain(const MachineInstr &MI) const override;
578
579 uint16_t getExecutionDomainCustom(const MachineInstr &MI) const;
580
581 void setExecutionDomain(MachineInstr &MI, unsigned Domain) const override;
582
583 bool setExecutionDomainCustom(MachineInstr &MI, unsigned Domain) const;
584
585 unsigned
586 getPartialRegUpdateClearance(const MachineInstr &MI, unsigned OpNum,
587 const TargetRegisterInfo *TRI) const override;
588 unsigned getUndefRegClearance(const MachineInstr &MI, unsigned OpNum,
589 const TargetRegisterInfo *TRI) const override;
590 void breakPartialRegDependency(MachineInstr &MI, unsigned OpNum,
591 const TargetRegisterInfo *TRI) const override;
592
593 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
594 unsigned OpNum,
595 ArrayRef<MachineOperand> MOs,
596 MachineBasicBlock::iterator InsertPt,
597 unsigned Size, Align Alignment,
598 bool AllowCommute, MachineInstr *&CopyMI,
599 VirtRegMap *VRM = nullptr) const;
600
601 bool isHighLatencyDef(int opc) const override;
602
603 bool hasHighOperandLatency(const TargetSchedModel &SchedModel,
604 const MachineRegisterInfo *MRI,
605 const MachineInstr &DefMI, unsigned DefIdx,
606 const MachineInstr &UseMI,
607 unsigned UseIdx) const override;
608
609 bool useMachineCombiner() const override { return true; }
610
611 bool isAssociativeAndCommutative(const MachineInstr &Inst,
612 bool Invert) const override;
613
614 bool hasReassociableOperands(const MachineInstr &Inst,
615 const MachineBasicBlock *MBB) const override;
616
617 void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2,
618 MachineInstr &NewMI1,
619 MachineInstr &NewMI2) const override;
620
621 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
622 Register &SrcReg2, int64_t &CmpMask,
623 int64_t &CmpValue) const override;
624
625 /// Check if there exists an earlier instruction that operates on the same
626 /// source operands and sets eflags in the same way as CMP and remove CMP if
627 /// possible.
628 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
629 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
630 const MachineRegisterInfo *MRI) const override;
631
632 bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
633 MachineRegisterInfo *MRI) const override;
634
635 std::pair<unsigned, unsigned>
636 decomposeMachineOperandsTargetFlags(unsigned TF) const override;
637
638 ArrayRef<std::pair<unsigned, const char *>>
639 getSerializableDirectMachineOperandTargetFlags() const override;
640
641 std::optional<std::unique_ptr<outliner::OutlinedFunction>>
642 getOutliningCandidateInfo(
643 const MachineModuleInfo &MMI,
644 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
645 unsigned MinRepeats) const override;
646
647 bool isFunctionSafeToOutlineFrom(MachineFunction &MF,
648 bool OutlineFromLinkOnceODRs) const override;
649
650 outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI,
651 MachineBasicBlock::iterator &MIT,
652 unsigned Flags) const override;
653
654 void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF,
655 const outliner::OutlinedFunction &OF) const override;
656
657 MachineBasicBlock::iterator
658 insertOutlinedCall(Module &M, MachineBasicBlock &MBB,
659 MachineBasicBlock::iterator &It, MachineFunction &MF,
660 outliner::Candidate &C) const override;
661
662 void buildClearRegister(Register Reg, MachineBasicBlock &MBB,
663 MachineBasicBlock::iterator Iter, DebugLoc &DL,
664 bool AllowSideEffects = true) const override;
665
666 bool verifyInstruction(const MachineInstr &MI,
667 StringRef &ErrInfo) const override;
668#define GET_INSTRINFO_HELPER_DECLS
669#include "X86GenInstrInfo.inc"
670
671 static bool hasLockPrefix(const MachineInstr &MI) {
672 return MI.getDesc().TSFlags & X86II::LOCK;
673 }
674
675 std::optional<ParamLoadedValue>
676 describeLoadedValue(const MachineInstr &MI, Register Reg) const override;
677
678protected:
679 MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
680 unsigned CommuteOpIdx1,
681 unsigned CommuteOpIdx2) const override;
682
683 std::optional<DestSourcePair>
684 isCopyInstrImpl(const MachineInstr &MI) const override;
685
686 bool getMachineCombinerPatterns(MachineInstr &Root,
687 SmallVectorImpl<unsigned> &Patterns,
688 bool DoRegPressureReduce) const override;
689
690 /// When getMachineCombinerPatterns() finds potential patterns,
691 /// this function generates the instructions that could replace the
692 /// original code sequence.
693 void genAlternativeCodeSequence(
694 MachineInstr &Root, unsigned Pattern,
695 SmallVectorImpl<MachineInstr *> &InsInstrs,
696 SmallVectorImpl<MachineInstr *> &DelInstrs,
697 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const override;
698
699 /// When calculate the latency of the root instruction, accumulate the
700 /// latency of the sequence to the root latency.
701 /// \param Root - Instruction that could be combined with one of its operands
702 /// For X86 instruction (vpmaddwd + vpmaddwd) -> vpdpwssd, the vpmaddwd
703 /// is not in the critical path, so the root latency only include vpmaddwd.
704 bool accumulateInstrSeqToRootLatency(MachineInstr &Root) const override {
705 return false;
706 }
707
708 void getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops,
709 int FI) const override;
710
711private:
712 /// This is a helper for convertToThreeAddress for 8 and 16-bit instructions.
713 /// We use 32-bit LEA to form 3-address code by promoting to a 32-bit
714 /// super-register and then truncating back down to a 8/16-bit sub-register.
715 MachineInstr *convertToThreeAddressWithLEA(unsigned MIOpc, MachineInstr &MI,
716 LiveVariables *LV,
717 LiveIntervals *LIS,
718 bool Is8BitOp) const;
719
720 /// Handles memory folding for special case instructions, for instance those
721 /// requiring custom manipulation of the address.
722 MachineInstr *foldMemoryOperandCustom(MachineFunction &MF, MachineInstr &MI,
723 unsigned OpNum,
724 ArrayRef<MachineOperand> MOs,
725 MachineBasicBlock::iterator InsertPt,
726 unsigned Size, Align Alignment) const;
727
728 MachineInstr *foldMemoryBroadcast(MachineFunction &MF, MachineInstr &MI,
729 unsigned OpNum,
730 ArrayRef<MachineOperand> MOs,
731 MachineBasicBlock::iterator InsertPt,
732 unsigned BitsSize, bool AllowCommute) const;
733
734 /// isFrameOperand - Return true and the FrameIndex if the specified
735 /// operand and follow operands form a reference to the stack frame.
736 bool isFrameOperand(const MachineInstr &MI, unsigned int Op,
737 int &FrameIndex) const;
738
739 /// Returns true iff the routine could find two commutable operands in the
740 /// given machine instruction with 3 vector inputs.
741 /// The 'SrcOpIdx1' and 'SrcOpIdx2' are INPUT and OUTPUT arguments. Their
742 /// input values can be re-defined in this method only if the input values
743 /// are not pre-defined, which is designated by the special value
744 /// 'CommuteAnyOperandIndex' assigned to it.
745 /// If both of indices are pre-defined and refer to some operands, then the
746 /// method simply returns true if the corresponding operands are commutable
747 /// and returns false otherwise.
748 ///
749 /// For example, calling this method this way:
750 /// unsigned Op1 = 1, Op2 = CommuteAnyOperandIndex;
751 /// findThreeSrcCommutedOpIndices(MI, Op1, Op2);
752 /// can be interpreted as a query asking to find an operand that would be
753 /// commutable with the operand#1.
754 ///
755 /// If IsIntrinsic is set, operand 1 will be ignored for commuting.
756 bool findThreeSrcCommutedOpIndices(const MachineInstr &MI,
757 unsigned &SrcOpIdx1,
758 unsigned &SrcOpIdx2,
759 bool IsIntrinsic = false) const;
760
761 /// Returns true when instruction \p FlagI produces the same flags as \p OI.
762 /// The caller should pass in the results of calling analyzeCompare on \p OI:
763 /// \p SrcReg, \p SrcReg2, \p ImmMask, \p ImmValue.
764 /// If the flags match \p OI as if it had the input operands swapped then the
765 /// function succeeds and sets \p IsSwapped to true.
766 ///
767 /// Examples of OI, FlagI pairs returning true:
768 /// CMP %1, 42 and CMP %1, 42
769 /// CMP %1, %2 and %3 = SUB %1, %2
770 /// TEST %1, %1 and %2 = SUB %1, 0
771 /// CMP %1, %2 and %3 = SUB %2, %1 ; IsSwapped=true
772 bool isRedundantFlagInstr(const MachineInstr &FlagI, Register SrcReg,
773 Register SrcReg2, int64_t ImmMask, int64_t ImmValue,
774 const MachineInstr &OI, bool *IsSwapped,
775 int64_t *ImmDelta) const;
776
777 /// Used by optimizeCompareInstr when the backward search for an equivalent
778 /// flag producer reaches a block (\p MultiPredMBB) with multiple
779 /// predecessors. Searches the dominator tree for an instruction that produces
780 /// the same flags as the compare \p CmpInstr and dominates it, such that on
781 /// every path from that producer to \p CmpInstr all EFLAGS clobbers can be
782 /// converted to their NF (no-flags) variants. Those clobbers are appended to
783 /// \p InstsToUpdate as (instruction, NF opcode) pairs for the caller to
784 /// rewrite. Restricted to producers that yield identical flags: it succeeds
785 /// only when isRedundantFlagInstr reports no operand swap and no immediate
786 /// delta, so on success \p IsSwapped is false and \p ImmDelta is 0. Returns
787 /// the flag producer on success, or nullptr otherwise. Requires the NF
788 /// feature (APX).
789 MachineInstr *findDominatingRedundantFlagInstr(
790 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2,
791 int64_t CmpMask, int64_t CmpValue, MachineBasicBlock *MultiPredMBB,
792 bool &IsSwapped, int64_t &ImmDelta,
793 SmallVectorImpl<std::pair<MachineInstr *, unsigned>> &InstsToUpdate)
794 const;
795
796 /// Commute operands of \p MI for memory fold.
797 ///
798 /// \param Idx1 the index of operand to be commuted.
799 ///
800 /// \returns the index of operand that is commuted with \p Idx1. If the method
801 /// fails to commute the operands, it will return \p Idx1.
802 unsigned commuteOperandsForFold(MachineInstr &MI, unsigned Idx1) const;
803
804 MachineInstr *
805 insertCodePrefetchInstr(MachineBasicBlock &MBB,
806 MachineBasicBlock::iterator InsertBefore,
807 const GlobalValue *GV) const override;
808};
809} // namespace llvm
810
811#endif
812