1//===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that X86 uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
15#define LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
16
17#include "X86SelectionDAGInfo.h"
18#include "llvm/CodeGen/MachineFunction.h"
19#include "llvm/CodeGen/TargetLowering.h"
20
21namespace llvm {
22 class X86Subtarget;
23 class X86TargetMachine;
24
25 namespace X86 {
26 /// Current rounding mode is represented in bits 11:10 of FPSR. These
27 /// values are same as corresponding constants for rounding mode used
28 /// in glibc.
29 enum RoundingMode {
30 rmInvalid = -1, // For handle Invalid rounding mode
31 rmToNearest = 0, // FE_TONEAREST
32 rmDownward = 1 << 10, // FE_DOWNWARD
33 rmUpward = 2 << 10, // FE_UPWARD
34 rmTowardZero = 3 << 10, // FE_TOWARDZERO
35 rmMask = 3 << 10 // Bit mask selecting rounding mode
36 };
37 }
38
39 /// Define some predicates that are used for node matching.
40 namespace X86 {
41 /// Returns true if Elt is a constant zero or floating point constant +0.0.
42 bool isZeroNode(SDValue Elt);
43
44 /// Returns true of the given offset can be
45 /// fit into displacement field of the instruction.
46 bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M,
47 bool hasSymbolicDisplacement);
48
49 /// Determines whether the callee is required to pop its
50 /// own arguments. Callee pop is necessary to support tail calls.
51 bool isCalleePop(CallingConv::ID CallingConv,
52 bool is64Bit, bool IsVarArg, bool GuaranteeTCO);
53
54 /// If Op is a constant whose elements are all the same constant or
55 /// undefined, return true and return the constant value in \p SplatVal.
56 /// If we have undef bits that don't cover an entire element, we treat these
57 /// as zero if AllowPartialUndefs is set, else we fail and return false.
58 bool isConstantSplat(SDValue Op, APInt &SplatVal,
59 bool AllowPartialUndefs = true);
60
61 /// Check if Op is a load operation that could be folded into some other x86
62 /// instruction as a memory operand. Example: vpaddd (%rdi), %xmm0, %xmm0.
63 bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
64 bool AssumeSingleUse = false,
65 bool IgnoreAlignment = false);
66
67 /// Check if Op is a load operation that could be folded into a vector splat
68 /// instruction as a memory operand. Example: vbroadcastss 16(%rdi), %xmm2.
69 bool mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
70 const X86Subtarget &Subtarget,
71 bool AssumeSingleUse = false);
72
73 /// Check if Op is a value that could be used to fold a store into some
74 /// other x86 instruction as a memory operand. Ex: pextrb $0, %xmm0, (%rdi).
75 bool mayFoldIntoStore(SDValue Op);
76
77 /// Check if Op is an operation that could be folded into a zero extend x86
78 /// instruction.
79 bool mayFoldIntoZeroExtend(SDValue Op);
80
81 /// True if the target supports the extended frame for async Swift
82 /// functions.
83 bool isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget,
84 const MachineFunction &MF);
85
86 /// Convert LLVM rounding mode to X86 rounding mode.
87 int getRoundingModeX86(unsigned RM);
88
89 } // end namespace X86
90
91 //===--------------------------------------------------------------------===//
92 // X86 Implementation of the TargetLowering interface
93 class X86TargetLowering final : public TargetLowering {
94 // Copying needed for an outgoing byval argument.
95 enum ByValCopyKind {
96 // Argument is already in the correct location, no copy needed.
97 NoCopy,
98 // Argument value is currently in the local stack frame, needs copying to
99 // outgoing arguemnt area.
100 CopyOnce,
101 // Argument value is currently in the outgoing argument area, but not at
102 // the correct offset, so needs copying via a temporary in local stack
103 // space.
104 CopyViaTemp,
105 };
106
107 public:
108 explicit X86TargetLowering(const X86TargetMachine &TM,
109 const X86Subtarget &STI);
110
111 unsigned getJumpTableEncoding() const override;
112 bool useSoftFloat() const override;
113
114 void markLibCallAttributes(MachineFunction *MF, unsigned CC,
115 ArgListTy &Args) const override;
116
117 MVT getScalarShiftAmountTy(const DataLayout &, EVT VT) const override {
118 return MVT::i8;
119 }
120
121 const MCExpr *
122 LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
123 const MachineBasicBlock *MBB, unsigned uid,
124 MCContext &Ctx) const override;
125
126 /// Returns relocation base for the given PIC jumptable.
127 SDValue getPICJumpTableRelocBase(SDValue Table,
128 SelectionDAG &DAG) const override;
129 const MCExpr *
130 getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
131 unsigned JTI, MCContext &Ctx) const override;
132
133 /// Return the desired alignment for ByVal aggregate
134 /// function arguments in the caller parameter area. For X86, aggregates
135 /// that contains are placed at 16-byte boundaries while the rest are at
136 /// 4-byte boundaries.
137 Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const override;
138
139 EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
140 const AttributeList &FuncAttributes) const override;
141
142 /// Returns true if it's safe to use load / store of the
143 /// specified type to expand memcpy / memset inline. This is mostly true
144 /// for all types except for some special cases. For example, on X86
145 /// targets without SSE2 f64 load / store are done with fldl / fstpl which
146 /// also does type conversion. Note the specified type doesn't have to be
147 /// legal as the hook is used before type legalization.
148 bool isSafeMemOpType(MVT VT) const override;
149
150 bool isMemoryAccessFast(EVT VT, Align Alignment) const;
151
152 /// Returns true if the target allows unaligned memory accesses of the
153 /// specified type. Returns whether it is "fast" in the last argument.
154 bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment,
155 MachineMemOperand::Flags Flags,
156 unsigned *Fast) const override;
157
158 /// This function returns true if the memory access is aligned or if the
159 /// target allows this specific unaligned memory access. If the access is
160 /// allowed, the optional final parameter returns a relative speed of the
161 /// access (as defined by the target).
162 bool allowsMemoryAccess(
163 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
164 Align Alignment,
165 MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
166 unsigned *Fast = nullptr) const override;
167
168 bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
169 const MachineMemOperand &MMO,
170 unsigned *Fast) const {
171 return allowsMemoryAccess(Context, DL, VT, AddrSpace: MMO.getAddrSpace(),
172 Alignment: MMO.getAlign(), Flags: MMO.getFlags(), Fast);
173 }
174
175 /// Provide custom lowering hooks for some operations.
176 ///
177 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
178
179 /// Replace the results of node with an illegal result
180 /// type with new values built out of custom code.
181 ///
182 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
183 SelectionDAG &DAG) const override;
184
185 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
186
187 bool preferABDSToABSWithNSW(EVT VT) const override;
188
189 bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT,
190 EVT ExtVT) const override;
191
192 bool isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode Cond,
193 EVT VT) const override;
194
195 /// Return true if the target has native support for
196 /// the specified value type and it is 'desirable' to use the type for the
197 /// given node type. e.g. On x86 i16 is legal, but undesirable since i16
198 /// instruction encodings are longer and some i16 instructions are slow.
199 bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override;
200 bool isTypeDesirableForOp(SDNode *N, EVT VT) const override;
201
202 /// Return true if the target has native support for the
203 /// specified value type and it is 'desirable' to use the type. e.g. On x86
204 /// i16 is legal, but undesirable since i16 instruction encodings are longer
205 /// and some i16 instructions are slow.
206 bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override;
207
208 /// Return prefered fold type, Abs if this is a vector, AddAnd if its an
209 /// integer, None otherwise.
210 TargetLowering::AndOrSETCCFoldKind
211 isDesirableToCombineLogicOpOfSETCC(const SDNode *LogicOp,
212 const SDNode *SETCC0,
213 const SDNode *SETCC1) const override;
214
215 /// Return the newly negated expression if the cost is not expensive and
216 /// set the cost in \p Cost to indicate that if it is cheaper or neutral to
217 /// do the negation.
218 SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG,
219 bool LegalOperations, bool ForCodeSize,
220 NegatibleCost &Cost,
221 unsigned Depth) const override;
222
223 MachineBasicBlock *
224 EmitInstrWithCustomInserter(MachineInstr &MI,
225 MachineBasicBlock *MBB) const override;
226
227 /// Do not merge vector stores after legalization because that may conflict
228 /// with x86-specific store splitting optimizations.
229 bool mergeStoresAfterLegalization(EVT MemVT) const override {
230 return !MemVT.isVector();
231 }
232
233 bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
234 const MachineFunction &MF) const override;
235
236 bool isCheapToSpeculateCttz(Type *Ty) const override;
237
238 bool isCheapToSpeculateCtlz(Type *Ty) const override;
239
240 bool isCtlzFast() const override;
241
242 bool preferZeroCompareBranch() const override;
243
244 bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const override {
245 // If the pair to store is a mixture of float and int values, we will
246 // save two bitwise instructions and one float-to-int instruction and
247 // increase one store instruction. There is potentially a more
248 // significant benefit because it avoids the float->int domain switch
249 // for input value. So It is more likely a win.
250 if ((LTy.isFloatingPoint() && HTy.isInteger()) ||
251 (LTy.isInteger() && HTy.isFloatingPoint()))
252 return true;
253 // If the pair only contains int values, we will save two bitwise
254 // instructions and increase one store instruction (costing one more
255 // store buffer). Since the benefit is more blurred so we leave
256 // such pair out until we get testcase to prove it is a win.
257 return false;
258 }
259
260 bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
261
262 bool hasAndNotCompare(SDValue Y) const override;
263
264 bool hasAndNot(SDValue Y) const override;
265
266 bool hasBitTest(SDValue X, SDValue Y) const override;
267
268 bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
269 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
270 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
271 SelectionDAG &DAG) const override;
272
273 unsigned preferedOpcodeForCmpEqPiecesOfOperand(
274 EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
275 const APInt &ShiftOrRotateAmt,
276 const std::optional<APInt> &AndMask) const override;
277
278 bool preferIncOfAddToSubOfNot(EVT VT) const override;
279
280 bool preferScalarizeSplat(SDNode *N) const override;
281
282 CondMergingParams
283 getJumpConditionMergingParams(Instruction::BinaryOps Opc, const Value *Lhs,
284 const Value *Rhs,
285 const Function *F) const override;
286
287 bool shouldFoldConstantShiftPairToMask(const SDNode *N) const override;
288
289 bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override;
290
291 bool
292 shouldTransformSignedTruncationCheck(EVT XVT,
293 unsigned KeptBits) const override {
294 // For vectors, we don't have a preference..
295 if (XVT.isVector())
296 return false;
297
298 auto VTIsOk = [](EVT VT) -> bool {
299 return VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
300 VT == MVT::i64;
301 };
302
303 // We are ok with KeptBitsVT being byte/word/dword, what MOVS supports.
304 // XVT will be larger than KeptBitsVT.
305 MVT KeptBitsVT = MVT::getIntegerVT(BitWidth: KeptBits);
306 return VTIsOk(XVT) && VTIsOk(KeptBitsVT);
307 }
308
309 ShiftLegalizationStrategy
310 preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N,
311 unsigned ExpansionFactor) const override;
312
313 bool shouldSplatInsEltVarIndex(EVT VT) const override;
314
315 bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override {
316 // Converting to sat variants holds little benefit on X86 as we will just
317 // need to saturate the value back using fp arithmatic.
318 return Op != ISD::FP_TO_UINT_SAT && isOperationLegalOrCustom(Op, VT);
319 }
320
321 bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
322 return VT.isScalarInteger();
323 }
324
325 /// Vector-sized comparisons are fast using PCMPEQ + PMOVMSK or PTEST.
326 MVT hasFastEqualityCompare(unsigned NumBits) const override;
327
328 /// Return the value type to use for ISD::SETCC.
329 EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
330 EVT VT) const override;
331
332 bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
333 const APInt &DemandedElts,
334 TargetLoweringOpt &TLO) const override;
335
336 /// Determine which of the bits specified in Mask are known to be either
337 /// zero or one and return them in the KnownZero/KnownOne bitsets.
338 void computeKnownBitsForTargetNode(const SDValue Op,
339 KnownBits &Known,
340 const APInt &DemandedElts,
341 const SelectionDAG &DAG,
342 unsigned Depth = 0) const override;
343
344 /// Determine the number of bits in the operation that are sign bits.
345 unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
346 const APInt &DemandedElts,
347 const SelectionDAG &DAG,
348 unsigned Depth) const override;
349
350 bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op,
351 const APInt &DemandedElts,
352 APInt &KnownUndef,
353 APInt &KnownZero,
354 TargetLoweringOpt &TLO,
355 unsigned Depth) const override;
356
357 bool SimplifyDemandedVectorEltsForTargetShuffle(SDValue Op,
358 const APInt &DemandedElts,
359 unsigned MaskIndex,
360 TargetLoweringOpt &TLO,
361 unsigned Depth) const;
362
363 unsigned getPreferredShrunkVectorSizeInBits(
364 SDValue Op, const APInt &DemandedElts) const override;
365
366 bool SimplifyDemandedBitsForTargetNode(SDValue Op,
367 const APInt &DemandedBits,
368 const APInt &DemandedElts,
369 KnownBits &Known,
370 TargetLoweringOpt &TLO,
371 unsigned Depth) const override;
372
373 SDValue SimplifyMultipleUseDemandedBitsForTargetNode(
374 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
375 SelectionDAG &DAG, unsigned Depth) const override;
376
377 bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(
378 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
379 UndefPoisonKind Kind, unsigned Depth) const override;
380
381 bool canCreateUndefOrPoisonForTargetNode(SDValue Op,
382 const APInt &DemandedElts,
383 const SelectionDAG &DAG,
384 UndefPoisonKind Kind,
385 bool ConsiderFlags,
386 unsigned Depth) const override;
387
388 bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts,
389 APInt &UndefElts, const SelectionDAG &DAG,
390 unsigned Depth) const override;
391
392 bool isTargetCanonicalConstantNode(SDValue Op) const override {
393 // Peek through bitcasts/extracts/inserts to see if we have a vector
394 // load/broadcast from memory.
395 while (Op.getOpcode() == ISD::BITCAST ||
396 Op.getOpcode() == ISD::EXTRACT_SUBVECTOR ||
397 (Op.getOpcode() == ISD::INSERT_SUBVECTOR &&
398 Op.getOperand(i: 0).isUndef()))
399 Op = Op.getOperand(i: Op.getOpcode() == ISD::INSERT_SUBVECTOR ? 1 : 0);
400
401 return Op.getOpcode() == X86ISD::VBROADCAST_LOAD ||
402 Op.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD ||
403 (Op.getOpcode() == ISD::LOAD &&
404 getTargetConstantFromLoad(LD: cast<LoadSDNode>(Val&: Op))) ||
405 TargetLowering::isTargetCanonicalConstantNode(Op);
406 }
407
408 bool isTargetCanonicalSelect(SDNode *N) const override;
409
410 const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const override;
411
412 SDValue unwrapAddress(SDValue N) const override;
413
414 SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const;
415
416 ConstraintType getConstraintType(StringRef Constraint) const override;
417
418 /// Examine constraint string and operand type and determine a weight value.
419 /// The operand object must already have been set up with the operand type.
420 ConstraintWeight
421 getSingleConstraintMatchWeight(AsmOperandInfo &Info,
422 const char *Constraint) const override;
423
424 const char *LowerXConstraint(EVT ConstraintVT) const override;
425
426 /// Lower the specified operand into the Ops vector. If it is invalid, don't
427 /// add anything to Ops. If hasMemory is true it means one of the asm
428 /// constraint of the inline asm instruction being processed is 'm'.
429 void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint,
430 std::vector<SDValue> &Ops,
431 SelectionDAG &DAG) const override;
432
433 InlineAsm::ConstraintCode
434 getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
435 if (ConstraintCode == "v")
436 return InlineAsm::ConstraintCode::v;
437 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
438 }
439
440 /// Handle Lowering flag assembly outputs.
441 SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
442 const SDLoc &DL,
443 const AsmOperandInfo &Constraint,
444 SelectionDAG &DAG) const override;
445
446 /// Given a physical register constraint
447 /// (e.g. {edx}), return the register number and the register class for the
448 /// register. This should only be used for C_Register constraints. On
449 /// error, this returns a register number of 0.
450 std::pair<unsigned, const TargetRegisterClass *>
451 getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
452 StringRef Constraint, MVT VT) const override;
453
454 /// Return true if the addressing mode represented
455 /// by AM is legal for this target, for a load/store of the specified type.
456 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
457 Type *Ty, unsigned AS,
458 Instruction *I = nullptr) const override;
459
460 bool addressingModeSupportsTLS(const GlobalValue &GV) const override;
461
462 /// Return true if the specified immediate is legal
463 /// icmp immediate, that is the target has icmp instructions which can
464 /// compare a register against the immediate without having to materialize
465 /// the immediate into a register.
466 bool isLegalICmpImmediate(int64_t Imm) const override;
467
468 /// Return true if the specified immediate is legal
469 /// add immediate, that is the target has add instructions which can
470 /// add a register and the immediate without having to materialize
471 /// the immediate into a register.
472 bool isLegalAddImmediate(int64_t Imm) const override;
473
474 bool isLegalStoreImmediate(int64_t Imm) const override;
475
476 /// Add x86-specific opcodes to the default list.
477 bool isBinOp(unsigned Opcode) const override;
478
479 /// Returns true if the opcode is a commutative binary operation.
480 bool isCommutativeBinOp(unsigned Opcode) const override;
481
482 /// Return true if it's free to truncate a value of
483 /// type Ty1 to type Ty2. e.g. On x86 it's free to truncate a i32 value in
484 /// register EAX to i16 by referencing its sub-register AX.
485 bool isTruncateFree(Type *Ty1, Type *Ty2) const override;
486 bool isTruncateFree(EVT VT1, EVT VT2) const override;
487
488 bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override;
489
490 /// Return true if any actual instruction that defines a
491 /// value of type Ty1 implicit zero-extends the value to Ty2 in the result
492 /// register. This does not necessarily include registers defined in
493 /// unknown ways, such as incoming arguments, or copies from unknown
494 /// virtual registers. Also, if isTruncateFree(Ty2, Ty1) is true, this
495 /// does not necessarily apply to truncate instructions. e.g. on x86-64,
496 /// all instructions that define 32-bit values implicit zero-extend the
497 /// result out to 64 bits.
498 bool isZExtFree(Type *Ty1, Type *Ty2) const override;
499 bool isZExtFree(EVT VT1, EVT VT2) const override;
500 bool isZExtFree(SDValue Val, EVT VT2) const override;
501
502 bool shouldConvertPhiType(Type *From, Type *To) const override;
503
504 /// Return true if folding a vector load into ExtVal (a sign, zero, or any
505 /// extend node) is profitable.
506 bool isVectorLoadExtDesirable(SDValue) const override;
507
508 /// Return true if an FMA operation is faster than a pair of fmul and fadd
509 /// instructions. fmuladd intrinsics will be expanded to FMAs when this
510 /// method returns true, otherwise fmuladd is expanded to fmul + fadd.
511 bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
512 EVT VT) const override;
513
514 /// Return true if it's profitable to narrow operations of type SrcVT to
515 /// DestVT. e.g. on x86, it's profitable to narrow from i32 to i8 but not
516 /// from i32 to i16.
517 bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const override;
518
519 bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT,
520 unsigned SelectOpcode, SDValue X,
521 SDValue Y) const override;
522
523 /// Given an intrinsic, checks if on the target the intrinsic will need to
524 /// map to a MemIntrinsicNode (touches memory). If this is the case, it
525 /// returns true and stores the intrinsic information into the IntrinsicInfo
526 /// that was passed to the function.
527 void getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
528 const CallBase &I, MachineFunction &MF,
529 unsigned Intrinsic) const override;
530
531 /// Returns how to materialize the specified FP immediate.
532 FPImmAction getFPImmAction(const APFloat &Imm, EVT VT,
533 bool ForCodeSize) const override;
534
535 /// Targets can use this to indicate that they only support *some*
536 /// VECTOR_SHUFFLE operations, those with specific masks. By default, if a
537 /// target supports the VECTOR_SHUFFLE node, all mask values are assumed to
538 /// be legal.
539 bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override;
540
541 /// Similar to isShuffleMaskLegal. Targets can use this to indicate if there
542 /// is a suitable VECTOR_SHUFFLE that can be used to replace a VAND with a
543 /// constant pool entry.
544 bool isVectorClearMaskLegal(ArrayRef<int> Mask, EVT VT) const override;
545
546 /// Returns true if lowering to a jump table is allowed.
547 bool areJTsAllowed(const Function *Fn) const override;
548
549 MVT getPreferredSwitchConditionType(LLVMContext &Context,
550 EVT ConditionVT) const override;
551
552 /// If true, then instruction selection should
553 /// seek to shrink the FP constant of the specified type to a smaller type
554 /// in order to save space and / or reduce runtime.
555 bool ShouldShrinkFPConstant(EVT VT) const override;
556
557 /// Return true if we believe it is correct and profitable to reduce the
558 /// load node to a smaller type.
559 bool
560 shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
561 std::optional<unsigned> ByteOffset) const override;
562
563 /// Return true if the specified scalar FP type is computed in an SSE
564 /// register, not on the X87 floating point stack.
565 bool isScalarFPTypeInSSEReg(EVT VT) const;
566
567 /// Returns true if it is beneficial to convert a load of a constant
568 /// to just the constant itself.
569 bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
570 Type *Ty) const override;
571
572 bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const override;
573
574 bool convertSelectOfConstantsToMath(EVT VT) const override;
575
576 bool shouldNormalizeToSelectSequence(LLVMContext &Context, EVT VT,
577 EVT CCVT) const override;
578
579 bool decomposeMulByConstant(LLVMContext &Context, EVT VT,
580 SDValue C) const override;
581
582 /// Return the cost of EXTRACT_SUBVECTOR for this result type with this
583 /// index.
584 ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
585 unsigned Index) const override;
586
587 /// Scalar ops always have equal or better analysis/performance/power than
588 /// the vector equivalent, so this always makes sense if the scalar op is
589 /// supported.
590 bool shouldScalarizeBinop(SDValue) const override;
591
592 /// Extract of a scalar FP value from index 0 of a vector is free.
593 bool isExtractVecEltCheap(EVT VT, unsigned Index) const override {
594 EVT EltVT = VT.getScalarType();
595 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
596 }
597
598 /// Overflow nodes should get combined/lowered to optimal instructions
599 /// (they should allow eliminating explicit compares by getting flags from
600 /// math ops).
601 bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
602 bool MathUsed) const override;
603
604 bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem,
605 unsigned AddrSpace) const override {
606 // If we can replace more than 2 scalar stores, there will be a reduction
607 // in instructions even after we add a vector constant load.
608 return IsZero || NumElem > 2;
609 }
610
611 bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
612 const SelectionDAG &DAG,
613 const MachineMemOperand &MMO) const override;
614
615 bool isProfitableToCombineMinNumMaxNum(EVT VT) const override {
616 // X86 has instructions that correspond to cmp + select, so forming
617 // minnum/maxnum is not profitable.
618 return false;
619 }
620
621 Register getRegisterByName(const char* RegName, LLT VT,
622 const MachineFunction &MF) const override;
623
624 /// If a physical register, this returns the register that receives the
625 /// exception address on entry to an EH pad.
626 Register
627 getExceptionPointerRegister(ExceptionHandling EH,
628 const Constant *PersonalityFn) const override;
629
630 /// If a physical register, this returns the register that receives the
631 /// exception typeid on entry to a landing pad.
632 Register
633 getExceptionSelectorRegister(ExceptionHandling EH,
634 const Constant *PersonalityFn) const override;
635
636 bool needsFixedCatchObjects() const override;
637
638 /// This method returns a target specific FastISel object,
639 /// or null if the target does not support "fast" ISel.
640 FastISel *
641 createFastISel(FunctionLoweringInfo &funcInfo,
642 const TargetLibraryInfo *libInfo,
643 const LibcallLoweringInfo *libcallLowering) const override;
644
645 /// If the target has a standard location for the stack protector cookie,
646 /// returns the address of that location. Otherwise, returns nullptr.
647 Value *getIRStackGuard(IRBuilderBase &IRB,
648 const LibcallLoweringInfo &Libcalls) const override;
649
650 bool useLoadStackGuardNode(const Module &M) const override;
651 bool useStackGuardMixFP() const override;
652 void
653 insertSSPDeclarations(Module &M,
654 const LibcallLoweringInfo &Libcalls) const override;
655 SDValue emitStackGuardMixFP(SelectionDAG &DAG, SDValue Val,
656 const SDLoc &DL) const override;
657
658 /// Return true if the target stores SafeStack pointer at a fixed offset in
659 /// some non-standard address space, and populates the address space and
660 /// offset as appropriate.
661 Value *getSafeStackPointerLocation(
662 IRBuilderBase &IRB, const LibcallLoweringInfo &Libcalls) const override;
663
664 std::pair<SDValue, SDValue> BuildFILD(EVT DstVT, EVT SrcVT, const SDLoc &DL,
665 SDValue Chain, SDValue Pointer,
666 MachinePointerInfo PtrInfo,
667 Align Alignment,
668 SelectionDAG &DAG) const;
669
670 /// Customize the preferred legalization strategy for certain types.
671 LegalizeTypeAction getPreferredVectorAction(MVT VT) const override;
672
673 MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
674 EVT VT) const override;
675
676 unsigned getNumRegistersForCallingConv(LLVMContext &Context,
677 CallingConv::ID CC,
678 EVT VT) const override;
679
680 unsigned getVectorTypeBreakdownForCallingConv(
681 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
682 unsigned &NumIntermediates, MVT &RegisterVT) const override;
683
684 bool functionArgumentNeedsConsecutiveRegisters(
685 Type *Ty, CallingConv::ID CallConv, bool isVarArg,
686 const DataLayout &DL) const override;
687
688 bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
689
690 bool supportSwiftError() const override;
691
692 bool supportKCFIBundles() const override { return true; }
693
694 MachineInstr *EmitKCFICheck(MachineBasicBlock &MBB,
695 MachineBasicBlock::instr_iterator &MBBI,
696 const TargetInstrInfo *TII) const override;
697
698 bool hasStackProbeSymbol(const MachineFunction &MF) const override;
699 bool hasInlineStackProbe(const MachineFunction &MF) const override;
700 StringRef getStackProbeSymbolName(const MachineFunction &MF) const override;
701
702 unsigned getStackProbeSize(const MachineFunction &MF) const;
703
704 bool hasVectorBlend() const override { return true; }
705
706 unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
707
708 bool isInlineAsmTargetBranch(const SmallVectorImpl<StringRef> &AsmStrs,
709 unsigned OpNo) const override;
710
711 // An x86 address can use a symbol as its displacement, which the asm
712 // printer prints for a call target, e.g. "call foo@PLT".
713 bool supportsInlineAsmFuncOperands() const override { return true; }
714
715 SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
716 MachineMemOperand *MMO, SDValue &NewLoad,
717 SDValue Ptr, SDValue PassThru,
718 SDValue Mask) const override;
719 SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
720 MachineMemOperand *MMO, SDValue Ptr, SDValue Val,
721 SDValue Mask) const override;
722
723 /// Lower interleaved load(s) into target specific
724 /// instructions/intrinsics.
725 bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
726 ArrayRef<ShuffleVectorInst *> Shuffles,
727 ArrayRef<unsigned> Indices, unsigned Factor,
728 const APInt &GapMask) const override;
729
730 /// Lower interleaved store(s) into target specific
731 /// instructions/intrinsics.
732 bool lowerInterleavedStore(Instruction *Store, Value *Mask,
733 ShuffleVectorInst *SVI, unsigned Factor,
734 const APInt &GapMask) const override;
735
736 SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr,
737 int JTI, SelectionDAG &DAG) const override;
738
739 Align
740 getPrefLoopAlignment(MachineLoop *ML,
741 const MachineBasicBlock *BlockToAlign) const override;
742
743 EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const override {
744 if (VT == MVT::f80)
745 return EVT::getIntegerVT(Context, BitWidth: 96);
746 return TargetLoweringBase::getTypeToTransformTo(Context, VT);
747 }
748
749 /// Return true if \p VT has the rsqrt* based estimate of the square root,
750 /// or of its reciprocal if \p Reciprocal is set.
751 bool hasSqrtEstimate(EVT VT, bool Reciprocal) const;
752
753 protected:
754 std::pair<const TargetRegisterClass *, uint8_t>
755 findRepresentativeClass(const TargetRegisterInfo *TRI,
756 MVT VT) const override;
757
758 private:
759 /// Keep a reference to the X86Subtarget around so that we can
760 /// make the right decision when generating code for different targets.
761 const X86Subtarget &Subtarget;
762
763 /// A list of legal FP immediates.
764 std::vector<APFloat> LegalFPImmediates;
765
766 /// Indicate that this x86 target can instruction
767 /// select the specified FP immediate natively.
768 void addLegalFPImmediate(const APFloat& Imm) {
769 LegalFPImmediates.push_back(x: Imm);
770 }
771
772 SDValue LowerCallResult(SDValue Chain, SDValue InGlue,
773 CallingConv::ID CallConv, bool isVarArg,
774 const SmallVectorImpl<ISD::InputArg> &Ins,
775 const SDLoc &dl, SelectionDAG &DAG,
776 SmallVectorImpl<SDValue> &InVals,
777 uint32_t *RegMask) const;
778 SDValue LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
779 const SmallVectorImpl<ISD::InputArg> &ArgInfo,
780 const SDLoc &dl, SelectionDAG &DAG,
781 const CCValAssign &VA, MachineFrameInfo &MFI,
782 unsigned i) const;
783 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
784 const SDLoc &dl, SelectionDAG &DAG,
785 const CCValAssign &VA,
786 ISD::ArgFlagsTy Flags, bool isByval) const;
787
788 // Call lowering helpers.
789
790 /// Check whether the call is eligible for sibling call optimization.
791 bool
792 isEligibleForSiblingCallOpt(TargetLowering::CallLoweringInfo &CLI,
793 CCState &CCInfo,
794 SmallVectorImpl<CCValAssign> &ArgLocs) const;
795 SDValue EmitTailCallLoadRetAddr(SelectionDAG &DAG, SDValue &OutRetAddr,
796 SDValue Chain, bool IsTailCall,
797 bool Is64Bit, int FPDiff,
798 const SDLoc &dl) const;
799
800 unsigned GetAlignedArgumentStackSize(unsigned StackSize,
801 SelectionDAG &DAG) const;
802
803 unsigned getAddressSpace() const;
804
805 SDValue FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned,
806 SDValue &Chain) const;
807 SDValue LRINT_LLRINTHelper(SDNode *N, SelectionDAG &DAG) const;
808
809 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
810 SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
811 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
812 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
813
814 unsigned getGlobalWrapperKind(const GlobalValue *GV,
815 const unsigned char OpFlags) const;
816 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
817 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
818 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
819 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
820 SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
821
822 /// Creates target global address or external symbol nodes for calls or
823 /// other uses.
824 SDValue LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG, bool ForCall,
825 bool *IsImpCall) const;
826
827 SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
828 SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
829 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
830 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
831 SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
832 SDValue LowerLRINT_LLRINT(SDValue Op, SelectionDAG &DAG) const;
833 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
834 SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
835 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
836 SDValue LowerConditionalBranch(SDValue Op, SelectionDAG &DAG) const;
837 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
838 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
839 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
840 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
841 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
842 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
843 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
844 SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
845 ByValCopyKind ByValNeedsCopyForTailCall(SelectionDAG &DAG, SDValue Src,
846 SDValue Dst,
847 ISD::ArgFlagsTy Flags) const;
848 SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
849 SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
850 SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
851 SDValue lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
852 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
853 SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
854 SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
855 SDValue LowerGET_FPENV_MEM(SDValue Op, SelectionDAG &DAG) const;
856 SDValue LowerSET_FPENV_MEM(SDValue Op, SelectionDAG &DAG) const;
857 SDValue LowerRESET_FPENV(SDValue Op, SelectionDAG &DAG) const;
858 SDValue LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const;
859 SDValue LowerWin64_FP_TO_INT128(SDValue Op, SelectionDAG &DAG,
860 SDValue &Chain) const;
861 SDValue LowerWin64_INT128_TO_FP(SDValue Op, SelectionDAG &DAG) const;
862 SDValue LowerGC_TRANSITION(SDValue Op, SelectionDAG &DAG) const;
863 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
864 SDValue lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const;
865 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
866 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
867 SDValue LowerBF16_TO_FP(SDValue Op, SelectionDAG &DAG) const;
868 SDValue LowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;
869
870 SDValue
871 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
872 const SmallVectorImpl<ISD::InputArg> &Ins,
873 const SDLoc &dl, SelectionDAG &DAG,
874 SmallVectorImpl<SDValue> &InVals) const override;
875 SDValue LowerCall(CallLoweringInfo &CLI,
876 SmallVectorImpl<SDValue> &InVals) const override;
877
878 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
879 const SmallVectorImpl<ISD::OutputArg> &Outs,
880 const SmallVectorImpl<SDValue> &OutVals,
881 const SDLoc &dl, SelectionDAG &DAG) const override;
882
883 bool supportSplitCSR(MachineFunction *MF) const override {
884 return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
885 MF->getFunction().hasFnAttribute(Kind: Attribute::NoUnwind);
886 }
887 void initializeSplitCSR(MachineBasicBlock *Entry) const override;
888 void insertCopiesSplitCSR(
889 MachineBasicBlock *Entry,
890 const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
891
892 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
893
894 bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
895
896 EVT getTypeForExtReturn(LLVMContext &Context, EVT VT,
897 ISD::NodeType ExtendKind) const override;
898
899 bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
900 bool isVarArg,
901 const SmallVectorImpl<ISD::OutputArg> &Outs,
902 LLVMContext &Context,
903 const Type *RetTy) const override;
904
905 const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
906 ArrayRef<MCPhysReg> getRoundingControlRegisters() const override;
907
908 TargetLoweringBase::AtomicExpansionKind
909 shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
910
911 TargetLoweringBase::AtomicExpansionKind
912 shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
913 TargetLoweringBase::AtomicExpansionKind
914 shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const override;
915 TargetLoweringBase::AtomicExpansionKind
916 shouldExpandLogicAtomicRMWInIR(const AtomicRMWInst *AI) const;
917 void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
918 void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
919 void emitExpandAtomicRMW(AtomicRMWInst *AI) const override;
920
921 LoadInst *
922 lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;
923
924 bool shouldIssueAtomicLoadForAtomicEmulationLoop() const override {
925 return false;
926 }
927
928 bool needsCmpXchgNb(unsigned OpWidth) const;
929
930 void SetupEntryBlockForSjLj(MachineInstr &MI, MachineBasicBlock *MBB,
931 MachineBasicBlock *DispatchBB, int FI) const;
932
933 // Utility function to emit the low-level va_arg code for X86-64.
934 MachineBasicBlock *
935 EmitVAARGWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const;
936
937 /// Utility function to emit the xmm reg save portion of va_start.
938 MachineBasicBlock *EmitLoweredCascadedSelect(MachineInstr &MI1,
939 MachineInstr &MI2,
940 MachineBasicBlock *BB) const;
941
942 MachineBasicBlock *EmitLoweredSelect(MachineInstr &I,
943 MachineBasicBlock *BB) const;
944
945 MachineBasicBlock *EmitLoweredCatchRet(MachineInstr &MI,
946 MachineBasicBlock *BB) const;
947
948 MachineBasicBlock *EmitLoweredSegAlloca(MachineInstr &MI,
949 MachineBasicBlock *BB) const;
950
951 MachineBasicBlock *EmitLoweredProbedAlloca(MachineInstr &MI,
952 MachineBasicBlock *BB) const;
953
954 MachineBasicBlock *EmitLoweredTLSCall(MachineInstr &MI,
955 MachineBasicBlock *BB) const;
956
957 MachineBasicBlock *EmitLoweredIndirectThunk(MachineInstr &MI,
958 MachineBasicBlock *BB) const;
959
960 MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI,
961 MachineBasicBlock *MBB) const;
962
963 void emitSetJmpShadowStackFix(MachineInstr &MI,
964 MachineBasicBlock *MBB) const;
965
966 MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr &MI,
967 MachineBasicBlock *MBB) const;
968
969 MachineBasicBlock *emitLongJmpShadowStackFix(MachineInstr &MI,
970 MachineBasicBlock *MBB) const;
971
972 MachineBasicBlock *EmitSjLjDispatchBlock(MachineInstr &MI,
973 MachineBasicBlock *MBB) const;
974
975 MachineBasicBlock *emitPatchableEventCall(MachineInstr &MI,
976 MachineBasicBlock *MBB) const;
977
978 /// Emit flags for the given setcc condition and operands. Also returns the
979 /// corresponding X86 condition code constant in X86CC.
980 SDValue emitFlagsForSetcc(SDValue Op0, SDValue Op1, ISD::CondCode CC,
981 const SDLoc &dl, SelectionDAG &DAG,
982 SDValue &X86CC) const;
983
984 bool optimizeFMulOrFDivAsShiftAddBitcast(SDNode *N, SDValue FPConst,
985 SDValue IntPow2) const override;
986
987 /// Check if replacement of SQRT with RSQRT should be disabled.
988 bool isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const override;
989
990 /// Use rsqrt* to speed up sqrt calculations.
991 SDValue getSqrtEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,
992 int &RefinementSteps, bool &UseOneConstNR,
993 bool Reciprocal) const override;
994
995 /// Use rcp* to speed up fdiv calculations.
996 SDValue getRecipEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,
997 int &RefinementSteps) const override;
998
999 /// Reassociate floating point divisions into multiply by reciprocal.
1000 unsigned combineRepeatedFPDivisors() const override;
1001
1002 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
1003 SmallVectorImpl<SDNode *> &Created) const override;
1004
1005 SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,
1006 SDValue V2) const;
1007 };
1008
1009 namespace X86 {
1010 FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
1011 const TargetLibraryInfo *libInfo,
1012 const LibcallLoweringInfo *libcallLowering);
1013 } // end namespace X86
1014
1015 // X86 specific Gather/Scatter nodes.
1016 // The class has the same order of operands as MaskedGatherScatterSDNode for
1017 // convenience.
1018 class X86MaskedGatherScatterSDNode : public MemIntrinsicSDNode {
1019 public:
1020 // This is a intended as a utility and should never be directly created.
1021 X86MaskedGatherScatterSDNode() = delete;
1022 ~X86MaskedGatherScatterSDNode() = delete;
1023
1024 const SDValue &getBasePtr() const { return getOperand(Num: 3); }
1025 const SDValue &getIndex() const { return getOperand(Num: 4); }
1026 const SDValue &getMask() const { return getOperand(Num: 2); }
1027 const SDValue &getScale() const { return getOperand(Num: 5); }
1028
1029 static bool classof(const SDNode *N) {
1030 return N->getOpcode() == X86ISD::MGATHER ||
1031 N->getOpcode() == X86ISD::MSCATTER;
1032 }
1033 };
1034
1035 class X86MaskedGatherSDNode : public X86MaskedGatherScatterSDNode {
1036 public:
1037 const SDValue &getPassThru() const { return getOperand(Num: 1); }
1038
1039 static bool classof(const SDNode *N) {
1040 return N->getOpcode() == X86ISD::MGATHER;
1041 }
1042 };
1043
1044 class X86MaskedScatterSDNode : public X86MaskedGatherScatterSDNode {
1045 public:
1046 const SDValue &getValue() const { return getOperand(Num: 1); }
1047
1048 static bool classof(const SDNode *N) {
1049 return N->getOpcode() == X86ISD::MSCATTER;
1050 }
1051 };
1052
1053 /// Generate unpacklo/unpackhi shuffle mask.
1054 void createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, bool Lo,
1055 bool Unary);
1056
1057 /// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
1058 /// imposed by AVX and specific to the unary pattern. Example:
1059 /// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
1060 /// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
1061 void createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, bool Lo);
1062
1063} // end namespace llvm
1064
1065#endif // LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
1066