1//===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines a DAG pattern matching instruction selector for X86,
10// converting from a legalized dag to a X86 dag.
11//
12//===----------------------------------------------------------------------===//
13
14#include "X86.h"
15#include "X86MachineFunctionInfo.h"
16#include "X86Subtarget.h"
17#include "X86TargetMachine.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/CodeGen/MachineModuleInfo.h"
20#include "llvm/CodeGen/SelectionDAGISel.h"
21#include "llvm/Config/llvm-config.h"
22#include "llvm/IR/ConstantRange.h"
23#include "llvm/IR/Function.h"
24#include "llvm/IR/Instructions.h"
25#include "llvm/IR/Intrinsics.h"
26#include "llvm/IR/IntrinsicsX86.h"
27#include "llvm/IR/Module.h"
28#include "llvm/IR/Type.h"
29#include "llvm/Support/Debug.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Support/KnownBits.h"
32#include "llvm/Support/MathExtras.h"
33#include <cstdint>
34
35using namespace llvm;
36
37#define DEBUG_TYPE "x86-isel"
38#define PASS_NAME "X86 DAG->DAG Instruction Selection"
39
40STATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor");
41
42static cl::opt<bool> AndImmShrink("x86-and-imm-shrink", cl::init(Val: true),
43 cl::desc("Enable setting constant bits to reduce size of mask immediates"),
44 cl::Hidden);
45
46static cl::opt<bool> EnablePromoteAnyextLoad(
47 "x86-promote-anyext-load", cl::init(Val: true),
48 cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden);
49
50extern cl::opt<bool> IndirectBranchTracking;
51
52//===----------------------------------------------------------------------===//
53// Pattern Matcher Implementation
54//===----------------------------------------------------------------------===//
55
56namespace {
57 /// This corresponds to X86AddressMode, but uses SDValue's instead of register
58 /// numbers for the leaves of the matched tree.
59 struct X86ISelAddressMode {
60 enum {
61 RegBase,
62 FrameIndexBase
63 } BaseType = RegBase;
64
65 // This is really a union, discriminated by BaseType!
66 SDValue Base_Reg;
67 int Base_FrameIndex = 0;
68
69 unsigned Scale = 1;
70 SDValue IndexReg;
71 int32_t Disp = 0;
72 SDValue Segment;
73 const GlobalValue *GV = nullptr;
74 const Constant *CP = nullptr;
75 const BlockAddress *BlockAddr = nullptr;
76 const char *ES = nullptr;
77 MCSymbol *MCSym = nullptr;
78 int JT = -1;
79 Align Alignment; // CP alignment.
80 unsigned char SymbolFlags = X86II::MO_NO_FLAG; // X86II::MO_*
81 bool NegateIndex = false;
82
83 X86ISelAddressMode() = default;
84
85 bool hasSymbolicDisplacement() const {
86 return GV != nullptr || CP != nullptr || ES != nullptr ||
87 MCSym != nullptr || JT != -1 || BlockAddr != nullptr;
88 }
89
90 bool hasBaseOrIndexReg() const {
91 return BaseType == FrameIndexBase ||
92 IndexReg.getNode() != nullptr || Base_Reg.getNode() != nullptr;
93 }
94
95 /// Return true if this addressing mode is already RIP-relative.
96 bool isRIPRelative() const {
97 if (BaseType != RegBase) return false;
98 if (RegisterSDNode *RegNode =
99 dyn_cast_or_null<RegisterSDNode>(Val: Base_Reg.getNode()))
100 return RegNode->getReg() == X86::RIP;
101 return false;
102 }
103
104 void setBaseReg(SDValue Reg) {
105 BaseType = RegBase;
106 Base_Reg = Reg;
107 }
108
109#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
110 void dump(SelectionDAG *DAG = nullptr) {
111 dbgs() << "X86ISelAddressMode " << this << '\n';
112 dbgs() << "Base_Reg ";
113 if (Base_Reg.getNode())
114 Base_Reg.getNode()->dump(DAG);
115 else
116 dbgs() << "nul\n";
117 if (BaseType == FrameIndexBase)
118 dbgs() << " Base.FrameIndex " << Base_FrameIndex << '\n';
119 dbgs() << " Scale " << Scale << '\n'
120 << "IndexReg ";
121 if (NegateIndex)
122 dbgs() << "negate ";
123 if (IndexReg.getNode())
124 IndexReg.getNode()->dump(DAG);
125 else
126 dbgs() << "nul\n";
127 dbgs() << " Disp " << Disp << '\n'
128 << "GV ";
129 if (GV)
130 GV->dump();
131 else
132 dbgs() << "nul";
133 dbgs() << " CP ";
134 if (CP)
135 CP->dump();
136 else
137 dbgs() << "nul";
138 dbgs() << '\n'
139 << "ES ";
140 if (ES)
141 dbgs() << ES;
142 else
143 dbgs() << "nul";
144 dbgs() << " MCSym ";
145 if (MCSym)
146 dbgs() << MCSym;
147 else
148 dbgs() << "nul";
149 dbgs() << " JT" << JT << " Align" << Alignment.value() << '\n';
150 }
151#endif
152 };
153}
154
155namespace {
156 //===--------------------------------------------------------------------===//
157 /// ISel - X86-specific code to select X86 machine instructions for
158 /// SelectionDAG operations.
159 ///
160 class X86DAGToDAGISel final : public SelectionDAGISel {
161 /// Keep a pointer to the X86Subtarget around so that we can
162 /// make the right decision when generating code for different targets.
163 const X86Subtarget *Subtarget;
164
165 /// If true, selector should try to optimize for minimum code size.
166 bool OptForMinSize;
167
168 /// Disable direct TLS access through segment registers.
169 bool IndirectTlsSegRefs;
170
171 public:
172 X86DAGToDAGISel() = delete;
173
174 explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOptLevel OptLevel)
175 : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr),
176 OptForMinSize(false), IndirectTlsSegRefs(false) {}
177
178 bool runOnMachineFunction(MachineFunction &MF) override {
179 // Reset the subtarget each time through.
180 Subtarget = &MF.getSubtarget<X86Subtarget>();
181 IndirectTlsSegRefs = MF.getFunction().hasFnAttribute(
182 Kind: "indirect-tls-seg-refs");
183
184 // OptFor[Min]Size are used in pattern predicates that isel is matching.
185 OptForMinSize = MF.getFunction().hasMinSize();
186 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
187 }
188
189 void emitFunctionEntryCode() override;
190
191 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
192
193 void PreprocessISelDAG() override;
194 void PostprocessISelDAG() override;
195
196// Include the pieces autogenerated from the target description.
197#include "X86GenDAGISel.inc"
198
199 private:
200 void Select(SDNode *N) override;
201
202 bool foldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM);
203 bool matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM,
204 bool AllowSegmentRegForX32 = false);
205 bool matchWrapper(SDValue N, X86ISelAddressMode &AM);
206 bool matchAddress(SDValue N, X86ISelAddressMode &AM);
207 bool matchVectorAddress(SDValue N, X86ISelAddressMode &AM);
208 bool matchAdd(SDValue &N, X86ISelAddressMode &AM, unsigned Depth);
209 SDValue matchIndexRecursively(SDValue N, X86ISelAddressMode &AM,
210 unsigned Depth);
211 bool matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
212 unsigned Depth);
213 bool matchVectorAddressRecursively(SDValue N, X86ISelAddressMode &AM,
214 unsigned Depth);
215 bool matchAddressBase(SDValue N, X86ISelAddressMode &AM);
216 bool selectAddr(SDNode *Parent, SDValue N, SDValue &Base, SDValue &Scale,
217 SDValue &Index, SDValue &Disp, SDValue &Segment,
218 bool HasNDDM = true);
219 bool selectNDDAddr(SDNode *Parent, SDValue N, SDValue &Base, SDValue &Scale,
220 SDValue &Index, SDValue &Disp, SDValue &Segment);
221 bool selectVectorAddr(MemSDNode *Parent, SDValue BasePtr, SDValue IndexOp,
222 SDValue ScaleOp, SDValue &Base, SDValue &Scale,
223 SDValue &Index, SDValue &Disp, SDValue &Segment);
224 bool selectMOV64Imm32(SDValue N, SDValue &Imm);
225 bool selectLEAAddr(SDValue N, SDValue &Base,
226 SDValue &Scale, SDValue &Index, SDValue &Disp,
227 SDValue &Segment);
228 bool selectLEA64_Addr(SDValue N, SDValue &Base, SDValue &Scale,
229 SDValue &Index, SDValue &Disp, SDValue &Segment);
230 bool selectTLSADDRAddr(SDValue N, SDValue &Base,
231 SDValue &Scale, SDValue &Index, SDValue &Disp,
232 SDValue &Segment);
233 bool selectRelocImm(SDValue N, SDValue &Op);
234
235 bool tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
236 SDValue &Base, SDValue &Scale,
237 SDValue &Index, SDValue &Disp,
238 SDValue &Segment);
239
240 // Convenience method where P is also root.
241 bool tryFoldLoad(SDNode *P, SDValue N,
242 SDValue &Base, SDValue &Scale,
243 SDValue &Index, SDValue &Disp,
244 SDValue &Segment) {
245 return tryFoldLoad(Root: P, P, N, Base, Scale, Index, Disp, Segment);
246 }
247
248 bool tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
249 SDValue &Base, SDValue &Scale,
250 SDValue &Index, SDValue &Disp,
251 SDValue &Segment);
252
253 bool isProfitableToFormMaskedOp(SDNode *N) const;
254
255 /// Implement addressing mode selection for inline asm expressions.
256 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
257 InlineAsm::ConstraintCode ConstraintID,
258 std::vector<SDValue> &OutOps) override;
259
260 void emitSpecialCodeForMain();
261
262 inline void getAddressOperands(X86ISelAddressMode &AM, const SDLoc &DL,
263 MVT VT, SDValue &Base, SDValue &Scale,
264 SDValue &Index, SDValue &Disp,
265 SDValue &Segment) {
266 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
267 Base = CurDAG->getTargetFrameIndex(
268 FI: AM.Base_FrameIndex, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
269 else if (AM.Base_Reg.getNode())
270 Base = AM.Base_Reg;
271 else
272 Base = CurDAG->getRegister(Reg: 0, VT);
273
274 Scale = getI8Imm(Imm: AM.Scale, DL);
275
276#define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)
277#define GET_NDM_IF_ENABLED(OPC) \
278 (Subtarget->hasNDD() && Subtarget->hasNDDM() ? OPC##_ND : OPC)
279 // Negate the index if needed.
280 if (AM.NegateIndex) {
281 unsigned NegOpc;
282 switch (VT.SimpleTy) {
283 default:
284 llvm_unreachable("Unsupported VT!");
285 case MVT::i64:
286 NegOpc = GET_ND_IF_ENABLED(X86::NEG64r);
287 break;
288 case MVT::i32:
289 NegOpc = GET_ND_IF_ENABLED(X86::NEG32r);
290 break;
291 case MVT::i16:
292 NegOpc = GET_ND_IF_ENABLED(X86::NEG16r);
293 break;
294 case MVT::i8:
295 NegOpc = GET_ND_IF_ENABLED(X86::NEG8r);
296 break;
297 }
298 SDValue Neg = SDValue(CurDAG->getMachineNode(Opcode: NegOpc, dl: DL, VT1: VT, VT2: MVT::i32,
299 Ops: AM.IndexReg), 0);
300 AM.IndexReg = Neg;
301 }
302
303 if (AM.IndexReg.getNode())
304 Index = AM.IndexReg;
305 else
306 Index = CurDAG->getRegister(Reg: 0, VT);
307
308 // These are 32-bit even in 64-bit mode since RIP-relative offset
309 // is 32-bit.
310 if (AM.GV)
311 Disp = CurDAG->getTargetGlobalAddress(GV: AM.GV, DL: SDLoc(),
312 VT: MVT::i32, offset: AM.Disp,
313 TargetFlags: AM.SymbolFlags);
314 else if (AM.CP)
315 Disp = CurDAG->getTargetConstantPool(C: AM.CP, VT: MVT::i32, Align: AM.Alignment,
316 Offset: AM.Disp, TargetFlags: AM.SymbolFlags);
317 else if (AM.ES) {
318 assert(!AM.Disp && "Non-zero displacement is ignored with ES.");
319 Disp = CurDAG->getTargetExternalSymbol(Sym: AM.ES, VT: MVT::i32, TargetFlags: AM.SymbolFlags);
320 } else if (AM.MCSym) {
321 assert(!AM.Disp && "Non-zero displacement is ignored with MCSym.");
322 assert(AM.SymbolFlags == 0 && "oo");
323 Disp = CurDAG->getMCSymbol(Sym: AM.MCSym, VT: MVT::i32);
324 } else if (AM.JT != -1) {
325 assert(!AM.Disp && "Non-zero displacement is ignored with JT.");
326 Disp = CurDAG->getTargetJumpTable(JTI: AM.JT, VT: MVT::i32, TargetFlags: AM.SymbolFlags);
327 } else if (AM.BlockAddr)
328 Disp = CurDAG->getTargetBlockAddress(BA: AM.BlockAddr, VT: MVT::i32, Offset: AM.Disp,
329 TargetFlags: AM.SymbolFlags);
330 else
331 Disp = CurDAG->getSignedTargetConstant(Val: AM.Disp, DL, VT: MVT::i32);
332
333 if (AM.Segment.getNode())
334 Segment = AM.Segment;
335 else
336 Segment = CurDAG->getRegister(Reg: 0, VT: MVT::i16);
337 }
338
339 // Utility function to determine whether it is AMX SDNode right after
340 // lowering but before ISEL.
341 bool isAMXSDNode(SDNode *N) const {
342 // Check if N is AMX SDNode:
343 // 1. check result type;
344 // 2. check operand type;
345 for (unsigned Idx = 0, E = N->getNumValues(); Idx != E; ++Idx) {
346 if (N->getValueType(ResNo: Idx) == MVT::x86amx)
347 return true;
348 }
349 for (unsigned Idx = 0, E = N->getNumOperands(); Idx != E; ++Idx) {
350 SDValue Op = N->getOperand(Num: Idx);
351 if (Op.getValueType() == MVT::x86amx)
352 return true;
353 }
354 return false;
355 }
356
357 // Utility function to determine whether we should avoid selecting
358 // immediate forms of instructions for better code size or not.
359 // At a high level, we'd like to avoid such instructions when
360 // we have similar constants used within the same basic block
361 // that can be kept in a register.
362 //
363 bool shouldAvoidImmediateInstFormsForSize(SDNode *N) const {
364 uint32_t UseCount = 0;
365
366 // Do not want to hoist if we're not optimizing for size.
367 // TODO: We'd like to remove this restriction.
368 // See the comment in X86InstrInfo.td for more info.
369 if (!CurDAG->shouldOptForSize())
370 return false;
371
372 // Walk all the users of the immediate.
373 for (const SDNode *User : N->users()) {
374 if (UseCount >= 2)
375 break;
376
377 // This user is already selected. Count it as a legitimate use and
378 // move on.
379 if (User->isMachineOpcode()) {
380 UseCount++;
381 continue;
382 }
383
384 // We want to count stores of immediates as real uses.
385 if (User->getOpcode() == ISD::STORE &&
386 User->getOperand(Num: 1).getNode() == N) {
387 UseCount++;
388 continue;
389 }
390
391 // We don't currently match users that have > 2 operands (except
392 // for stores, which are handled above)
393 // Those instruction won't match in ISEL, for now, and would
394 // be counted incorrectly.
395 // This may change in the future as we add additional instruction
396 // types.
397 if (User->getNumOperands() != 2)
398 continue;
399
400 // If this is a sign-extended 8-bit integer immediate used in an ALU
401 // instruction, there is probably an opcode encoding to save space.
402 auto *C = dyn_cast<ConstantSDNode>(Val: N);
403 if (C && isInt<8>(x: C->getSExtValue()))
404 continue;
405
406 // Immediates that are used for offsets as part of stack
407 // manipulation should be left alone. These are typically
408 // used to indicate SP offsets for argument passing and
409 // will get pulled into stores/pushes (implicitly).
410 if (User->getOpcode() == X86ISD::ADD ||
411 User->getOpcode() == ISD::ADD ||
412 User->getOpcode() == X86ISD::SUB ||
413 User->getOpcode() == ISD::SUB) {
414
415 // Find the other operand of the add/sub.
416 SDValue OtherOp = User->getOperand(Num: 0);
417 if (OtherOp.getNode() == N)
418 OtherOp = User->getOperand(Num: 1);
419
420 // Don't count if the other operand is SP.
421 RegisterSDNode *RegNode;
422 if (OtherOp->getOpcode() == ISD::CopyFromReg &&
423 (RegNode = dyn_cast_or_null<RegisterSDNode>(
424 Val: OtherOp->getOperand(Num: 1).getNode())))
425 if ((RegNode->getReg() == X86::ESP) ||
426 (RegNode->getReg() == X86::RSP))
427 continue;
428 }
429
430 // ... otherwise, count this and move on.
431 UseCount++;
432 }
433
434 // If we have more than 1 use, then recommend for hoisting.
435 return (UseCount > 1);
436 }
437
438 /// Return a target constant with the specified value of type i8.
439 inline SDValue getI8Imm(unsigned Imm, const SDLoc &DL) {
440 return CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i8);
441 }
442
443 /// Return a target constant with the specified value, of type i32.
444 inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
445 return CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32);
446 }
447
448 /// Return a target constant with the specified value, of type i64.
449 inline SDValue getI64Imm(uint64_t Imm, const SDLoc &DL) {
450 return CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i64);
451 }
452
453 SDValue getExtractVEXTRACTImmediate(SDNode *N, unsigned VecWidth,
454 const SDLoc &DL) {
455 assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width");
456 uint64_t Index = N->getConstantOperandVal(Num: 1);
457 MVT VecVT = N->getOperand(Num: 0).getSimpleValueType();
458 return getI8Imm(Imm: (Index * VecVT.getScalarSizeInBits()) / VecWidth, DL);
459 }
460
461 SDValue getInsertVINSERTImmediate(SDNode *N, unsigned VecWidth,
462 const SDLoc &DL) {
463 assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width");
464 uint64_t Index = N->getConstantOperandVal(Num: 2);
465 MVT VecVT = N->getSimpleValueType(ResNo: 0);
466 return getI8Imm(Imm: (Index * VecVT.getScalarSizeInBits()) / VecWidth, DL);
467 }
468
469 SDValue getPermuteVINSERTCommutedImmediate(SDNode *N, unsigned VecWidth,
470 const SDLoc &DL) {
471 assert(VecWidth == 128 && "Unexpected vector width");
472 uint64_t Index = N->getConstantOperandVal(Num: 2);
473 MVT VecVT = N->getSimpleValueType(ResNo: 0);
474 uint64_t InsertIdx = (Index * VecVT.getScalarSizeInBits()) / VecWidth;
475 assert((InsertIdx == 0 || InsertIdx == 1) && "Bad insertf128 index");
476 // vinsert(0,sub,vec) -> [sub0][vec1] -> vperm2x128(0x30,vec,sub)
477 // vinsert(1,sub,vec) -> [vec0][sub0] -> vperm2x128(0x02,vec,sub)
478 return getI8Imm(Imm: InsertIdx ? 0x02 : 0x30, DL);
479 }
480
481 SDValue getSBBZero(SDNode *N) {
482 SDLoc dl(N);
483 MVT VT = N->getSimpleValueType(ResNo: 0);
484
485 // Create zero.
486 SDVTList VTs = CurDAG->getVTList(VT1: MVT::i32, VT2: MVT::i32);
487 SDValue Zero =
488 SDValue(CurDAG->getMachineNode(Opcode: X86::MOV32r0, dl, VTs, Ops: {}), 0);
489 if (VT == MVT::i64) {
490 Zero = SDValue(
491 CurDAG->getMachineNode(
492 Opcode: TargetOpcode::SUBREG_TO_REG, dl, VT: MVT::i64, Op1: Zero,
493 Op2: CurDAG->getTargetConstant(Val: X86::sub_32bit, DL: dl, VT: MVT::i32)),
494 0);
495 }
496
497 // Copy flags to the EFLAGS register and glue it to next node.
498 unsigned Opcode = N->getOpcode();
499 assert((Opcode == X86ISD::SBB || Opcode == X86ISD::SETCC_CARRY) &&
500 "Unexpected opcode for SBB materialization");
501 unsigned FlagOpIndex = Opcode == X86ISD::SBB ? 2 : 1;
502 SDValue EFLAGS =
503 CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl, Reg: X86::EFLAGS,
504 N: N->getOperand(Num: FlagOpIndex), Glue: SDValue());
505
506 // Create a 64-bit instruction if the result is 64-bits otherwise use the
507 // 32-bit version.
508 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
509 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
510 VTs = CurDAG->getVTList(VT1: SBBVT, VT2: MVT::i32);
511 return SDValue(
512 CurDAG->getMachineNode(Opcode: Opc, dl, VTs,
513 Ops: {Zero, Zero, EFLAGS, EFLAGS.getValue(R: 1)}),
514 0);
515 }
516
517 // Helper to detect unneeded and instructions on shift amounts. Called
518 // from PatFrags in tablegen.
519 bool isUnneededShiftMask(SDNode *N, unsigned Width) const {
520 assert(N->getOpcode() == ISD::AND && "Unexpected opcode");
521 const APInt &Val = N->getConstantOperandAPInt(Num: 1);
522
523 if (Val.countr_one() >= Width)
524 return true;
525
526 APInt Mask = Val | CurDAG->computeKnownBits(Op: N->getOperand(Num: 0)).Zero;
527 return Mask.countr_one() >= Width;
528 }
529
530 /// Return an SDNode that returns the value of the global base register.
531 /// Output instructions required to initialize the global base register,
532 /// if necessary.
533 SDNode *getGlobalBaseReg();
534
535 /// Return a reference to the TargetMachine, casted to the target-specific
536 /// type.
537 const X86TargetMachine &getTargetMachine() const {
538 return static_cast<const X86TargetMachine &>(TM);
539 }
540
541 /// Return a reference to the TargetInstrInfo, casted to the target-specific
542 /// type.
543 const X86InstrInfo *getInstrInfo() const {
544 return Subtarget->getInstrInfo();
545 }
546
547 /// Return a condition code of the given SDNode
548 X86::CondCode getCondFromNode(SDNode *N) const;
549
550 /// Address-mode matching performs shift-of-and to and-of-shift
551 /// reassociation in order to expose more scaled addressing
552 /// opportunities.
553 bool ComplexPatternFuncMutatesDAG() const override {
554 return true;
555 }
556
557 bool isSExtAbsoluteSymbolRef(unsigned Width, SDNode *N) const;
558
559 // Indicates we should prefer to use a non-temporal load for this load.
560 bool useNonTemporalLoad(LoadSDNode *N) const {
561 if (!N->isNonTemporal())
562 return false;
563
564 unsigned StoreSize = N->getMemoryVT().getStoreSize();
565
566 if (N->getAlign().value() < StoreSize)
567 return false;
568
569 switch (StoreSize) {
570 default: llvm_unreachable("Unsupported store size");
571 case 4:
572 case 8:
573 return false;
574 case 16:
575 return Subtarget->hasSSE41();
576 case 32:
577 return Subtarget->hasAVX2();
578 case 64:
579 return Subtarget->hasAVX512();
580 }
581 }
582
583 bool foldLoadStoreIntoMemOperand(SDNode *Node);
584 MachineSDNode *matchBEXTRFromAndImm(SDNode *Node);
585 bool matchBitExtract(SDNode *Node);
586 bool shrinkAndImmediate(SDNode *N);
587 bool isMaskZeroExtended(SDNode *N) const;
588 bool tryShiftAmountMod(SDNode *N);
589 bool tryShrinkShlLogicImm(SDNode *N);
590 bool tryVPTERNLOG(SDNode *N);
591 bool matchVPTERNLOG(SDNode *Root, SDNode *ParentA, SDNode *ParentB,
592 SDNode *ParentC, SDValue A, SDValue B, SDValue C,
593 uint8_t Imm);
594 bool tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue Mask);
595 bool tryMatchBitSelect(SDNode *N);
596
597 MachineSDNode *emitPCMPISTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad,
598 const SDLoc &dl, MVT VT, SDNode *Node);
599 MachineSDNode *emitPCMPESTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad,
600 const SDLoc &dl, MVT VT, SDNode *Node,
601 SDValue &InGlue);
602
603 bool tryOptimizeRem8Extend(SDNode *N);
604
605 bool onlyUsesZeroFlag(SDValue Flags) const;
606 bool hasNoSignFlagUses(SDValue Flags) const;
607 bool hasNoCarryFlagUses(SDValue Flags) const;
608 bool checkTCRetEnoughRegs(SDNode *N) const;
609 };
610
611 class X86DAGToDAGISelLegacy : public SelectionDAGISelLegacy {
612 public:
613 static char ID;
614 explicit X86DAGToDAGISelLegacy(X86TargetMachine &tm,
615 CodeGenOptLevel OptLevel)
616 : SelectionDAGISelLegacy(
617 ID, std::make_unique<X86DAGToDAGISel>(args&: tm, args&: OptLevel)) {}
618 };
619}
620
621char X86DAGToDAGISelLegacy::ID = 0;
622
623INITIALIZE_PASS(X86DAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
624
625// Returns true if this masked compare can be implemented legally with this
626// type.
627static bool isLegalMaskCompare(SDNode *N, const X86Subtarget *Subtarget) {
628 unsigned Opcode = N->getOpcode();
629 if (Opcode == X86ISD::CMPM || Opcode == X86ISD::CMPMM ||
630 Opcode == X86ISD::STRICT_CMPM || Opcode == ISD::SETCC ||
631 Opcode == X86ISD::CMPMM_SAE || Opcode == X86ISD::VFPCLASS) {
632 // We can get 256-bit 8 element types here without VLX being enabled. When
633 // this happens we will use 512-bit operations and the mask will not be
634 // zero extended.
635 EVT OpVT = N->getOperand(Num: 0).getValueType();
636 // The first operand of X86ISD::STRICT_CMPM is chain, so we need to get the
637 // second operand.
638 if (Opcode == X86ISD::STRICT_CMPM)
639 OpVT = N->getOperand(Num: 1).getValueType();
640 if (OpVT.is256BitVector() || OpVT.is128BitVector())
641 return Subtarget->hasVLX();
642
643 return true;
644 }
645 // Scalar opcodes use 128 bit registers, but aren't subject to the VLX check.
646 if (Opcode == X86ISD::VFPCLASSS || Opcode == X86ISD::FSETCCM ||
647 Opcode == X86ISD::FSETCCM_SAE)
648 return true;
649
650 return false;
651}
652
653// Returns true if we can assume the writer of the mask has zero extended it
654// for us.
655bool X86DAGToDAGISel::isMaskZeroExtended(SDNode *N) const {
656 // If this is an AND, check if we have a compare on either side. As long as
657 // one side guarantees the mask is zero extended, the AND will preserve those
658 // zeros.
659 if (N->getOpcode() == ISD::AND)
660 return isLegalMaskCompare(N: N->getOperand(Num: 0).getNode(), Subtarget) ||
661 isLegalMaskCompare(N: N->getOperand(Num: 1).getNode(), Subtarget);
662
663 return isLegalMaskCompare(N, Subtarget);
664}
665
666bool
667X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
668 if (OptLevel == CodeGenOptLevel::None)
669 return false;
670
671 if (!N.hasOneUse())
672 return false;
673
674 if (N.getOpcode() != ISD::LOAD)
675 return true;
676
677 // Don't fold non-temporal loads if we have an instruction for them.
678 if (useNonTemporalLoad(N: cast<LoadSDNode>(Val&: N)))
679 return false;
680
681 // If N is a load, do additional profitability checks.
682 if (U == Root) {
683 switch (U->getOpcode()) {
684 default: break;
685 case X86ISD::ADD:
686 case X86ISD::ADC:
687 case X86ISD::SUB:
688 case X86ISD::SBB:
689 case X86ISD::AND:
690 case X86ISD::XOR:
691 case X86ISD::OR:
692 case ISD::ADD:
693 case ISD::UADDO_CARRY:
694 case ISD::AND:
695 case ISD::OR:
696 case ISD::XOR: {
697 SDValue Op1 = U->getOperand(Num: 1);
698
699 // If the other operand is a 8-bit immediate we should fold the immediate
700 // instead. This reduces code size.
701 // e.g.
702 // movl 4(%esp), %eax
703 // addl $4, %eax
704 // vs.
705 // movl $4, %eax
706 // addl 4(%esp), %eax
707 // The former is 2 bytes shorter. In case where the increment is 1, then
708 // the saving can be 4 bytes (by using incl %eax).
709 if (auto *Imm = dyn_cast<ConstantSDNode>(Val&: Op1)) {
710 if (Imm->getAPIntValue().isSignedIntN(N: 8))
711 return false;
712
713 // If this is a 64-bit AND with an immediate that fits in 32-bits,
714 // prefer using the smaller and over folding the load. This is needed to
715 // make sure immediates created by shrinkAndImmediate are always folded.
716 // Ideally we would narrow the load during DAG combine and get the
717 // best of both worlds.
718 if (U->getOpcode() == ISD::AND &&
719 Imm->getAPIntValue().getBitWidth() == 64 &&
720 Imm->getAPIntValue().isIntN(N: 32))
721 return false;
722
723 // If this really a zext_inreg that can be represented with a movzx
724 // instruction, prefer that.
725 // TODO: We could shrink the load and fold if it is non-volatile.
726 if (U->getOpcode() == ISD::AND &&
727 (Imm->getAPIntValue() == UINT8_MAX ||
728 Imm->getAPIntValue() == UINT16_MAX ||
729 Imm->getAPIntValue() == UINT32_MAX))
730 return false;
731
732 // ADD/SUB with can negate the immediate and use the opposite operation
733 // to fit 128 into a sign extended 8 bit immediate.
734 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB) &&
735 (-Imm->getAPIntValue()).isSignedIntN(N: 8))
736 return false;
737
738 if ((U->getOpcode() == X86ISD::ADD || U->getOpcode() == X86ISD::SUB) &&
739 (-Imm->getAPIntValue()).isSignedIntN(N: 8) &&
740 hasNoCarryFlagUses(Flags: SDValue(U, 1)))
741 return false;
742 }
743
744 // If the other operand is a TLS address, we should fold it instead.
745 // This produces
746 // movl %gs:0, %eax
747 // leal i@NTPOFF(%eax), %eax
748 // instead of
749 // movl $i@NTPOFF, %eax
750 // addl %gs:0, %eax
751 // if the block also has an access to a second TLS address this will save
752 // a load.
753 // FIXME: This is probably also true for non-TLS addresses.
754 if (Op1.getOpcode() == X86ISD::Wrapper) {
755 SDValue Val = Op1.getOperand(i: 0);
756 if (Val.getOpcode() == ISD::TargetGlobalTLSAddress)
757 return false;
758 }
759
760 // Don't fold load if this matches the BTS/BTR/BTC patterns.
761 // BTS: (or X, (shl 1, n))
762 // BTR: (and X, (rotl -2, n))
763 // BTC: (xor X, (shl 1, n))
764 if (U->getOpcode() == ISD::OR || U->getOpcode() == ISD::XOR) {
765 if (U->getOperand(Num: 0).getOpcode() == ISD::SHL &&
766 isOneConstant(V: U->getOperand(Num: 0).getOperand(i: 0)))
767 return false;
768
769 if (U->getOperand(Num: 1).getOpcode() == ISD::SHL &&
770 isOneConstant(V: U->getOperand(Num: 1).getOperand(i: 0)))
771 return false;
772 }
773 if (U->getOpcode() == ISD::AND) {
774 SDValue U0 = U->getOperand(Num: 0);
775 SDValue U1 = U->getOperand(Num: 1);
776 if (U0.getOpcode() == ISD::ROTL) {
777 auto *C = dyn_cast<ConstantSDNode>(Val: U0.getOperand(i: 0));
778 if (C && C->getSExtValue() == -2)
779 return false;
780 }
781
782 if (U1.getOpcode() == ISD::ROTL) {
783 auto *C = dyn_cast<ConstantSDNode>(Val: U1.getOperand(i: 0));
784 if (C && C->getSExtValue() == -2)
785 return false;
786 }
787 }
788
789 break;
790 }
791 case ISD::SHL:
792 case ISD::SRA:
793 case ISD::SRL:
794 // Don't fold a load into a shift by immediate. The BMI2 instructions
795 // support folding a load, but not an immediate. The legacy instructions
796 // support folding an immediate, but can't fold a load. Folding an
797 // immediate is preferable to folding a load.
798 if (isa<ConstantSDNode>(Val: U->getOperand(Num: 1)))
799 return false;
800
801 break;
802 }
803 }
804
805 // Prevent folding a load if this can implemented with an insert_subreg or
806 // a move that implicitly zeroes.
807 if (Root->getOpcode() == ISD::INSERT_SUBVECTOR &&
808 isNullConstant(V: Root->getOperand(Num: 2)) &&
809 (Root->getOperand(Num: 0).isUndef() ||
810 ISD::isBuildVectorAllZeros(N: Root->getOperand(Num: 0).getNode())))
811 return false;
812
813 return true;
814}
815
816// Indicates it is profitable to form an AVX512 masked operation. Returning
817// false will favor a masked register-register masked move or vblendm and the
818// operation will be selected separately.
819bool X86DAGToDAGISel::isProfitableToFormMaskedOp(SDNode *N) const {
820 assert(
821 (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::SELECTS) &&
822 "Unexpected opcode!");
823
824 // If the operation has additional users, the operation will be duplicated.
825 // Check the use count to prevent that.
826 // FIXME: Are there cheap opcodes we might want to duplicate?
827 return N->getOperand(Num: 1).hasOneUse();
828}
829
830/// Replace the original chain operand of the call with
831/// load's chain operand and move load below the call's chain operand.
832static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load,
833 SDValue Call, SDValue OrigChain) {
834 SmallVector<SDValue, 8> Ops;
835 SDValue Chain = OrigChain.getOperand(i: 0);
836 if (Chain.getNode() == Load.getNode())
837 Ops.push_back(Elt: Load.getOperand(i: 0));
838 else {
839 assert(Chain.getOpcode() == ISD::TokenFactor &&
840 "Unexpected chain operand");
841 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i)
842 if (Chain.getOperand(i).getNode() == Load.getNode())
843 Ops.push_back(Elt: Load.getOperand(i: 0));
844 else
845 Ops.push_back(Elt: Chain.getOperand(i));
846 SDValue NewChain =
847 CurDAG->getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Load), VT: MVT::Other, Ops);
848 Ops.clear();
849 Ops.push_back(Elt: NewChain);
850 }
851 Ops.append(in_start: OrigChain->op_begin() + 1, in_end: OrigChain->op_end());
852 CurDAG->UpdateNodeOperands(N: OrigChain.getNode(), Ops);
853 CurDAG->UpdateNodeOperands(N: Load.getNode(), Op1: Call.getOperand(i: 0),
854 Op2: Load.getOperand(i: 1), Op3: Load.getOperand(i: 2));
855
856 Ops.clear();
857 Ops.push_back(Elt: SDValue(Load.getNode(), 1));
858 Ops.append(in_start: Call->op_begin() + 1, in_end: Call->op_end());
859 CurDAG->UpdateNodeOperands(N: Call.getNode(), Ops);
860}
861
862/// Return true if call address is a load and it can be
863/// moved below CALLSEQ_START and the chains leading up to the call.
864/// Return the CALLSEQ_START by reference as a second output.
865/// In the case of a tail call, there isn't a callseq node between the call
866/// chain and the load.
867static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) {
868 // The transformation is somewhat dangerous if the call's chain was glued to
869 // the call. After MoveBelowOrigChain the load is moved between the call and
870 // the chain, this can create a cycle if the load is not folded. So it is
871 // *really* important that we are sure the load will be folded.
872 if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse())
873 return false;
874 auto *LD = dyn_cast<LoadSDNode>(Val: Callee.getNode());
875 if (!LD ||
876 !LD->isSimple() ||
877 LD->getAddressingMode() != ISD::UNINDEXED ||
878 LD->getExtensionType() != ISD::NON_EXTLOAD)
879 return false;
880
881 // If the load's outgoing chain has more than one use, we can't (currently)
882 // move the load since we'd most likely create a loop. TODO: Maybe it could
883 // work if moveBelowOrigChain() updated *all* the chain users.
884 if (!Callee.getValue(R: 1).hasOneUse())
885 return false;
886
887 // Now let's find the callseq_start.
888 while (HasCallSeq && Chain.getOpcode() != ISD::CALLSEQ_START) {
889 if (!Chain.hasOneUse())
890 return false;
891 Chain = Chain.getOperand(i: 0);
892 }
893
894 while (true) {
895 if (!Chain.getNumOperands())
896 return false;
897
898 // It's not safe to move the callee (a load) across e.g. a store.
899 // Conservatively abort if the chain contains a node other than the ones
900 // below.
901 switch (Chain.getNode()->getOpcode()) {
902 case ISD::CALLSEQ_START:
903 case ISD::CopyToReg:
904 case ISD::LOAD:
905 break;
906 default:
907 return false;
908 }
909
910 if (Chain.getOperand(i: 0).getNode() == Callee.getNode())
911 return true;
912 if (Chain.getOperand(i: 0).getOpcode() == ISD::TokenFactor &&
913 Chain.getOperand(i: 0).getValue(R: 0).hasOneUse() &&
914 Callee.getValue(R: 1).isOperandOf(N: Chain.getOperand(i: 0).getNode()) &&
915 Callee.getValue(R: 1).hasOneUse())
916 return true;
917
918 // Look past CopyToRegs. We only walk one path, so the chain mustn't branch.
919 if (Chain.getOperand(i: 0).getOpcode() == ISD::CopyToReg &&
920 Chain.getOperand(i: 0).getValue(R: 0).hasOneUse()) {
921 Chain = Chain.getOperand(i: 0);
922 continue;
923 }
924
925 return false;
926 }
927}
928
929static bool isEndbrImm64(uint64_t Imm) {
930// There may be some other prefix bytes between 0xF3 and 0x0F1EFA.
931// i.g: 0xF3660F1EFA, 0xF3670F1EFA
932 if ((Imm & 0x00FFFFFF) != 0x0F1EFA)
933 return false;
934
935 uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
936 0x65, 0x66, 0x67, 0xf0, 0xf2};
937 int i = 24; // 24bit 0x0F1EFA has matched
938 while (i < 64) {
939 uint8_t Byte = (Imm >> i) & 0xFF;
940 if (Byte == 0xF3)
941 return true;
942 if (!llvm::is_contained(Range&: OptionalPrefixBytes, Element: Byte))
943 return false;
944 i += 8;
945 }
946
947 return false;
948}
949
950static bool needBWI(MVT VT) {
951 return (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v64i8);
952}
953
954void X86DAGToDAGISel::PreprocessISelDAG() {
955 bool MadeChange = false;
956 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
957 E = CurDAG->allnodes_end(); I != E; ) {
958 SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues.
959
960 // This is for CET enhancement.
961 //
962 // ENDBR32 and ENDBR64 have specific opcodes:
963 // ENDBR32: F3 0F 1E FB
964 // ENDBR64: F3 0F 1E FA
965 // And we want that attackers won’t find unintended ENDBR32/64
966 // opcode matches in the binary
967 // Here’s an example:
968 // If the compiler had to generate asm for the following code:
969 // a = 0xF30F1EFA
970 // it could, for example, generate:
971 // mov 0xF30F1EFA, dword ptr[a]
972 // In such a case, the binary would include a gadget that starts
973 // with a fake ENDBR64 opcode. Therefore, we split such generation
974 // into multiple operations, let it not shows in the binary
975 if (N->getOpcode() == ISD::Constant) {
976 MVT VT = N->getSimpleValueType(ResNo: 0);
977 int64_t Imm = cast<ConstantSDNode>(Val: N)->getSExtValue();
978 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB;
979 if (Imm == EndbrImm || isEndbrImm64(Imm)) {
980 // Check that the cf-protection-branch is enabled.
981 Metadata *CFProtectionBranch =
982 MF->getFunction().getParent()->getModuleFlag(
983 Key: "cf-protection-branch");
984 if (CFProtectionBranch || IndirectBranchTracking) {
985 SDLoc dl(N);
986 SDValue Complement = CurDAG->getConstant(Val: ~Imm, DL: dl, VT, isTarget: false, isOpaque: true);
987 Complement = CurDAG->getNOT(DL: dl, Val: Complement, VT);
988 --I;
989 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Complement);
990 ++I;
991 MadeChange = true;
992 continue;
993 }
994 }
995 }
996
997 // If this is a target specific AND node with no flag usages, turn it back
998 // into ISD::AND to enable test instruction matching.
999 if (N->getOpcode() == X86ISD::AND && !N->hasAnyUseOfValue(Value: 1)) {
1000 SDValue Res = CurDAG->getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1001 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
1002 --I;
1003 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Res);
1004 ++I;
1005 MadeChange = true;
1006 continue;
1007 }
1008
1009 // Convert vector increment or decrement to sub/add with an all-ones
1010 // constant:
1011 // add X, <1, 1...> --> sub X, <-1, -1...>
1012 // sub X, <1, 1...> --> add X, <-1, -1...>
1013 // The all-ones vector constant can be materialized using a pcmpeq
1014 // instruction that is commonly recognized as an idiom (has no register
1015 // dependency), so that's better/smaller than loading a splat 1 constant.
1016 //
1017 // But don't do this if it would inhibit a potentially profitable load
1018 // folding opportunity for the other operand. That only occurs with the
1019 // intersection of:
1020 // (1) The other operand (op0) is load foldable.
1021 // (2) The op is an add (otherwise, we are *creating* an add and can still
1022 // load fold the other op).
1023 // (3) The target has AVX (otherwise, we have a destructive add and can't
1024 // load fold the other op without killing the constant op).
1025 // (4) The constant 1 vector has multiple uses (so it is profitable to load
1026 // into a register anyway).
1027 auto mayPreventLoadFold = [&]() {
1028 return X86::mayFoldLoad(Op: N->getOperand(Num: 0), Subtarget: *Subtarget) &&
1029 N->getOpcode() == ISD::ADD && Subtarget->hasAVX() &&
1030 !N->getOperand(Num: 1).hasOneUse();
1031 };
1032 if ((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
1033 N->getSimpleValueType(ResNo: 0).isVector() && !mayPreventLoadFold()) {
1034 APInt SplatVal;
1035 if (!ISD::isBuildVectorOfConstantSDNodes(
1036 N: peekThroughBitcasts(V: N->getOperand(Num: 0)).getNode()) &&
1037 X86::isConstantSplat(Op: N->getOperand(Num: 1), SplatVal) &&
1038 SplatVal.isOne()) {
1039 SDLoc DL(N);
1040
1041 MVT VT = N->getSimpleValueType(ResNo: 0);
1042 unsigned NumElts = VT.getSizeInBits() / 32;
1043 SDValue AllOnes =
1044 CurDAG->getAllOnesConstant(DL, VT: MVT::getVectorVT(VT: MVT::i32, NumElements: NumElts));
1045 AllOnes = CurDAG->getBitcast(VT, V: AllOnes);
1046
1047 unsigned NewOpcode = N->getOpcode() == ISD::ADD ? ISD::SUB : ISD::ADD;
1048 SDValue Res =
1049 CurDAG->getNode(Opcode: NewOpcode, DL, VT, N1: N->getOperand(Num: 0), N2: AllOnes);
1050 --I;
1051 CurDAG->ReplaceAllUsesWith(From: N, To: Res.getNode());
1052 ++I;
1053 MadeChange = true;
1054 continue;
1055 }
1056 }
1057
1058 switch (N->getOpcode()) {
1059 case X86ISD::VBROADCAST: {
1060 MVT VT = N->getSimpleValueType(ResNo: 0);
1061 // Emulate v32i16/v64i8 broadcast without BWI.
1062 if (!Subtarget->hasBWI() && needBWI(VT)) {
1063 MVT NarrowVT = VT.getHalfNumVectorElementsVT();
1064 SDLoc dl(N);
1065 SDValue NarrowBCast =
1066 CurDAG->getNode(Opcode: X86ISD::VBROADCAST, DL: dl, VT: NarrowVT, Operand: N->getOperand(Num: 0));
1067 SDValue Res =
1068 CurDAG->getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT, N1: CurDAG->getUNDEF(VT),
1069 N2: NarrowBCast, N3: CurDAG->getIntPtrConstant(Val: 0, DL: dl));
1070 unsigned Index = NarrowVT.getVectorMinNumElements();
1071 Res = CurDAG->getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT, N1: Res, N2: NarrowBCast,
1072 N3: CurDAG->getIntPtrConstant(Val: Index, DL: dl));
1073
1074 --I;
1075 CurDAG->ReplaceAllUsesWith(From: N, To: Res.getNode());
1076 ++I;
1077 MadeChange = true;
1078 continue;
1079 }
1080
1081 break;
1082 }
1083 case X86ISD::VBROADCAST_LOAD: {
1084 MVT VT = N->getSimpleValueType(ResNo: 0);
1085 // Emulate v32i16/v64i8 broadcast without BWI.
1086 if (!Subtarget->hasBWI() && needBWI(VT)) {
1087 MVT NarrowVT = VT.getHalfNumVectorElementsVT();
1088 auto *MemNode = cast<MemSDNode>(Val: N);
1089 SDLoc dl(N);
1090 SDVTList VTs = CurDAG->getVTList(VT1: NarrowVT, VT2: MVT::Other);
1091 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()};
1092 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode(
1093 Opcode: X86ISD::VBROADCAST_LOAD, dl, VTList: VTs, Ops, MemVT: MemNode->getMemoryVT(),
1094 MMO: MemNode->getMemOperand());
1095 SDValue Res =
1096 CurDAG->getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT, N1: CurDAG->getUNDEF(VT),
1097 N2: NarrowBCast, N3: CurDAG->getIntPtrConstant(Val: 0, DL: dl));
1098 unsigned Index = NarrowVT.getVectorMinNumElements();
1099 Res = CurDAG->getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT, N1: Res, N2: NarrowBCast,
1100 N3: CurDAG->getIntPtrConstant(Val: Index, DL: dl));
1101
1102 --I;
1103 SDValue To[] = {Res, NarrowBCast.getValue(R: 1)};
1104 CurDAG->ReplaceAllUsesWith(From: N, To);
1105 ++I;
1106 MadeChange = true;
1107 continue;
1108 }
1109
1110 break;
1111 }
1112 case ISD::LOAD: {
1113 // If this is a XMM/YMM load of the same lower bits as another YMM/ZMM
1114 // load, then just extract the lower subvector and avoid the second load.
1115 auto *Ld = cast<LoadSDNode>(Val: N);
1116 MVT VT = N->getSimpleValueType(ResNo: 0);
1117 if (!ISD::isNormalLoad(N: Ld) || !Ld->isSimple() ||
1118 !(VT.is128BitVector() || VT.is256BitVector()))
1119 break;
1120
1121 MVT MaxVT = VT;
1122 SDNode *MaxLd = nullptr;
1123 SDValue Ptr = Ld->getBasePtr();
1124 SDValue Chain = Ld->getChain();
1125 for (SDNode *User : Ptr->users()) {
1126 auto *UserLd = dyn_cast<LoadSDNode>(Val: User);
1127 MVT UserVT = User->getSimpleValueType(ResNo: 0);
1128 if (User != N && UserLd && ISD::isNormalLoad(N: User) &&
1129 UserLd->getBasePtr() == Ptr && UserLd->getChain() == Chain &&
1130 !User->hasAnyUseOfValue(Value: 1) &&
1131 (UserVT.is256BitVector() || UserVT.is512BitVector()) &&
1132 UserVT.getSizeInBits() > VT.getSizeInBits() &&
1133 (!MaxLd || UserVT.getSizeInBits() > MaxVT.getSizeInBits())) {
1134 MaxLd = User;
1135 MaxVT = UserVT;
1136 }
1137 }
1138 if (MaxLd) {
1139 SDLoc dl(N);
1140 unsigned NumSubElts = VT.getSizeInBits() / MaxVT.getScalarSizeInBits();
1141 MVT SubVT = MVT::getVectorVT(VT: MaxVT.getScalarType(), NumElements: NumSubElts);
1142 SDValue Extract = CurDAG->getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: dl, VT: SubVT,
1143 N1: SDValue(MaxLd, 0),
1144 N2: CurDAG->getIntPtrConstant(Val: 0, DL: dl));
1145 SDValue Res = CurDAG->getBitcast(VT, V: Extract);
1146
1147 --I;
1148 SDValue To[] = {Res, SDValue(MaxLd, 1)};
1149 CurDAG->ReplaceAllUsesWith(From: N, To);
1150 ++I;
1151 MadeChange = true;
1152 continue;
1153 }
1154 break;
1155 }
1156 case ISD::VSELECT: {
1157 // Replace VSELECT with non-mask conditions with with BLENDV/VPTERNLOG.
1158 EVT EleVT = N->getOperand(Num: 0).getValueType().getVectorElementType();
1159 if (EleVT == MVT::i1)
1160 break;
1161
1162 assert(Subtarget->hasSSE41() && "Expected SSE4.1 support!");
1163 assert(N->getValueType(0).getVectorElementType() != MVT::i16 &&
1164 "We can't replace VSELECT with BLENDV in vXi16!");
1165 SDValue R;
1166 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(Op: N->getOperand(Num: 0)) ==
1167 EleVT.getSizeInBits()) {
1168 R = CurDAG->getNode(Opcode: X86ISD::VPTERNLOG, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1169 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2),
1170 N4: CurDAG->getTargetConstant(Val: 0xCA, DL: SDLoc(N), VT: MVT::i8));
1171 } else {
1172 R = CurDAG->getNode(Opcode: X86ISD::BLENDV, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1173 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1),
1174 N3: N->getOperand(Num: 2));
1175 }
1176 --I;
1177 CurDAG->ReplaceAllUsesWith(From: N, To: R.getNode());
1178 ++I;
1179 MadeChange = true;
1180 continue;
1181 }
1182 case ISD::FP_ROUND:
1183 case ISD::STRICT_FP_ROUND:
1184 case ISD::FP_TO_SINT:
1185 case ISD::FP_TO_UINT:
1186 case ISD::STRICT_FP_TO_SINT:
1187 case ISD::STRICT_FP_TO_UINT: {
1188 // Replace vector fp_to_s/uint with their X86 specific equivalent so we
1189 // don't need 2 sets of patterns.
1190 if (!N->getSimpleValueType(ResNo: 0).isVector())
1191 break;
1192
1193 unsigned NewOpc;
1194 switch (N->getOpcode()) {
1195 default: llvm_unreachable("Unexpected opcode!");
1196 case ISD::FP_ROUND: NewOpc = X86ISD::VFPROUND; break;
1197 case ISD::STRICT_FP_ROUND: NewOpc = X86ISD::STRICT_VFPROUND; break;
1198 case ISD::STRICT_FP_TO_SINT: NewOpc = X86ISD::STRICT_CVTTP2SI; break;
1199 case ISD::FP_TO_SINT: NewOpc = X86ISD::CVTTP2SI; break;
1200 case ISD::STRICT_FP_TO_UINT: NewOpc = X86ISD::STRICT_CVTTP2UI; break;
1201 case ISD::FP_TO_UINT: NewOpc = X86ISD::CVTTP2UI; break;
1202 }
1203 SDValue Res;
1204 if (N->isStrictFPOpcode())
1205 Res =
1206 CurDAG->getNode(Opcode: NewOpc, DL: SDLoc(N), ResultTys: {N->getValueType(ResNo: 0), MVT::Other},
1207 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1)});
1208 else
1209 Res =
1210 CurDAG->getNode(Opcode: NewOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1211 Operand: N->getOperand(Num: 0));
1212 --I;
1213 CurDAG->ReplaceAllUsesWith(From: N, To: Res.getNode());
1214 ++I;
1215 MadeChange = true;
1216 continue;
1217 }
1218 case ISD::SHL:
1219 case ISD::SRA:
1220 case ISD::SRL: {
1221 // Replace vector shifts with their X86 specific equivalent so we don't
1222 // need 2 sets of patterns.
1223 if (!N->getValueType(ResNo: 0).isVector())
1224 break;
1225
1226 unsigned NewOpc;
1227 switch (N->getOpcode()) {
1228 default: llvm_unreachable("Unexpected opcode!");
1229 case ISD::SHL: NewOpc = X86ISD::VSHLV; break;
1230 case ISD::SRA: NewOpc = X86ISD::VSRAV; break;
1231 case ISD::SRL: NewOpc = X86ISD::VSRLV; break;
1232 }
1233 SDValue Res = CurDAG->getNode(Opcode: NewOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1234 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
1235 --I;
1236 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Res);
1237 ++I;
1238 MadeChange = true;
1239 continue;
1240 }
1241 case ISD::ANY_EXTEND:
1242 case ISD::ANY_EXTEND_VECTOR_INREG: {
1243 // Replace vector any extend with the zero extend equivalents so we don't
1244 // need 2 sets of patterns. Ignore vXi1 extensions.
1245 if (!N->getValueType(ResNo: 0).isVector())
1246 break;
1247
1248 unsigned NewOpc;
1249 if (N->getOperand(Num: 0).getScalarValueSizeInBits() == 1) {
1250 assert(N->getOpcode() == ISD::ANY_EXTEND &&
1251 "Unexpected opcode for mask vector!");
1252 NewOpc = ISD::SIGN_EXTEND;
1253 } else {
1254 NewOpc = N->getOpcode() == ISD::ANY_EXTEND
1255 ? ISD::ZERO_EXTEND
1256 : ISD::ZERO_EXTEND_VECTOR_INREG;
1257 }
1258
1259 SDValue Res = CurDAG->getNode(Opcode: NewOpc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1260 Operand: N->getOperand(Num: 0));
1261 --I;
1262 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Res);
1263 ++I;
1264 MadeChange = true;
1265 continue;
1266 }
1267 case ISD::FCEIL:
1268 case ISD::STRICT_FCEIL:
1269 case ISD::FFLOOR:
1270 case ISD::STRICT_FFLOOR:
1271 case ISD::FTRUNC:
1272 case ISD::STRICT_FTRUNC:
1273 case ISD::FROUNDEVEN:
1274 case ISD::STRICT_FROUNDEVEN:
1275 case ISD::FNEARBYINT:
1276 case ISD::STRICT_FNEARBYINT:
1277 case ISD::FRINT:
1278 case ISD::STRICT_FRINT: {
1279 // Replace fp rounding with their X86 specific equivalent so we don't
1280 // need 2 sets of patterns.
1281 unsigned Imm;
1282 switch (N->getOpcode()) {
1283 default: llvm_unreachable("Unexpected opcode!");
1284 case ISD::STRICT_FCEIL:
1285 case ISD::FCEIL: Imm = 0xA; break;
1286 case ISD::STRICT_FFLOOR:
1287 case ISD::FFLOOR: Imm = 0x9; break;
1288 case ISD::STRICT_FTRUNC:
1289 case ISD::FTRUNC: Imm = 0xB; break;
1290 case ISD::STRICT_FROUNDEVEN:
1291 case ISD::FROUNDEVEN: Imm = 0x8; break;
1292 case ISD::STRICT_FNEARBYINT:
1293 case ISD::FNEARBYINT: Imm = 0xC; break;
1294 case ISD::STRICT_FRINT:
1295 case ISD::FRINT: Imm = 0x4; break;
1296 }
1297 SDLoc dl(N);
1298 bool IsStrict = N->isStrictFPOpcode();
1299 SDValue Res;
1300 if (IsStrict)
1301 Res = CurDAG->getNode(Opcode: X86ISD::STRICT_VRNDSCALE, DL: dl,
1302 ResultTys: {N->getValueType(ResNo: 0), MVT::Other},
1303 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1),
1304 CurDAG->getTargetConstant(Val: Imm, DL: dl, VT: MVT::i32)});
1305 else
1306 Res = CurDAG->getNode(Opcode: X86ISD::VRNDSCALE, DL: dl, VT: N->getValueType(ResNo: 0),
1307 N1: N->getOperand(Num: 0),
1308 N2: CurDAG->getTargetConstant(Val: Imm, DL: dl, VT: MVT::i32));
1309 --I;
1310 CurDAG->ReplaceAllUsesWith(From: N, To: Res.getNode());
1311 ++I;
1312 MadeChange = true;
1313 continue;
1314 }
1315 case X86ISD::FANDN:
1316 case X86ISD::FAND:
1317 case X86ISD::FOR:
1318 case X86ISD::FXOR: {
1319 // Widen scalar fp logic ops to vector to reduce isel patterns.
1320 // FIXME: Can we do this during lowering/combine.
1321 MVT VT = N->getSimpleValueType(ResNo: 0);
1322 if (VT.isVector() || VT == MVT::f128)
1323 break;
1324
1325 MVT VecVT = VT == MVT::f64 ? MVT::v2f64
1326 : VT == MVT::f32 ? MVT::v4f32
1327 : MVT::v8f16;
1328
1329 SDLoc dl(N);
1330 SDValue Op0 = CurDAG->getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: VecVT,
1331 Operand: N->getOperand(Num: 0));
1332 SDValue Op1 = CurDAG->getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: VecVT,
1333 Operand: N->getOperand(Num: 1));
1334
1335 SDValue Res;
1336 if (Subtarget->hasSSE2()) {
1337 EVT IntVT = EVT(VecVT).changeVectorElementTypeToInteger();
1338 Op0 = CurDAG->getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: Op0);
1339 Op1 = CurDAG->getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: Op1);
1340 unsigned Opc;
1341 switch (N->getOpcode()) {
1342 default: llvm_unreachable("Unexpected opcode!");
1343 case X86ISD::FANDN: Opc = X86ISD::ANDNP; break;
1344 case X86ISD::FAND: Opc = ISD::AND; break;
1345 case X86ISD::FOR: Opc = ISD::OR; break;
1346 case X86ISD::FXOR: Opc = ISD::XOR; break;
1347 }
1348 Res = CurDAG->getNode(Opcode: Opc, DL: dl, VT: IntVT, N1: Op0, N2: Op1);
1349 Res = CurDAG->getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecVT, Operand: Res);
1350 } else {
1351 Res = CurDAG->getNode(Opcode: N->getOpcode(), DL: dl, VT: VecVT, N1: Op0, N2: Op1);
1352 }
1353 Res = CurDAG->getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT, N1: Res,
1354 N2: CurDAG->getIntPtrConstant(Val: 0, DL: dl));
1355 --I;
1356 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Res);
1357 ++I;
1358 MadeChange = true;
1359 continue;
1360 }
1361 }
1362
1363 if (OptLevel != CodeGenOptLevel::None &&
1364 // Only do this when the target can fold the load into the call or
1365 // jmp.
1366 !Subtarget->useIndirectThunkCalls() &&
1367 ((N->getOpcode() == X86ISD::CALL && !Subtarget->slowTwoMemOps() &&
1368 !Subtarget->slowIndirectCall()) ||
1369 (N->getOpcode() == X86ISD::TC_RETURN &&
1370 (Subtarget->is64Bit() ||
1371 !getTargetMachine().isPositionIndependent())))) {
1372 /// Also try moving call address load from outside callseq_start to just
1373 /// before the call to allow it to be folded.
1374 ///
1375 /// [Load chain]
1376 /// ^
1377 /// |
1378 /// [Load]
1379 /// ^ ^
1380 /// | |
1381 /// / \--
1382 /// / |
1383 ///[CALLSEQ_START] |
1384 /// ^ |
1385 /// | |
1386 /// [LOAD/C2Reg] |
1387 /// | |
1388 /// \ /
1389 /// \ /
1390 /// [CALL]
1391 bool HasCallSeq = N->getOpcode() == X86ISD::CALL;
1392 SDValue Chain = N->getOperand(Num: 0);
1393 SDValue Load = N->getOperand(Num: 1);
1394 if (!isCalleeLoad(Callee: Load, Chain, HasCallSeq))
1395 continue;
1396 if (N->getOpcode() == X86ISD::TC_RETURN && !checkTCRetEnoughRegs(N))
1397 continue;
1398 moveBelowOrigChain(CurDAG, Load, Call: SDValue(N, 0), OrigChain: Chain);
1399 ++NumLoadMoved;
1400 MadeChange = true;
1401 continue;
1402 }
1403
1404 // Lower fpround and fpextend nodes that target the FP stack to be store and
1405 // load to the stack. This is a gross hack. We would like to simply mark
1406 // these as being illegal, but when we do that, legalize produces these when
1407 // it expands calls, then expands these in the same legalize pass. We would
1408 // like dag combine to be able to hack on these between the call expansion
1409 // and the node legalization. As such this pass basically does "really
1410 // late" legalization of these inline with the X86 isel pass.
1411 // FIXME: This should only happen when not compiled with -O0.
1412 switch (N->getOpcode()) {
1413 default: continue;
1414 case ISD::FP_ROUND:
1415 case ISD::FP_EXTEND:
1416 {
1417 MVT SrcVT = N->getOperand(Num: 0).getSimpleValueType();
1418 MVT DstVT = N->getSimpleValueType(ResNo: 0);
1419
1420 // If any of the sources are vectors, no fp stack involved.
1421 if (SrcVT.isVector() || DstVT.isVector())
1422 continue;
1423
1424 // If the source and destination are SSE registers, then this is a legal
1425 // conversion that should not be lowered.
1426 const X86TargetLowering *X86Lowering =
1427 static_cast<const X86TargetLowering *>(TLI);
1428 bool SrcIsSSE = X86Lowering->isScalarFPTypeInSSEReg(VT: SrcVT);
1429 bool DstIsSSE = X86Lowering->isScalarFPTypeInSSEReg(VT: DstVT);
1430 if (SrcIsSSE && DstIsSSE)
1431 continue;
1432
1433 if (!SrcIsSSE && !DstIsSSE) {
1434 // If this is an FPStack extension, it is a noop.
1435 if (N->getOpcode() == ISD::FP_EXTEND)
1436 continue;
1437 // If this is a value-preserving FPStack truncation, it is a noop.
1438 if (N->getConstantOperandVal(Num: 1))
1439 continue;
1440 }
1441
1442 // Here we could have an FP stack truncation or an FPStack <-> SSE convert.
1443 // FPStack has extload and truncstore. SSE can fold direct loads into other
1444 // operations. Based on this, decide what we want to do.
1445 MVT MemVT = (N->getOpcode() == ISD::FP_ROUND) ? DstVT : SrcVT;
1446 SDValue MemTmp = CurDAG->CreateStackTemporary(VT: MemVT);
1447 int SPFI = cast<FrameIndexSDNode>(Val&: MemTmp)->getIndex();
1448 MachinePointerInfo MPI =
1449 MachinePointerInfo::getFixedStack(MF&: CurDAG->getMachineFunction(), FI: SPFI);
1450 SDLoc dl(N);
1451
1452 // FIXME: optimize the case where the src/dest is a load or store?
1453
1454 SDValue Store = CurDAG->getTruncStore(
1455 Chain: CurDAG->getEntryNode(), dl, Val: N->getOperand(Num: 0), Ptr: MemTmp, PtrInfo: MPI, SVT: MemVT);
1456 SDValue Result = CurDAG->getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: DstVT, Chain: Store,
1457 Ptr: MemTmp, PtrInfo: MPI, MemVT);
1458
1459 // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the
1460 // extload we created. This will cause general havok on the dag because
1461 // anything below the conversion could be folded into other existing nodes.
1462 // To avoid invalidating 'I', back it up to the convert node.
1463 --I;
1464 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result);
1465 break;
1466 }
1467
1468 //The sequence of events for lowering STRICT_FP versions of these nodes requires
1469 //dealing with the chain differently, as there is already a preexisting chain.
1470 case ISD::STRICT_FP_ROUND:
1471 case ISD::STRICT_FP_EXTEND:
1472 {
1473 MVT SrcVT = N->getOperand(Num: 1).getSimpleValueType();
1474 MVT DstVT = N->getSimpleValueType(ResNo: 0);
1475
1476 // If any of the sources are vectors, no fp stack involved.
1477 if (SrcVT.isVector() || DstVT.isVector())
1478 continue;
1479
1480 // If the source and destination are SSE registers, then this is a legal
1481 // conversion that should not be lowered.
1482 const X86TargetLowering *X86Lowering =
1483 static_cast<const X86TargetLowering *>(TLI);
1484 bool SrcIsSSE = X86Lowering->isScalarFPTypeInSSEReg(VT: SrcVT);
1485 bool DstIsSSE = X86Lowering->isScalarFPTypeInSSEReg(VT: DstVT);
1486 if (SrcIsSSE && DstIsSSE)
1487 continue;
1488
1489 if (!SrcIsSSE && !DstIsSSE) {
1490 // If this is an FPStack extension, it is a noop.
1491 if (N->getOpcode() == ISD::STRICT_FP_EXTEND)
1492 continue;
1493 // If this is a value-preserving FPStack truncation, it is a noop.
1494 if (N->getConstantOperandVal(Num: 2))
1495 continue;
1496 }
1497
1498 // Here we could have an FP stack truncation or an FPStack <-> SSE convert.
1499 // FPStack has extload and truncstore. SSE can fold direct loads into other
1500 // operations. Based on this, decide what we want to do.
1501 MVT MemVT = (N->getOpcode() == ISD::STRICT_FP_ROUND) ? DstVT : SrcVT;
1502 SDValue MemTmp = CurDAG->CreateStackTemporary(VT: MemVT);
1503 int SPFI = cast<FrameIndexSDNode>(Val&: MemTmp)->getIndex();
1504 MachinePointerInfo MPI =
1505 MachinePointerInfo::getFixedStack(MF&: CurDAG->getMachineFunction(), FI: SPFI);
1506 SDLoc dl(N);
1507
1508 // FIXME: optimize the case where the src/dest is a load or store?
1509
1510 //Since the operation is StrictFP, use the preexisting chain.
1511 SDValue Store, Result;
1512 if (!SrcIsSSE) {
1513 SDVTList VTs = CurDAG->getVTList(VT: MVT::Other);
1514 SDValue Ops[] = {N->getOperand(Num: 0), N->getOperand(Num: 1), MemTmp};
1515 Store = CurDAG->getMemIntrinsicNode(Opcode: X86ISD::FST, dl, VTList: VTs, Ops, MemVT,
1516 PtrInfo: MPI, /*Align*/ Alignment: std::nullopt,
1517 Flags: MachineMemOperand::MOStore);
1518 if (N->getFlags().hasNoFPExcept()) {
1519 SDNodeFlags Flags = Store->getFlags();
1520 Flags.setNoFPExcept(true);
1521 Store->setFlags(Flags);
1522 }
1523 } else {
1524 assert(SrcVT == MemVT && "Unexpected VT!");
1525 Store = CurDAG->getStore(Chain: N->getOperand(Num: 0), dl, Val: N->getOperand(Num: 1), Ptr: MemTmp,
1526 PtrInfo: MPI);
1527 }
1528
1529 if (!DstIsSSE) {
1530 SDVTList VTs = CurDAG->getVTList(VT1: DstVT, VT2: MVT::Other);
1531 SDValue Ops[] = {Store, MemTmp};
1532 Result = CurDAG->getMemIntrinsicNode(
1533 Opcode: X86ISD::FLD, dl, VTList: VTs, Ops, MemVT, PtrInfo: MPI,
1534 /*Align*/ Alignment: std::nullopt, Flags: MachineMemOperand::MOLoad);
1535 if (N->getFlags().hasNoFPExcept()) {
1536 SDNodeFlags Flags = Result->getFlags();
1537 Flags.setNoFPExcept(true);
1538 Result->setFlags(Flags);
1539 }
1540 } else {
1541 assert(DstVT == MemVT && "Unexpected VT!");
1542 Result = CurDAG->getLoad(VT: DstVT, dl, Chain: Store, Ptr: MemTmp, PtrInfo: MPI);
1543 }
1544
1545 // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the
1546 // extload we created. This will cause general havok on the dag because
1547 // anything below the conversion could be folded into other existing nodes.
1548 // To avoid invalidating 'I', back it up to the convert node.
1549 --I;
1550 CurDAG->ReplaceAllUsesWith(From: N, To: Result.getNode());
1551 break;
1552 }
1553 }
1554
1555
1556 // Now that we did that, the node is dead. Increment the iterator to the
1557 // next node to process, then delete N.
1558 ++I;
1559 MadeChange = true;
1560 }
1561
1562 // Remove any dead nodes that may have been left behind.
1563 if (MadeChange)
1564 CurDAG->RemoveDeadNodes();
1565}
1566
1567// Look for a redundant movzx/movsx that can occur after an 8-bit divrem.
1568bool X86DAGToDAGISel::tryOptimizeRem8Extend(SDNode *N) {
1569 unsigned Opc = N->getMachineOpcode();
1570 if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 &&
1571 Opc != X86::MOVSX64rr8)
1572 return false;
1573
1574 SDValue N0 = N->getOperand(Num: 0);
1575
1576 // We need to be extracting the lower bit of an extend.
1577 if (!N0.isMachineOpcode() ||
1578 N0.getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG ||
1579 N0.getConstantOperandVal(i: 1) != X86::sub_8bit)
1580 return false;
1581
1582 // We're looking for either a movsx or movzx to match the original opcode.
1583 unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX
1584 : X86::MOVSX32rr8_NOREX;
1585 SDValue N00 = N0.getOperand(i: 0);
1586 if (!N00.isMachineOpcode() || N00.getMachineOpcode() != ExpectedOpc)
1587 return false;
1588
1589 if (Opc == X86::MOVSX64rr8) {
1590 // If we had a sign extend from 8 to 64 bits. We still need to go from 32
1591 // to 64.
1592 MachineSDNode *Extend = CurDAG->getMachineNode(Opcode: X86::MOVSX64rr32, dl: SDLoc(N),
1593 VT: MVT::i64, Op1: N00);
1594 ReplaceUses(F: N, T: Extend);
1595 } else {
1596 // Ok we can drop this extend and just use the original extend.
1597 ReplaceUses(F: N, T: N00.getNode());
1598 }
1599
1600 return true;
1601}
1602
1603void X86DAGToDAGISel::PostprocessISelDAG() {
1604 // Skip peepholes at -O0.
1605 if (TM.getOptLevel() == CodeGenOptLevel::None)
1606 return;
1607
1608 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
1609
1610 bool MadeChange = false;
1611 while (Position != CurDAG->allnodes_begin()) {
1612 SDNode *N = &*--Position;
1613 // Skip dead nodes and any non-machine opcodes.
1614 if (N->use_empty() || !N->isMachineOpcode())
1615 continue;
1616
1617 if (tryOptimizeRem8Extend(N)) {
1618 MadeChange = true;
1619 continue;
1620 }
1621
1622 unsigned Opc = N->getMachineOpcode();
1623 switch (Opc) {
1624 default:
1625 continue;
1626 // ANDrr/rm + TESTrr+ -> TESTrr/TESTmr
1627 case X86::TEST8rr:
1628 case X86::TEST16rr:
1629 case X86::TEST32rr:
1630 case X86::TEST64rr:
1631 // ANDrr/rm + CTESTrr -> CTESTrr/CTESTmr
1632 case X86::CTEST8rr:
1633 case X86::CTEST16rr:
1634 case X86::CTEST32rr:
1635 case X86::CTEST64rr: {
1636 auto &Op0 = N->getOperand(Num: 0);
1637 if (Op0 != N->getOperand(Num: 1) || !Op0->hasNUsesOfValue(NUses: 2, Value: Op0.getResNo()) ||
1638 !Op0.isMachineOpcode())
1639 continue;
1640 SDValue And = N->getOperand(Num: 0);
1641#define CASE_ND(OP) \
1642 case X86::OP: \
1643 case X86::OP##_ND:
1644 switch (And.getMachineOpcode()) {
1645 default:
1646 continue;
1647 CASE_ND(AND8rr)
1648 CASE_ND(AND16rr)
1649 CASE_ND(AND32rr)
1650 CASE_ND(AND64rr) {
1651 if (And->hasAnyUseOfValue(Value: 1))
1652 continue;
1653 SmallVector<SDValue> Ops(N->op_values());
1654 Ops[0] = And.getOperand(i: 0);
1655 Ops[1] = And.getOperand(i: 1);
1656 MachineSDNode *Test =
1657 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VT: MVT::i32, Ops);
1658 ReplaceUses(F: N, T: Test);
1659 MadeChange = true;
1660 continue;
1661 }
1662 CASE_ND(AND8rm)
1663 CASE_ND(AND16rm)
1664 CASE_ND(AND32rm)
1665 CASE_ND(AND64rm) {
1666 if (And->hasAnyUseOfValue(Value: 1))
1667 continue;
1668 unsigned NewOpc;
1669 bool IsCTESTCC = X86::isCTESTCC(Opcode: Opc);
1670#define FROM_TO(A, B) \
1671 CASE_ND(A) NewOpc = IsCTESTCC ? X86::C##B : X86::B; \
1672 break;
1673 switch (And.getMachineOpcode()) {
1674 FROM_TO(AND8rm, TEST8mr);
1675 FROM_TO(AND16rm, TEST16mr);
1676 FROM_TO(AND32rm, TEST32mr);
1677 FROM_TO(AND64rm, TEST64mr);
1678 }
1679#undef FROM_TO
1680#undef CASE_ND
1681 // Need to swap the memory and register operand.
1682 SmallVector<SDValue> Ops = {And.getOperand(i: 1), And.getOperand(i: 2),
1683 And.getOperand(i: 3), And.getOperand(i: 4),
1684 And.getOperand(i: 5), And.getOperand(i: 0)};
1685 // CC, Cflags.
1686 if (IsCTESTCC) {
1687 Ops.push_back(Elt: N->getOperand(Num: 2));
1688 Ops.push_back(Elt: N->getOperand(Num: 3));
1689 }
1690 // Chain of memory load
1691 Ops.push_back(Elt: And.getOperand(i: 6));
1692 // Glue
1693 if (IsCTESTCC)
1694 Ops.push_back(Elt: N->getOperand(Num: 4));
1695
1696 MachineSDNode *Test = CurDAG->getMachineNode(
1697 Opcode: NewOpc, dl: SDLoc(N), VT1: MVT::i32, VT2: MVT::Other, Ops);
1698 CurDAG->setNodeMemRefs(
1699 N: Test, NewMemRefs: cast<MachineSDNode>(Val: And.getNode())->memoperands());
1700 ReplaceUses(F: And.getValue(R: 2), T: SDValue(Test, 1));
1701 ReplaceUses(F: SDValue(N, 0), T: SDValue(Test, 0));
1702 MadeChange = true;
1703 continue;
1704 }
1705 }
1706 }
1707 // Look for a KAND+KORTEST and turn it into KTEST if only the zero flag is
1708 // used. We're doing this late so we can prefer to fold the AND into masked
1709 // comparisons. Doing that can be better for the live range of the mask
1710 // register.
1711 case X86::KORTESTBkk:
1712 case X86::KORTESTWkk:
1713 case X86::KORTESTDkk:
1714 case X86::KORTESTQkk: {
1715 SDValue Op0 = N->getOperand(Num: 0);
1716 if (Op0 != N->getOperand(Num: 1) || !N->isOnlyUserOf(N: Op0.getNode()) ||
1717 !Op0.isMachineOpcode() || !onlyUsesZeroFlag(Flags: SDValue(N, 0)))
1718 continue;
1719#define CASE(A) \
1720 case X86::A: \
1721 break;
1722 switch (Op0.getMachineOpcode()) {
1723 default:
1724 continue;
1725 CASE(KANDBkk)
1726 CASE(KANDWkk)
1727 CASE(KANDDkk)
1728 CASE(KANDQkk)
1729 }
1730 unsigned NewOpc;
1731#define FROM_TO(A, B) \
1732 case X86::A: \
1733 NewOpc = X86::B; \
1734 break;
1735 switch (Opc) {
1736 FROM_TO(KORTESTBkk, KTESTBkk)
1737 FROM_TO(KORTESTWkk, KTESTWkk)
1738 FROM_TO(KORTESTDkk, KTESTDkk)
1739 FROM_TO(KORTESTQkk, KTESTQkk)
1740 }
1741 // KANDW is legal with AVX512F, but KTESTW requires AVX512DQ. The other
1742 // KAND instructions and KTEST use the same ISA feature.
1743 if (NewOpc == X86::KTESTWkk && !Subtarget->hasDQI())
1744 continue;
1745#undef FROM_TO
1746 MachineSDNode *KTest = CurDAG->getMachineNode(
1747 Opcode: NewOpc, dl: SDLoc(N), VT: MVT::i32, Op1: Op0.getOperand(i: 0), Op2: Op0.getOperand(i: 1));
1748 ReplaceUses(F: N, T: KTest);
1749 MadeChange = true;
1750 continue;
1751 }
1752 // Attempt to remove vectors moves that were inserted to zero upper bits.
1753 case TargetOpcode::SUBREG_TO_REG: {
1754 unsigned SubRegIdx = N->getConstantOperandVal(Num: 1);
1755 if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
1756 continue;
1757
1758 SDValue Move = N->getOperand(Num: 0);
1759 if (!Move.isMachineOpcode())
1760 continue;
1761
1762 // Make sure its one of the move opcodes we recognize.
1763 switch (Move.getMachineOpcode()) {
1764 default:
1765 continue;
1766 CASE(VMOVAPDrr) CASE(VMOVUPDrr)
1767 CASE(VMOVAPSrr) CASE(VMOVUPSrr)
1768 CASE(VMOVDQArr) CASE(VMOVDQUrr)
1769 CASE(VMOVAPDYrr) CASE(VMOVUPDYrr)
1770 CASE(VMOVAPSYrr) CASE(VMOVUPSYrr)
1771 CASE(VMOVDQAYrr) CASE(VMOVDQUYrr)
1772 CASE(VMOVAPDZ128rr) CASE(VMOVUPDZ128rr)
1773 CASE(VMOVAPSZ128rr) CASE(VMOVUPSZ128rr)
1774 CASE(VMOVDQA32Z128rr) CASE(VMOVDQU32Z128rr)
1775 CASE(VMOVDQA64Z128rr) CASE(VMOVDQU64Z128rr)
1776 CASE(VMOVAPDZ256rr) CASE(VMOVUPDZ256rr)
1777 CASE(VMOVAPSZ256rr) CASE(VMOVUPSZ256rr)
1778 CASE(VMOVDQA32Z256rr) CASE(VMOVDQU32Z256rr)
1779 CASE(VMOVDQA64Z256rr) CASE(VMOVDQU64Z256rr)
1780 }
1781#undef CASE
1782
1783 SDValue In = Move.getOperand(i: 0);
1784 if (!In.isMachineOpcode() ||
1785 In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
1786 continue;
1787
1788 // Make sure the instruction has a VEX, XOP, or EVEX prefix. This covers
1789 // the SHA instructions which use a legacy encoding.
1790 uint64_t TSFlags = getInstrInfo()->get(Opcode: In.getMachineOpcode()).TSFlags;
1791 if ((TSFlags & X86II::EncodingMask) != X86II::VEX &&
1792 (TSFlags & X86II::EncodingMask) != X86II::EVEX &&
1793 (TSFlags & X86II::EncodingMask) != X86II::XOP)
1794 continue;
1795
1796 // Producing instruction is another vector instruction. We can drop the
1797 // move.
1798 CurDAG->UpdateNodeOperands(N, Op1: In, Op2: N->getOperand(Num: 1));
1799 MadeChange = true;
1800 }
1801 }
1802 }
1803
1804 if (MadeChange)
1805 CurDAG->RemoveDeadNodes();
1806}
1807
1808
1809/// Emit any code that needs to be executed only in the main function.
1810void X86DAGToDAGISel::emitSpecialCodeForMain() {
1811 if (Subtarget->isTargetCygMing()) {
1812 TargetLowering::ArgListTy Args;
1813 auto &DL = CurDAG->getDataLayout();
1814
1815 TargetLowering::CallLoweringInfo CLI(*CurDAG);
1816 CLI.setChain(CurDAG->getRoot())
1817 .setCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *CurDAG->getContext()),
1818 Target: CurDAG->getExternalSymbol(Sym: "__main", VT: TLI->getPointerTy(DL)),
1819 ArgsList: std::move(Args));
1820 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo();
1821 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
1822 CurDAG->setRoot(Result.second);
1823 }
1824}
1825
1826void X86DAGToDAGISel::emitFunctionEntryCode() {
1827 // If this is main, emit special code for main.
1828 const Function &F = MF->getFunction();
1829 if (F.hasExternalLinkage() && F.getName() == "main")
1830 emitSpecialCodeForMain();
1831}
1832
1833static bool isDispSafeForFrameIndexOrRegBase(int64_t Val) {
1834 // We can run into an issue where a frame index or a register base
1835 // includes a displacement that, when added to the explicit displacement,
1836 // will overflow the displacement field. Assuming that the
1837 // displacement fits into a 31-bit integer (which is only slightly more
1838 // aggressive than the current fundamental assumption that it fits into
1839 // a 32-bit integer), a 31-bit disp should always be safe.
1840 return isInt<31>(x: Val);
1841}
1842
1843bool X86DAGToDAGISel::foldOffsetIntoAddress(uint64_t Offset,
1844 X86ISelAddressMode &AM) {
1845 // We may have already matched a displacement and the caller just added the
1846 // symbolic displacement. So we still need to do the checks even if Offset
1847 // is zero.
1848
1849 int64_t Val = AM.Disp + Offset;
1850
1851 // Cannot combine ExternalSymbol displacements with integer offsets.
1852 if (Val != 0 && (AM.ES || AM.MCSym))
1853 return true;
1854
1855 CodeModel::Model M = TM.getCodeModel();
1856 if (Subtarget->is64Bit()) {
1857 if (Val != 0 &&
1858 !X86::isOffsetSuitableForCodeModel(Offset: Val, M,
1859 hasSymbolicDisplacement: AM.hasSymbolicDisplacement()))
1860 return true;
1861 // In addition to the checks required for a register base, check that
1862 // we do not try to use an unsafe Disp with a frame index.
1863 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
1864 !isDispSafeForFrameIndexOrRegBase(Val))
1865 return true;
1866 // In ILP32 (x32) mode, pointers are 32 bits and need to be zero-extended to
1867 // 64 bits. Instructions with 32-bit register addresses perform this zero
1868 // extension for us and we can safely ignore the high bits of Offset.
1869 // Instructions with only a 32-bit immediate address do not, though: they
1870 // sign extend instead. This means only address the low 2GB of address space
1871 // is directly addressable, we need indirect addressing for the high 2GB of
1872 // address space.
1873 // TODO: Some of the earlier checks may be relaxed for ILP32 mode as the
1874 // implicit zero extension of instructions would cover up any problem.
1875 // However, we have asserts elsewhere that get triggered if we do, so keep
1876 // the checks for now.
1877 // TODO: We would actually be able to accept these, as well as the same
1878 // addresses in LP64 mode, by adding the EIZ pseudo-register as an operand
1879 // to get an address size override to be emitted. However, this
1880 // pseudo-register is not part of any register class and therefore causes
1881 // MIR verification to fail.
1882 if (Subtarget->isTarget64BitILP32() &&
1883 !isDispSafeForFrameIndexOrRegBase(Val: (uint32_t)Val) &&
1884 !AM.hasBaseOrIndexReg())
1885 return true;
1886 } else if (Subtarget->is16Bit()) {
1887 // In 16-bit mode, displacements are limited to [-65535,65535] for FK_Data_2
1888 // fixups of unknown signedness. See X86AsmBackend::applyFixup.
1889 if (Val < -(int64_t)UINT16_MAX || Val > (int64_t)UINT16_MAX)
1890 return true;
1891 } else if (AM.hasBaseOrIndexReg() && !isDispSafeForFrameIndexOrRegBase(Val))
1892 // For 32-bit X86, make sure the displacement still isn't close to the
1893 // expressible limit.
1894 return true;
1895 AM.Disp = Val;
1896 return false;
1897}
1898
1899bool X86DAGToDAGISel::matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM,
1900 bool AllowSegmentRegForX32) {
1901 SDValue Address = N->getOperand(Num: 1);
1902
1903 // load gs:0 -> GS segment register.
1904 // load fs:0 -> FS segment register.
1905 //
1906 // This optimization is generally valid because the GNU TLS model defines that
1907 // gs:0 (or fs:0 on X86-64) contains its own address. However, for X86-64 mode
1908 // with 32-bit registers, as we get in ILP32 mode, those registers are first
1909 // zero-extended to 64 bits and then added it to the base address, which gives
1910 // unwanted results when the register holds a negative value.
1911 // For more information see http://people.redhat.com/drepper/tls.pdf
1912 if (isNullConstant(V: Address) && AM.Segment.getNode() == nullptr &&
1913 !IndirectTlsSegRefs &&
1914 (Subtarget->isTargetGlibc() || Subtarget->isTargetMusl() ||
1915 Subtarget->isTargetAndroid() || Subtarget->isTargetFuchsia())) {
1916 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32)
1917 return true;
1918 switch (N->getPointerInfo().getAddrSpace()) {
1919 case X86AS::GS:
1920 AM.Segment = CurDAG->getRegister(Reg: X86::GS, VT: MVT::i16);
1921 return false;
1922 case X86AS::FS:
1923 AM.Segment = CurDAG->getRegister(Reg: X86::FS, VT: MVT::i16);
1924 return false;
1925 // Address space X86AS::SS is not handled here, because it is not used to
1926 // address TLS areas.
1927 }
1928 }
1929
1930 return true;
1931}
1932
1933/// Try to match X86ISD::Wrapper and X86ISD::WrapperRIP nodes into an addressing
1934/// mode. These wrap things that will resolve down into a symbol reference.
1935/// If no match is possible, this returns true, otherwise it returns false.
1936bool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) {
1937 // If the addressing mode already has a symbol as the displacement, we can
1938 // never match another symbol.
1939 if (AM.hasSymbolicDisplacement())
1940 return true;
1941
1942 bool IsRIPRelTLS = false;
1943 bool IsRIPRel = N.getOpcode() == X86ISD::WrapperRIP;
1944 if (IsRIPRel) {
1945 SDValue Val = N.getOperand(i: 0);
1946 if (Val.getOpcode() == ISD::TargetGlobalTLSAddress)
1947 IsRIPRelTLS = true;
1948 }
1949
1950 // We can't use an addressing mode in the 64-bit large code model.
1951 // Global TLS addressing is an exception. In the medium code model,
1952 // we use can use a mode when RIP wrappers are present.
1953 // That signifies access to globals that are known to be "near",
1954 // such as the GOT itself.
1955 CodeModel::Model M = TM.getCodeModel();
1956 if (Subtarget->is64Bit() && M == CodeModel::Large && !IsRIPRelTLS)
1957 return true;
1958
1959 // Base and index reg must be 0 in order to use %rip as base.
1960 if (IsRIPRel && AM.hasBaseOrIndexReg())
1961 return true;
1962
1963 // Make a local copy in case we can't do this fold.
1964 X86ISelAddressMode Backup = AM;
1965
1966 int64_t Offset = 0;
1967 SDValue N0 = N.getOperand(i: 0);
1968 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: N0)) {
1969 AM.GV = G->getGlobal();
1970 AM.SymbolFlags = G->getTargetFlags();
1971 Offset = G->getOffset();
1972 } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(Val&: N0)) {
1973 AM.CP = CP->getConstVal();
1974 AM.Alignment = CP->getAlign();
1975 AM.SymbolFlags = CP->getTargetFlags();
1976 Offset = CP->getOffset();
1977 } else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Val&: N0)) {
1978 AM.ES = S->getSymbol();
1979 AM.SymbolFlags = S->getTargetFlags();
1980 } else if (auto *S = dyn_cast<MCSymbolSDNode>(Val&: N0)) {
1981 AM.MCSym = S->getMCSymbol();
1982 } else if (auto *J = dyn_cast<JumpTableSDNode>(Val&: N0)) {
1983 AM.JT = J->getIndex();
1984 AM.SymbolFlags = J->getTargetFlags();
1985 } else if (auto *BA = dyn_cast<BlockAddressSDNode>(Val&: N0)) {
1986 AM.BlockAddr = BA->getBlockAddress();
1987 AM.SymbolFlags = BA->getTargetFlags();
1988 Offset = BA->getOffset();
1989 } else
1990 llvm_unreachable("Unhandled symbol reference node.");
1991
1992 // Can't use an addressing mode with large globals.
1993 if (Subtarget->is64Bit() && !IsRIPRel && AM.GV &&
1994 TM.isLargeGlobalValue(GV: AM.GV)) {
1995 AM = Backup;
1996 return true;
1997 }
1998
1999 if (foldOffsetIntoAddress(Offset, AM)) {
2000 AM = Backup;
2001 return true;
2002 }
2003
2004 if (IsRIPRel)
2005 AM.setBaseReg(CurDAG->getRegister(Reg: X86::RIP, VT: MVT::i64));
2006
2007 // Commit the changes now that we know this fold is safe.
2008 return false;
2009}
2010
2011/// Add the specified node to the specified addressing mode, returning true if
2012/// it cannot be done. This just pattern matches for the addressing mode.
2013bool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) {
2014 if (matchAddressRecursively(N, AM, Depth: 0))
2015 return true;
2016
2017 // Post-processing: Make a second attempt to fold a load, if we now know
2018 // that there will not be any other register. This is only performed for
2019 // 64-bit ILP32 mode since 32-bit mode and 64-bit LP64 mode will have folded
2020 // any foldable load the first time.
2021 if (Subtarget->isTarget64BitILP32() &&
2022 AM.BaseType == X86ISelAddressMode::RegBase &&
2023 AM.Base_Reg.getNode() != nullptr && AM.IndexReg.getNode() == nullptr) {
2024 SDValue Save_Base_Reg = AM.Base_Reg;
2025 if (auto *LoadN = dyn_cast<LoadSDNode>(Val&: Save_Base_Reg)) {
2026 AM.Base_Reg = SDValue();
2027 if (matchLoadInAddress(N: LoadN, AM, /*AllowSegmentRegForX32=*/true))
2028 AM.Base_Reg = Save_Base_Reg;
2029 }
2030 }
2031
2032 // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has
2033 // a smaller encoding and avoids a scaled-index.
2034 if (AM.Scale == 2 &&
2035 AM.BaseType == X86ISelAddressMode::RegBase &&
2036 AM.Base_Reg.getNode() == nullptr) {
2037 AM.Base_Reg = AM.IndexReg;
2038 AM.Scale = 1;
2039 }
2040
2041 // Post-processing: Convert foo to foo(%rip), even in non-PIC mode,
2042 // because it has a smaller encoding.
2043 if (TM.getCodeModel() != CodeModel::Large &&
2044 (!AM.GV || !TM.isLargeGlobalValue(GV: AM.GV)) && Subtarget->is64Bit() &&
2045 AM.Scale == 1 && AM.BaseType == X86ISelAddressMode::RegBase &&
2046 AM.Base_Reg.getNode() == nullptr && AM.IndexReg.getNode() == nullptr &&
2047 AM.SymbolFlags == X86II::MO_NO_FLAG && AM.hasSymbolicDisplacement()) {
2048 // However, when GV is a local function symbol and in the same section as
2049 // the current instruction, and AM.Disp is negative and near INT32_MIN,
2050 // referencing GV+Disp generates a relocation referencing the section symbol
2051 // with an even smaller offset, which might underflow. We should bail out if
2052 // the negative offset is too close to INT32_MIN. Actually, we are more
2053 // conservative here, using a smaller magic number also used by
2054 // isOffsetSuitableForCodeModel.
2055 if (isa_and_nonnull<Function>(Val: AM.GV) && AM.Disp < -16 * 1024 * 1024)
2056 return true;
2057
2058 AM.Base_Reg = CurDAG->getRegister(Reg: X86::RIP, VT: MVT::i64);
2059 }
2060
2061 return false;
2062}
2063
2064bool X86DAGToDAGISel::matchAdd(SDValue &N, X86ISelAddressMode &AM,
2065 unsigned Depth) {
2066 // Add an artificial use to this node so that we can keep track of
2067 // it if it gets CSE'd with a different node.
2068 HandleSDNode Handle(N);
2069
2070 X86ISelAddressMode Backup = AM;
2071 if (!matchAddressRecursively(N: N.getOperand(i: 0), AM, Depth: Depth+1) &&
2072 !matchAddressRecursively(N: Handle.getValue().getOperand(i: 1), AM, Depth: Depth+1))
2073 return false;
2074 AM = Backup;
2075
2076 // Try again after commutating the operands.
2077 if (!matchAddressRecursively(N: Handle.getValue().getOperand(i: 1), AM,
2078 Depth: Depth + 1) &&
2079 !matchAddressRecursively(N: Handle.getValue().getOperand(i: 0), AM, Depth: Depth + 1))
2080 return false;
2081 AM = Backup;
2082
2083 // If we couldn't fold both operands into the address at the same time,
2084 // see if we can just put each operand into a register and fold at least
2085 // the add.
2086 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2087 !AM.Base_Reg.getNode() &&
2088 !AM.IndexReg.getNode()) {
2089 N = Handle.getValue();
2090 AM.Base_Reg = N.getOperand(i: 0);
2091 AM.IndexReg = N.getOperand(i: 1);
2092 AM.Scale = 1;
2093 return false;
2094 }
2095 N = Handle.getValue();
2096 return true;
2097}
2098
2099// Insert a node into the DAG at least before the Pos node's position. This
2100// will reposition the node as needed, and will assign it a node ID that is <=
2101// the Pos node's ID. Note that this does *not* preserve the uniqueness of node
2102// IDs! The selection DAG must no longer depend on their uniqueness when this
2103// is used.
2104static void insertDAGNode(SelectionDAG &DAG, SDValue Pos, SDValue N) {
2105 if (N->getNodeId() == -1 ||
2106 (SelectionDAGISel::getUninvalidatedNodeId(N: N.getNode()) >
2107 SelectionDAGISel::getUninvalidatedNodeId(N: Pos.getNode()))) {
2108 DAG.RepositionNode(Position: Pos->getIterator(), N: N.getNode());
2109 // Mark Node as invalid for pruning as after this it may be a successor to a
2110 // selected node but otherwise be in the same position of Pos.
2111 // Conservatively mark it with the same -abs(Id) to assure node id
2112 // invariant is preserved.
2113 N->setNodeId(Pos->getNodeId());
2114 SelectionDAGISel::InvalidateNodeId(N: N.getNode());
2115 }
2116}
2117
2118// Transform "(X >> (8-C1)) & (0xff << C1)" to "((X >> 8) & 0xff) << C1" if
2119// safe. This allows us to convert the shift and and into an h-register
2120// extract and a scaled index. Returns false if the simplification is
2121// performed.
2122static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N,
2123 uint64_t Mask,
2124 SDValue Shift, SDValue X,
2125 X86ISelAddressMode &AM) {
2126 if (Shift.getOpcode() != ISD::SRL ||
2127 !isa<ConstantSDNode>(Val: Shift.getOperand(i: 1)) ||
2128 !Shift.hasOneUse())
2129 return true;
2130
2131 int ScaleLog = 8 - Shift.getConstantOperandVal(i: 1);
2132 if (ScaleLog <= 0 || ScaleLog >= 4 ||
2133 Mask != (0xffu << ScaleLog))
2134 return true;
2135
2136 MVT XVT = X.getSimpleValueType();
2137 MVT VT = N.getSimpleValueType();
2138 SDLoc DL(N);
2139 SDValue Eight = DAG.getConstant(Val: 8, DL, VT: MVT::i8);
2140 SDValue NewMask = DAG.getConstant(Val: 0xff, DL, VT: XVT);
2141 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL, VT: XVT, N1: X, N2: Eight);
2142 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: XVT, N1: Srl, N2: NewMask);
2143 SDValue Ext = DAG.getZExtOrTrunc(Op: And, DL, VT);
2144 SDValue ShlCount = DAG.getConstant(Val: ScaleLog, DL, VT: MVT::i8);
2145 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Ext, N2: ShlCount);
2146
2147 // Insert the new nodes into the topological ordering. We must do this in
2148 // a valid topological ordering as nothing is going to go back and re-sort
2149 // these nodes. We continually insert before 'N' in sequence as this is
2150 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no
2151 // hierarchy left to express.
2152 insertDAGNode(DAG, Pos: N, N: Eight);
2153 insertDAGNode(DAG, Pos: N, N: NewMask);
2154 insertDAGNode(DAG, Pos: N, N: Srl);
2155 insertDAGNode(DAG, Pos: N, N: And);
2156 insertDAGNode(DAG, Pos: N, N: Ext);
2157 insertDAGNode(DAG, Pos: N, N: ShlCount);
2158 insertDAGNode(DAG, Pos: N, N: Shl);
2159 DAG.ReplaceAllUsesWith(From: N, To: Shl);
2160 DAG.RemoveDeadNode(N: N.getNode());
2161 AM.IndexReg = Ext;
2162 AM.Scale = (1 << ScaleLog);
2163 return false;
2164}
2165
2166// Transforms "(X << C1) & C2" to "(X & (C2>>C1)) << C1" if safe and if this
2167// allows us to fold the shift into this addressing mode. Returns false if the
2168// transform succeeded.
2169static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N,
2170 X86ISelAddressMode &AM) {
2171 SDValue Shift = N.getOperand(i: 0);
2172
2173 // Use a signed mask so that shifting right will insert sign bits. These
2174 // bits will be removed when we shift the result left so it doesn't matter
2175 // what we use. This might allow a smaller immediate encoding.
2176 int64_t Mask = cast<ConstantSDNode>(Val: N->getOperand(Num: 1))->getSExtValue();
2177
2178 // If we have an any_extend feeding the AND, look through it to see if there
2179 // is a shift behind it. But only if the AND doesn't use the extended bits.
2180 // FIXME: Generalize this to other ANY_EXTEND than i32 to i64?
2181 bool FoundAnyExtend = false;
2182 if (Shift.getOpcode() == ISD::ANY_EXTEND && Shift.hasOneUse() &&
2183 Shift.getOperand(i: 0).getSimpleValueType() == MVT::i32 &&
2184 isUInt<32>(x: Mask)) {
2185 FoundAnyExtend = true;
2186 Shift = Shift.getOperand(i: 0);
2187 }
2188
2189 if (Shift.getOpcode() != ISD::SHL ||
2190 !isa<ConstantSDNode>(Val: Shift.getOperand(i: 1)))
2191 return true;
2192
2193 SDValue X = Shift.getOperand(i: 0);
2194
2195 // Not likely to be profitable if either the AND or SHIFT node has more
2196 // than one use (unless all uses are for address computation). Besides,
2197 // isel mechanism requires their node ids to be reused.
2198 if (!N.hasOneUse() || !Shift.hasOneUse())
2199 return true;
2200
2201 // Verify that the shift amount is something we can fold.
2202 unsigned ShiftAmt = Shift.getConstantOperandVal(i: 1);
2203 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
2204 return true;
2205
2206 MVT VT = N.getSimpleValueType();
2207 SDLoc DL(N);
2208 if (FoundAnyExtend) {
2209 SDValue NewX = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: X);
2210 insertDAGNode(DAG, Pos: N, N: NewX);
2211 X = NewX;
2212 }
2213
2214 SDValue NewMask = DAG.getSignedConstant(Val: Mask >> ShiftAmt, DL, VT);
2215 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: NewMask);
2216 SDValue NewShift = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: NewAnd, N2: Shift.getOperand(i: 1));
2217
2218 // Insert the new nodes into the topological ordering. We must do this in
2219 // a valid topological ordering as nothing is going to go back and re-sort
2220 // these nodes. We continually insert before 'N' in sequence as this is
2221 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no
2222 // hierarchy left to express.
2223 insertDAGNode(DAG, Pos: N, N: NewMask);
2224 insertDAGNode(DAG, Pos: N, N: NewAnd);
2225 insertDAGNode(DAG, Pos: N, N: NewShift);
2226 DAG.ReplaceAllUsesWith(From: N, To: NewShift);
2227 DAG.RemoveDeadNode(N: N.getNode());
2228
2229 AM.Scale = 1 << ShiftAmt;
2230 AM.IndexReg = NewAnd;
2231 return false;
2232}
2233
2234// Implement some heroics to detect shifts of masked values where the mask can
2235// be replaced by extending the shift and undoing that in the addressing mode
2236// scale. Patterns such as (shl (srl x, c1), c2) are canonicalized into (and
2237// (srl x, SHIFT), MASK) by DAGCombines that don't know the shl can be done in
2238// the addressing mode. This results in code such as:
2239//
2240// int f(short *y, int *lookup_table) {
2241// ...
2242// return *y + lookup_table[*y >> 11];
2243// }
2244//
2245// Turning into:
2246// movzwl (%rdi), %eax
2247// movl %eax, %ecx
2248// shrl $11, %ecx
2249// addl (%rsi,%rcx,4), %eax
2250//
2251// Instead of:
2252// movzwl (%rdi), %eax
2253// movl %eax, %ecx
2254// shrl $9, %ecx
2255// andl $124, %rcx
2256// addl (%rsi,%rcx), %eax
2257//
2258// Note that this function assumes the mask is provided as a mask *after* the
2259// value is shifted. The input chain may or may not match that, but computing
2260// such a mask is trivial.
2261static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N,
2262 uint64_t Mask,
2263 SDValue Shift, SDValue X,
2264 X86ISelAddressMode &AM) {
2265 if (Shift.getOpcode() != ISD::SRL || !Shift.hasOneUse() ||
2266 !isa<ConstantSDNode>(Val: Shift.getOperand(i: 1)))
2267 return true;
2268
2269 // We need to ensure that mask is a continuous run of bits.
2270 unsigned MaskIdx, MaskLen;
2271 if (!isShiftedMask_64(Value: Mask, MaskIdx, MaskLen))
2272 return true;
2273 unsigned MaskLZ = 64 - (MaskIdx + MaskLen);
2274
2275 unsigned ShiftAmt = Shift.getConstantOperandVal(i: 1);
2276
2277 // The amount of shift we're trying to fit into the addressing mode is taken
2278 // from the shifted mask index (number of trailing zeros of the mask).
2279 unsigned AMShiftAmt = MaskIdx;
2280
2281 // There is nothing we can do here unless the mask is removing some bits.
2282 // Also, the addressing mode can only represent shifts of 1, 2, or 3 bits.
2283 if (AMShiftAmt == 0 || AMShiftAmt > 3) return true;
2284
2285 // Scale the leading zero count down based on the actual size of the value.
2286 // Also scale it down based on the size of the shift.
2287 unsigned ScaleDown = (64 - X.getSimpleValueType().getSizeInBits()) + ShiftAmt;
2288 if (MaskLZ < ScaleDown)
2289 return true;
2290 MaskLZ -= ScaleDown;
2291
2292 // The final check is to ensure that any masked out high bits of X are
2293 // already known to be zero. Otherwise, the mask has a semantic impact
2294 // other than masking out a couple of low bits. Unfortunately, because of
2295 // the mask, zero extensions will be removed from operands in some cases.
2296 // This code works extra hard to look through extensions because we can
2297 // replace them with zero extensions cheaply if necessary.
2298 bool ReplacingAnyExtend = false;
2299 if (X.getOpcode() == ISD::ANY_EXTEND) {
2300 unsigned ExtendBits = X.getSimpleValueType().getSizeInBits() -
2301 X.getOperand(i: 0).getSimpleValueType().getSizeInBits();
2302 // Assume that we'll replace the any-extend with a zero-extend, and
2303 // narrow the search to the extended value.
2304 X = X.getOperand(i: 0);
2305 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
2306 ReplacingAnyExtend = true;
2307 }
2308 APInt MaskedHighBits =
2309 APInt::getHighBitsSet(numBits: X.getSimpleValueType().getSizeInBits(), hiBitsSet: MaskLZ);
2310 if (!DAG.MaskedValueIsZero(Op: X, Mask: MaskedHighBits))
2311 return true;
2312
2313 // We've identified a pattern that can be transformed into a single shift
2314 // and an addressing mode. Make it so.
2315 MVT VT = N.getSimpleValueType();
2316 if (ReplacingAnyExtend) {
2317 assert(X.getValueType() != VT);
2318 // We looked through an ANY_EXTEND node, insert a ZERO_EXTEND.
2319 SDValue NewX = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(X), VT, Operand: X);
2320 insertDAGNode(DAG, Pos: N, N: NewX);
2321 X = NewX;
2322 }
2323
2324 MVT XVT = X.getSimpleValueType();
2325 SDLoc DL(N);
2326 SDValue NewSRLAmt = DAG.getConstant(Val: ShiftAmt + AMShiftAmt, DL, VT: MVT::i8);
2327 SDValue NewSRL = DAG.getNode(Opcode: ISD::SRL, DL, VT: XVT, N1: X, N2: NewSRLAmt);
2328 SDValue NewExt = DAG.getZExtOrTrunc(Op: NewSRL, DL, VT);
2329 SDValue NewSHLAmt = DAG.getConstant(Val: AMShiftAmt, DL, VT: MVT::i8);
2330 SDValue NewSHL = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: NewExt, N2: NewSHLAmt);
2331
2332 // Insert the new nodes into the topological ordering. We must do this in
2333 // a valid topological ordering as nothing is going to go back and re-sort
2334 // these nodes. We continually insert before 'N' in sequence as this is
2335 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no
2336 // hierarchy left to express.
2337 insertDAGNode(DAG, Pos: N, N: NewSRLAmt);
2338 insertDAGNode(DAG, Pos: N, N: NewSRL);
2339 insertDAGNode(DAG, Pos: N, N: NewExt);
2340 insertDAGNode(DAG, Pos: N, N: NewSHLAmt);
2341 insertDAGNode(DAG, Pos: N, N: NewSHL);
2342 DAG.ReplaceAllUsesWith(From: N, To: NewSHL);
2343 DAG.RemoveDeadNode(N: N.getNode());
2344
2345 AM.Scale = 1 << AMShiftAmt;
2346 AM.IndexReg = NewExt;
2347 return false;
2348}
2349
2350// Transform "(X >> SHIFT) & (MASK << C1)" to
2351// "((X >> (SHIFT + C1)) & (MASK)) << C1". Everything before the SHL will be
2352// matched to a BEXTR later. Returns false if the simplification is performed.
2353static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N,
2354 uint64_t Mask,
2355 SDValue Shift, SDValue X,
2356 X86ISelAddressMode &AM,
2357 const X86Subtarget &Subtarget) {
2358 if (Shift.getOpcode() != ISD::SRL ||
2359 !isa<ConstantSDNode>(Val: Shift.getOperand(i: 1)) ||
2360 !Shift.hasOneUse() || !N.hasOneUse())
2361 return true;
2362
2363 // Only do this if BEXTR will be matched by matchBEXTRFromAndImm.
2364 if (!Subtarget.hasTBM() &&
2365 !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
2366 return true;
2367
2368 // We need to ensure that mask is a continuous run of bits.
2369 unsigned MaskIdx, MaskLen;
2370 if (!isShiftedMask_64(Value: Mask, MaskIdx, MaskLen))
2371 return true;
2372
2373 unsigned ShiftAmt = Shift.getConstantOperandVal(i: 1);
2374
2375 // The amount of shift we're trying to fit into the addressing mode is taken
2376 // from the shifted mask index (number of trailing zeros of the mask).
2377 unsigned AMShiftAmt = MaskIdx;
2378
2379 // There is nothing we can do here unless the mask is removing some bits.
2380 // Also, the addressing mode can only represent shifts of 1, 2, or 3 bits.
2381 if (AMShiftAmt == 0 || AMShiftAmt > 3) return true;
2382
2383 MVT XVT = X.getSimpleValueType();
2384 MVT VT = N.getSimpleValueType();
2385 SDLoc DL(N);
2386 SDValue NewSRLAmt = DAG.getConstant(Val: ShiftAmt + AMShiftAmt, DL, VT: MVT::i8);
2387 SDValue NewSRL = DAG.getNode(Opcode: ISD::SRL, DL, VT: XVT, N1: X, N2: NewSRLAmt);
2388 SDValue NewMask = DAG.getConstant(Val: Mask >> AMShiftAmt, DL, VT: XVT);
2389 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL, VT: XVT, N1: NewSRL, N2: NewMask);
2390 SDValue NewExt = DAG.getZExtOrTrunc(Op: NewAnd, DL, VT);
2391 SDValue NewSHLAmt = DAG.getConstant(Val: AMShiftAmt, DL, VT: MVT::i8);
2392 SDValue NewSHL = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: NewExt, N2: NewSHLAmt);
2393
2394 // Insert the new nodes into the topological ordering. We must do this in
2395 // a valid topological ordering as nothing is going to go back and re-sort
2396 // these nodes. We continually insert before 'N' in sequence as this is
2397 // essentially a pre-flattened and pre-sorted sequence of nodes. There is no
2398 // hierarchy left to express.
2399 insertDAGNode(DAG, Pos: N, N: NewSRLAmt);
2400 insertDAGNode(DAG, Pos: N, N: NewSRL);
2401 insertDAGNode(DAG, Pos: N, N: NewMask);
2402 insertDAGNode(DAG, Pos: N, N: NewAnd);
2403 insertDAGNode(DAG, Pos: N, N: NewExt);
2404 insertDAGNode(DAG, Pos: N, N: NewSHLAmt);
2405 insertDAGNode(DAG, Pos: N, N: NewSHL);
2406 DAG.ReplaceAllUsesWith(From: N, To: NewSHL);
2407 DAG.RemoveDeadNode(N: N.getNode());
2408
2409 AM.Scale = 1 << AMShiftAmt;
2410 AM.IndexReg = NewExt;
2411 return false;
2412}
2413
2414// Attempt to peek further into a scaled index register, collecting additional
2415// extensions / offsets / etc. Returns /p N if we can't peek any further.
2416SDValue X86DAGToDAGISel::matchIndexRecursively(SDValue N,
2417 X86ISelAddressMode &AM,
2418 unsigned Depth) {
2419 assert(AM.IndexReg.getNode() == nullptr && "IndexReg already matched");
2420 assert((AM.Scale == 1 || AM.Scale == 2 || AM.Scale == 4 || AM.Scale == 8) &&
2421 "Illegal index scale");
2422
2423 // Limit recursion.
2424 if (Depth >= SelectionDAG::MaxRecursionDepth)
2425 return N;
2426
2427 EVT VT = N.getValueType();
2428 unsigned Opc = N.getOpcode();
2429
2430 // index: add(x,c) -> index: x, disp + c
2431 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
2432 auto *AddVal = cast<ConstantSDNode>(Val: N.getOperand(i: 1));
2433 uint64_t Offset = (uint64_t)AddVal->getSExtValue() * AM.Scale;
2434 if (!foldOffsetIntoAddress(Offset, AM))
2435 return matchIndexRecursively(N: N.getOperand(i: 0), AM, Depth: Depth + 1);
2436 }
2437
2438 // index: add(x,x) -> index: x, scale * 2
2439 if (Opc == ISD::ADD && N.getOperand(i: 0) == N.getOperand(i: 1)) {
2440 if (AM.Scale <= 4) {
2441 AM.Scale *= 2;
2442 return matchIndexRecursively(N: N.getOperand(i: 0), AM, Depth: Depth + 1);
2443 }
2444 }
2445
2446 // index: shl(x,i) -> index: x, scale * (1 << i)
2447 if (Opc == X86ISD::VSHLI) {
2448 uint64_t ShiftAmt = N.getConstantOperandVal(i: 1);
2449 uint64_t ScaleAmt = 1ULL << ShiftAmt;
2450 if ((AM.Scale * ScaleAmt) <= 8) {
2451 AM.Scale *= ScaleAmt;
2452 return matchIndexRecursively(N: N.getOperand(i: 0), AM, Depth: Depth + 1);
2453 }
2454 }
2455
2456 // index: sext(add_nsw(x,c)) -> index: sext(x), disp + sext(c)
2457 // TODO: call matchIndexRecursively(AddSrc) if we won't corrupt sext?
2458 if (Opc == ISD::SIGN_EXTEND && !VT.isVector() && N.hasOneUse()) {
2459 SDValue Src = N.getOperand(i: 0);
2460 if (Src.getOpcode() == ISD::ADD && Src->getFlags().hasNoSignedWrap() &&
2461 Src.hasOneUse()) {
2462 if (CurDAG->isBaseWithConstantOffset(Op: Src)) {
2463 SDValue AddSrc = Src.getOperand(i: 0);
2464 auto *AddVal = cast<ConstantSDNode>(Val: Src.getOperand(i: 1));
2465 int64_t Offset = AddVal->getSExtValue();
2466 if (!foldOffsetIntoAddress(Offset: (uint64_t)Offset * AM.Scale, AM)) {
2467 SDLoc DL(N);
2468 SDValue ExtSrc = CurDAG->getNode(Opcode: Opc, DL, VT, Operand: AddSrc);
2469 SDValue ExtVal = CurDAG->getSignedConstant(Val: Offset, DL, VT);
2470 SDValue ExtAdd = CurDAG->getNode(Opcode: ISD::ADD, DL, VT, N1: ExtSrc, N2: ExtVal);
2471 insertDAGNode(DAG&: *CurDAG, Pos: N, N: ExtSrc);
2472 insertDAGNode(DAG&: *CurDAG, Pos: N, N: ExtVal);
2473 insertDAGNode(DAG&: *CurDAG, Pos: N, N: ExtAdd);
2474 CurDAG->ReplaceAllUsesWith(From: N, To: ExtAdd);
2475 CurDAG->RemoveDeadNode(N: N.getNode());
2476 return ExtSrc;
2477 }
2478 }
2479 }
2480 }
2481
2482 // index: zext(add_nuw(x,c)) -> index: zext(x), disp + zext(c)
2483 // index: zext(addlike(x,c)) -> index: zext(x), disp + zext(c)
2484 // TODO: call matchIndexRecursively(AddSrc) if we won't corrupt sext?
2485 if (Opc == ISD::ZERO_EXTEND && !VT.isVector() && N.hasOneUse()) {
2486 SDValue Src = N.getOperand(i: 0);
2487 unsigned SrcOpc = Src.getOpcode();
2488 if (((SrcOpc == ISD::ADD && Src->getFlags().hasNoUnsignedWrap()) ||
2489 CurDAG->isADDLike(Op: Src, /*NoWrap=*/true)) &&
2490 Src.hasOneUse()) {
2491 if (CurDAG->isBaseWithConstantOffset(Op: Src)) {
2492 SDValue AddSrc = Src.getOperand(i: 0);
2493 uint64_t Offset = Src.getConstantOperandVal(i: 1);
2494 if (!foldOffsetIntoAddress(Offset: Offset * AM.Scale, AM)) {
2495 SDLoc DL(N);
2496 SDValue Res;
2497 // If we're also scaling, see if we can use that as well.
2498 if (AddSrc.getOpcode() == ISD::SHL &&
2499 isa<ConstantSDNode>(Val: AddSrc.getOperand(i: 1))) {
2500 SDValue ShVal = AddSrc.getOperand(i: 0);
2501 uint64_t ShAmt = AddSrc.getConstantOperandVal(i: 1);
2502 APInt HiBits =
2503 APInt::getHighBitsSet(numBits: AddSrc.getScalarValueSizeInBits(), hiBitsSet: ShAmt);
2504 uint64_t ScaleAmt = 1ULL << ShAmt;
2505 if ((AM.Scale * ScaleAmt) <= 8 &&
2506 (AddSrc->getFlags().hasNoUnsignedWrap() ||
2507 CurDAG->MaskedValueIsZero(Op: ShVal, Mask: HiBits))) {
2508 AM.Scale *= ScaleAmt;
2509 SDValue ExtShVal = CurDAG->getNode(Opcode: Opc, DL, VT, Operand: ShVal);
2510 SDValue ExtShift = CurDAG->getNode(Opcode: ISD::SHL, DL, VT, N1: ExtShVal,
2511 N2: AddSrc.getOperand(i: 1));
2512 insertDAGNode(DAG&: *CurDAG, Pos: N, N: ExtShVal);
2513 insertDAGNode(DAG&: *CurDAG, Pos: N, N: ExtShift);
2514 AddSrc = ExtShift;
2515 Res = ExtShVal;
2516 }
2517 }
2518 SDValue ExtSrc = CurDAG->getNode(Opcode: Opc, DL, VT, Operand: AddSrc);
2519 SDValue ExtVal = CurDAG->getConstant(Val: Offset, DL, VT);
2520 SDValue ExtAdd = CurDAG->getNode(Opcode: SrcOpc, DL, VT, N1: ExtSrc, N2: ExtVal);
2521 insertDAGNode(DAG&: *CurDAG, Pos: N, N: ExtSrc);
2522 insertDAGNode(DAG&: *CurDAG, Pos: N, N: ExtVal);
2523 insertDAGNode(DAG&: *CurDAG, Pos: N, N: ExtAdd);
2524 CurDAG->ReplaceAllUsesWith(From: N, To: ExtAdd);
2525 CurDAG->RemoveDeadNode(N: N.getNode());
2526 return Res ? Res : ExtSrc;
2527 }
2528 }
2529 }
2530 }
2531
2532 // TODO: Handle extensions, shifted masks etc.
2533 return N;
2534}
2535
2536bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
2537 unsigned Depth) {
2538 LLVM_DEBUG({
2539 dbgs() << "MatchAddress: ";
2540 AM.dump(CurDAG);
2541 });
2542 // Limit recursion.
2543 if (Depth >= SelectionDAG::MaxRecursionDepth)
2544 return matchAddressBase(N, AM);
2545
2546 // If this is already a %rip relative address, we can only merge immediates
2547 // into it. Instead of handling this in every case, we handle it here.
2548 // RIP relative addressing: %rip + 32-bit displacement!
2549 if (AM.isRIPRelative()) {
2550 // FIXME: JumpTable and ExternalSymbol address currently don't like
2551 // displacements. It isn't very important, but this should be fixed for
2552 // consistency.
2553 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
2554 return true;
2555
2556 if (auto *Cst = dyn_cast<ConstantSDNode>(Val&: N))
2557 if (!foldOffsetIntoAddress(Offset: Cst->getSExtValue(), AM))
2558 return false;
2559 return true;
2560 }
2561
2562 switch (N.getOpcode()) {
2563 default: break;
2564 case ISD::LOCAL_RECOVER: {
2565 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
2566 if (const auto *ESNode = dyn_cast<MCSymbolSDNode>(Val: N.getOperand(i: 0))) {
2567 // Use the symbol and don't prefix it.
2568 AM.MCSym = ESNode->getMCSymbol();
2569 return false;
2570 }
2571 break;
2572 }
2573 case ISD::Constant: {
2574 uint64_t Val = cast<ConstantSDNode>(Val&: N)->getSExtValue();
2575 if (!foldOffsetIntoAddress(Offset: Val, AM))
2576 return false;
2577 break;
2578 }
2579
2580 case X86ISD::Wrapper:
2581 case X86ISD::WrapperRIP:
2582 if (!matchWrapper(N, AM))
2583 return false;
2584 break;
2585
2586 case ISD::LOAD:
2587 if (!matchLoadInAddress(N: cast<LoadSDNode>(Val&: N), AM))
2588 return false;
2589 break;
2590
2591 case ISD::FrameIndex:
2592 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2593 AM.Base_Reg.getNode() == nullptr &&
2594 (!Subtarget->is64Bit() || isDispSafeForFrameIndexOrRegBase(Val: AM.Disp))) {
2595 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
2596 AM.Base_FrameIndex = cast<FrameIndexSDNode>(Val&: N)->getIndex();
2597 return false;
2598 }
2599 break;
2600
2601 case ISD::SHL:
2602 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1)
2603 break;
2604
2605 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
2606 unsigned Val = CN->getZExtValue();
2607 // Note that we handle x<<1 as (,x,2) rather than (x,x) here so
2608 // that the base operand remains free for further matching. If
2609 // the base doesn't end up getting used, a post-processing step
2610 // in MatchAddress turns (,x,2) into (x,x), which is cheaper.
2611 if (Val == 1 || Val == 2 || Val == 3) {
2612 SDValue ShVal = N.getOperand(i: 0);
2613 AM.Scale = 1 << Val;
2614 AM.IndexReg = matchIndexRecursively(N: ShVal, AM, Depth: Depth + 1);
2615 return false;
2616 }
2617 }
2618 break;
2619
2620 case ISD::SRL: {
2621 // Scale must not be used already.
2622 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break;
2623
2624 // We only handle up to 64-bit values here as those are what matter for
2625 // addressing mode optimizations.
2626 assert(N.getSimpleValueType().getSizeInBits() <= 64 &&
2627 "Unexpected value size!");
2628
2629 SDValue And = N.getOperand(i: 0);
2630 if (And.getOpcode() != ISD::AND) break;
2631 SDValue X = And.getOperand(i: 0);
2632
2633 // The mask used for the transform is expected to be post-shift, but we
2634 // found the shift first so just apply the shift to the mask before passing
2635 // it down.
2636 if (!isa<ConstantSDNode>(Val: N.getOperand(i: 1)) ||
2637 !isa<ConstantSDNode>(Val: And.getOperand(i: 1)))
2638 break;
2639 uint64_t Mask = And.getConstantOperandVal(i: 1) >> N.getConstantOperandVal(i: 1);
2640
2641 // Try to fold the mask and shift into the scale, and return false if we
2642 // succeed.
2643 if (!foldMaskAndShiftToScale(DAG&: *CurDAG, N, Mask, Shift: N, X, AM))
2644 return false;
2645 break;
2646 }
2647
2648 case ISD::SMUL_LOHI:
2649 case ISD::UMUL_LOHI:
2650 // A mul_lohi where we need the low part can be folded as a plain multiply.
2651 if (N.getResNo() != 0) break;
2652 [[fallthrough]];
2653 case ISD::MUL:
2654 case X86ISD::MUL_IMM:
2655 // X*[3,5,9] -> X+X*[2,4,8]
2656 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2657 AM.Base_Reg.getNode() == nullptr &&
2658 AM.IndexReg.getNode() == nullptr) {
2659 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1)))
2660 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
2661 CN->getZExtValue() == 9) {
2662 AM.Scale = unsigned(CN->getZExtValue())-1;
2663
2664 SDValue MulVal = N.getOperand(i: 0);
2665 SDValue Reg;
2666
2667 // Okay, we know that we have a scale by now. However, if the scaled
2668 // value is an add of something and a constant, we can fold the
2669 // constant into the disp field here.
2670 if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() &&
2671 isa<ConstantSDNode>(Val: MulVal.getOperand(i: 1))) {
2672 Reg = MulVal.getOperand(i: 0);
2673 auto *AddVal = cast<ConstantSDNode>(Val: MulVal.getOperand(i: 1));
2674 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue();
2675 if (foldOffsetIntoAddress(Offset: Disp, AM))
2676 Reg = N.getOperand(i: 0);
2677 } else {
2678 Reg = N.getOperand(i: 0);
2679 }
2680
2681 AM.IndexReg = AM.Base_Reg = Reg;
2682 return false;
2683 }
2684 }
2685 break;
2686
2687 case ISD::SUB: {
2688 // Given A-B, if A can be completely folded into the address and
2689 // the index field with the index field unused, use -B as the index.
2690 // This is a win if a has multiple parts that can be folded into
2691 // the address. Also, this saves a mov if the base register has
2692 // other uses, since it avoids a two-address sub instruction, however
2693 // it costs an additional mov if the index register has other uses.
2694
2695 // Add an artificial use to this node so that we can keep track of
2696 // it if it gets CSE'd with a different node.
2697 HandleSDNode Handle(N);
2698
2699 // Test if the LHS of the sub can be folded.
2700 X86ISelAddressMode Backup = AM;
2701 if (matchAddressRecursively(N: N.getOperand(i: 0), AM, Depth: Depth+1)) {
2702 N = Handle.getValue();
2703 AM = Backup;
2704 break;
2705 }
2706 N = Handle.getValue();
2707 // Test if the index field is free for use.
2708 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
2709 AM = Backup;
2710 break;
2711 }
2712
2713 int Cost = 0;
2714 SDValue RHS = N.getOperand(i: 1);
2715 // If the RHS involves a register with multiple uses, this
2716 // transformation incurs an extra mov, due to the neg instruction
2717 // clobbering its operand.
2718 if (!RHS.getNode()->hasOneUse() ||
2719 RHS.getNode()->getOpcode() == ISD::CopyFromReg ||
2720 RHS.getNode()->getOpcode() == ISD::TRUNCATE ||
2721 RHS.getNode()->getOpcode() == ISD::ANY_EXTEND ||
2722 (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND &&
2723 RHS.getOperand(i: 0).getValueType() == MVT::i32))
2724 ++Cost;
2725 // If the base is a register with multiple uses, this
2726 // transformation may save a mov.
2727 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
2728 !AM.Base_Reg.getNode()->hasOneUse()) ||
2729 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
2730 --Cost;
2731 // If the folded LHS was interesting, this transformation saves
2732 // address arithmetic.
2733 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
2734 ((AM.Disp != 0) && (Backup.Disp == 0)) +
2735 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
2736 --Cost;
2737 // If it doesn't look like it may be an overall win, don't do it.
2738 if (Cost >= 0) {
2739 AM = Backup;
2740 break;
2741 }
2742
2743 // Ok, the transformation is legal and appears profitable. Go for it.
2744 // Negation will be emitted later to avoid creating dangling nodes if this
2745 // was an unprofitable LEA.
2746 AM.IndexReg = RHS;
2747 AM.NegateIndex = true;
2748 AM.Scale = 1;
2749 return false;
2750 }
2751
2752 case ISD::OR:
2753 case ISD::XOR:
2754 // See if we can treat the OR/XOR node as an ADD node.
2755 if (!CurDAG->isADDLike(Op: N))
2756 break;
2757 [[fallthrough]];
2758 case ISD::ADD:
2759 if (!matchAdd(N, AM, Depth))
2760 return false;
2761 break;
2762
2763 case ISD::AND: {
2764 // Perform some heroic transforms on an and of a constant-count shift
2765 // with a constant to enable use of the scaled offset field.
2766
2767 // Scale must not be used already.
2768 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break;
2769
2770 // We only handle up to 64-bit values here as those are what matter for
2771 // addressing mode optimizations.
2772 assert(N.getSimpleValueType().getSizeInBits() <= 64 &&
2773 "Unexpected value size!");
2774
2775 if (!isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
2776 break;
2777
2778 if (N.getOperand(i: 0).getOpcode() == ISD::SRL) {
2779 SDValue Shift = N.getOperand(i: 0);
2780 SDValue X = Shift.getOperand(i: 0);
2781
2782 uint64_t Mask = N.getConstantOperandVal(i: 1);
2783
2784 // Try to fold the mask and shift into an extract and scale.
2785 if (!foldMaskAndShiftToExtract(DAG&: *CurDAG, N, Mask, Shift, X, AM))
2786 return false;
2787
2788 // Try to fold the mask and shift directly into the scale.
2789 if (!foldMaskAndShiftToScale(DAG&: *CurDAG, N, Mask, Shift, X, AM))
2790 return false;
2791
2792 // Try to fold the mask and shift into BEXTR and scale.
2793 if (!foldMaskedShiftToBEXTR(DAG&: *CurDAG, N, Mask, Shift, X, AM, Subtarget: *Subtarget))
2794 return false;
2795 }
2796
2797 // Try to swap the mask and shift to place shifts which can be done as
2798 // a scale on the outside of the mask.
2799 if (!foldMaskedShiftToScaledMask(DAG&: *CurDAG, N, AM))
2800 return false;
2801
2802 break;
2803 }
2804 case ISD::ZERO_EXTEND: {
2805 // Try to widen a zexted shift left to the same size as its use, so we can
2806 // match the shift as a scale factor.
2807 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1)
2808 break;
2809
2810 SDValue Src = N.getOperand(i: 0);
2811
2812 // See if we can match a zext(addlike(x,c)).
2813 // TODO: Move more ZERO_EXTEND patterns into matchIndexRecursively.
2814 if (Src.getOpcode() == ISD::ADD || Src.getOpcode() == ISD::OR)
2815 if (SDValue Index = matchIndexRecursively(N, AM, Depth: Depth + 1))
2816 if (Index != N) {
2817 AM.IndexReg = Index;
2818 return false;
2819 }
2820
2821 // Peek through mask: zext(and(shl(x,c1),c2))
2822 APInt Mask = APInt::getAllOnes(numBits: Src.getScalarValueSizeInBits());
2823 if (Src.getOpcode() == ISD::AND && Src.hasOneUse())
2824 if (auto *MaskC = dyn_cast<ConstantSDNode>(Val: Src.getOperand(i: 1))) {
2825 Mask = MaskC->getAPIntValue();
2826 Src = Src.getOperand(i: 0);
2827 }
2828
2829 if (Src.getOpcode() == ISD::SHL && Src.hasOneUse() && N->hasOneUse()) {
2830 // Give up if the shift is not a valid scale factor [1,2,3].
2831 SDValue ShlSrc = Src.getOperand(i: 0);
2832 SDValue ShlAmt = Src.getOperand(i: 1);
2833 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val&: ShlAmt);
2834 if (!ShAmtC)
2835 break;
2836 unsigned ShAmtV = ShAmtC->getZExtValue();
2837 if (ShAmtV > 3)
2838 break;
2839
2840 // The narrow shift must only shift out zero bits (it must be 'nuw').
2841 // That makes it safe to widen to the destination type.
2842 APInt HighZeros =
2843 APInt::getHighBitsSet(numBits: ShlSrc.getValueSizeInBits(), hiBitsSet: ShAmtV);
2844 if (!Src->getFlags().hasNoUnsignedWrap() &&
2845 !CurDAG->MaskedValueIsZero(Op: ShlSrc, Mask: HighZeros & Mask))
2846 break;
2847
2848 // zext (shl nuw i8 %x, C1) to i32
2849 // --> shl (zext i8 %x to i32), (zext C1)
2850 // zext (and (shl nuw i8 %x, C1), C2) to i32
2851 // --> shl (zext i8 (and %x, C2 >> C1) to i32), (zext C1)
2852 MVT SrcVT = ShlSrc.getSimpleValueType();
2853 MVT VT = N.getSimpleValueType();
2854 SDLoc DL(N);
2855
2856 SDValue Res = ShlSrc;
2857 if (!Mask.isAllOnes()) {
2858 Res = CurDAG->getConstant(Val: Mask.lshr(shiftAmt: ShAmtV), DL, VT: SrcVT);
2859 insertDAGNode(DAG&: *CurDAG, Pos: N, N: Res);
2860 Res = CurDAG->getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: ShlSrc, N2: Res);
2861 insertDAGNode(DAG&: *CurDAG, Pos: N, N: Res);
2862 }
2863 SDValue Zext = CurDAG->getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Res);
2864 insertDAGNode(DAG&: *CurDAG, Pos: N, N: Zext);
2865 SDValue NewShl = CurDAG->getNode(Opcode: ISD::SHL, DL, VT, N1: Zext, N2: ShlAmt);
2866 insertDAGNode(DAG&: *CurDAG, Pos: N, N: NewShl);
2867 CurDAG->ReplaceAllUsesWith(From: N, To: NewShl);
2868 CurDAG->RemoveDeadNode(N: N.getNode());
2869
2870 // Convert the shift to scale factor.
2871 AM.Scale = 1 << ShAmtV;
2872 // If matchIndexRecursively is not called here,
2873 // Zext may be replaced by other nodes but later used to call a builder
2874 // method
2875 AM.IndexReg = matchIndexRecursively(N: Zext, AM, Depth: Depth + 1);
2876 return false;
2877 }
2878
2879 if (Src.getOpcode() == ISD::SRL && !Mask.isAllOnes()) {
2880 // Try to fold the mask and shift into an extract and scale.
2881 if (!foldMaskAndShiftToExtract(DAG&: *CurDAG, N, Mask: Mask.getZExtValue(), Shift: Src,
2882 X: Src.getOperand(i: 0), AM))
2883 return false;
2884
2885 // Try to fold the mask and shift directly into the scale.
2886 if (!foldMaskAndShiftToScale(DAG&: *CurDAG, N, Mask: Mask.getZExtValue(), Shift: Src,
2887 X: Src.getOperand(i: 0), AM))
2888 return false;
2889
2890 // Try to fold the mask and shift into BEXTR and scale.
2891 if (!foldMaskedShiftToBEXTR(DAG&: *CurDAG, N, Mask: Mask.getZExtValue(), Shift: Src,
2892 X: Src.getOperand(i: 0), AM, Subtarget: *Subtarget))
2893 return false;
2894 }
2895
2896 break;
2897 }
2898 }
2899
2900 return matchAddressBase(N, AM);
2901}
2902
2903/// Helper for MatchAddress. Add the specified node to the
2904/// specified addressing mode without any further recursion.
2905bool X86DAGToDAGISel::matchAddressBase(SDValue N, X86ISelAddressMode &AM) {
2906 // Is the base register already occupied?
2907 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
2908 // If so, check to see if the scale index register is set.
2909 if (!AM.IndexReg.getNode()) {
2910 AM.IndexReg = N;
2911 AM.Scale = 1;
2912 return false;
2913 }
2914
2915 // Otherwise, we cannot select it.
2916 return true;
2917 }
2918
2919 // Default, generate it as a register.
2920 AM.BaseType = X86ISelAddressMode::RegBase;
2921 AM.Base_Reg = N;
2922 return false;
2923}
2924
2925bool X86DAGToDAGISel::matchVectorAddressRecursively(SDValue N,
2926 X86ISelAddressMode &AM,
2927 unsigned Depth) {
2928 LLVM_DEBUG({
2929 dbgs() << "MatchVectorAddress: ";
2930 AM.dump(CurDAG);
2931 });
2932 // Limit recursion.
2933 if (Depth >= SelectionDAG::MaxRecursionDepth)
2934 return matchAddressBase(N, AM);
2935
2936 // TODO: Support other operations.
2937 switch (N.getOpcode()) {
2938 case ISD::Constant: {
2939 uint64_t Val = cast<ConstantSDNode>(Val&: N)->getSExtValue();
2940 if (!foldOffsetIntoAddress(Offset: Val, AM))
2941 return false;
2942 break;
2943 }
2944 case X86ISD::Wrapper:
2945 if (!matchWrapper(N, AM))
2946 return false;
2947 break;
2948 case ISD::ADD: {
2949 // Add an artificial use to this node so that we can keep track of
2950 // it if it gets CSE'd with a different node.
2951 HandleSDNode Handle(N);
2952
2953 X86ISelAddressMode Backup = AM;
2954 if (!matchVectorAddressRecursively(N: N.getOperand(i: 0), AM, Depth: Depth + 1) &&
2955 !matchVectorAddressRecursively(N: Handle.getValue().getOperand(i: 1), AM,
2956 Depth: Depth + 1))
2957 return false;
2958 AM = Backup;
2959
2960 // Try again after commuting the operands.
2961 if (!matchVectorAddressRecursively(N: Handle.getValue().getOperand(i: 1), AM,
2962 Depth: Depth + 1) &&
2963 !matchVectorAddressRecursively(N: Handle.getValue().getOperand(i: 0), AM,
2964 Depth: Depth + 1))
2965 return false;
2966 AM = Backup;
2967
2968 N = Handle.getValue();
2969 break;
2970 }
2971 }
2972
2973 return matchAddressBase(N, AM);
2974}
2975
2976/// Helper for selectVectorAddr. Handles things that can be folded into a
2977/// gather/scatter address. The index register and scale should have already
2978/// been handled.
2979bool X86DAGToDAGISel::matchVectorAddress(SDValue N, X86ISelAddressMode &AM) {
2980 return matchVectorAddressRecursively(N, AM, Depth: 0);
2981}
2982
2983bool X86DAGToDAGISel::selectVectorAddr(MemSDNode *Parent, SDValue BasePtr,
2984 SDValue IndexOp, SDValue ScaleOp,
2985 SDValue &Base, SDValue &Scale,
2986 SDValue &Index, SDValue &Disp,
2987 SDValue &Segment) {
2988 X86ISelAddressMode AM;
2989 AM.Scale = ScaleOp->getAsZExtVal();
2990
2991 // Attempt to match index patterns, as long as we're not relying on implicit
2992 // sign-extension, which is performed BEFORE scale.
2993 if (IndexOp.getScalarValueSizeInBits() == BasePtr.getScalarValueSizeInBits())
2994 AM.IndexReg = matchIndexRecursively(N: IndexOp, AM, Depth: 0);
2995 else
2996 AM.IndexReg = IndexOp;
2997
2998 unsigned AddrSpace = Parent->getPointerInfo().getAddrSpace();
2999 if (AddrSpace == X86AS::GS)
3000 AM.Segment = CurDAG->getRegister(Reg: X86::GS, VT: MVT::i16);
3001 if (AddrSpace == X86AS::FS)
3002 AM.Segment = CurDAG->getRegister(Reg: X86::FS, VT: MVT::i16);
3003 if (AddrSpace == X86AS::SS)
3004 AM.Segment = CurDAG->getRegister(Reg: X86::SS, VT: MVT::i16);
3005
3006 SDLoc DL(BasePtr);
3007 MVT VT = BasePtr.getSimpleValueType();
3008
3009 // Try to match into the base and displacement fields.
3010 if (matchVectorAddress(N: BasePtr, AM))
3011 return false;
3012
3013 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment);
3014 return true;
3015}
3016
3017/// Returns true if it is able to pattern match an addressing mode.
3018/// It returns the operands which make up the maximal addressing mode it can
3019/// match by reference.
3020///
3021/// Parent is the parent node of the addr operand that is being matched. It
3022/// is always a load, store, atomic node, or null. It is only null when
3023/// checking memory operands for inline asm nodes.
3024bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base,
3025 SDValue &Scale, SDValue &Index, SDValue &Disp,
3026 SDValue &Segment, bool HasNDDM) {
3027 X86ISelAddressMode AM;
3028
3029 if (Parent &&
3030 // This list of opcodes are all the nodes that have an "addr:$ptr" operand
3031 // that are not a MemSDNode, and thus don't have proper addrspace info.
3032 Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme
3033 Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores
3034 Parent->getOpcode() != X86ISD::TLSCALL && // Fixme
3035 Parent->getOpcode() != X86ISD::ENQCMD && // Fixme
3036 Parent->getOpcode() != X86ISD::ENQCMDS && // Fixme
3037 Parent->getOpcode() != X86ISD::EH_SJLJ_SETJMP && // setjmp
3038 Parent->getOpcode() != X86ISD::EH_SJLJ_LONGJMP) { // longjmp
3039 unsigned AddrSpace =
3040 cast<MemSDNode>(Val: Parent)->getPointerInfo().getAddrSpace();
3041 if (AddrSpace == X86AS::GS)
3042 AM.Segment = CurDAG->getRegister(Reg: X86::GS, VT: MVT::i16);
3043 if (AddrSpace == X86AS::FS)
3044 AM.Segment = CurDAG->getRegister(Reg: X86::FS, VT: MVT::i16);
3045 if (AddrSpace == X86AS::SS)
3046 AM.Segment = CurDAG->getRegister(Reg: X86::SS, VT: MVT::i16);
3047 }
3048
3049 // Save the DL and VT before calling matchAddress, it can invalidate N.
3050 SDLoc DL(N);
3051 MVT VT = N.getSimpleValueType();
3052
3053 if (matchAddress(N, AM))
3054 return false;
3055
3056 if (!HasNDDM && !AM.isRIPRelative())
3057 return false;
3058
3059 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment);
3060 return true;
3061}
3062
3063bool X86DAGToDAGISel::selectNDDAddr(SDNode *Parent, SDValue N, SDValue &Base,
3064 SDValue &Scale, SDValue &Index,
3065 SDValue &Disp, SDValue &Segment) {
3066 return selectAddr(Parent, N, Base, Scale, Index, Disp, Segment,
3067 HasNDDM: Subtarget->hasNDDM());
3068}
3069
3070bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) {
3071 // Cannot use 32 bit constants to reference objects in kernel/large code
3072 // model.
3073 if (TM.getCodeModel() == CodeModel::Kernel ||
3074 TM.getCodeModel() == CodeModel::Large)
3075 return false;
3076
3077 // In static codegen with small code model, we can get the address of a label
3078 // into a register with 'movl'
3079 if (N->getOpcode() != X86ISD::Wrapper)
3080 return false;
3081
3082 N = N.getOperand(i: 0);
3083
3084 // At least GNU as does not accept 'movl' for TPOFF relocations.
3085 // FIXME: We could use 'movl' when we know we are targeting MC.
3086 if (N->getOpcode() == ISD::TargetGlobalTLSAddress)
3087 return false;
3088
3089 Imm = N;
3090 // Small/medium code model can reference non-TargetGlobalAddress objects with
3091 // 32 bit constants.
3092 if (N->getOpcode() != ISD::TargetGlobalAddress) {
3093 return TM.getCodeModel() == CodeModel::Small ||
3094 TM.getCodeModel() == CodeModel::Medium;
3095 }
3096
3097 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val&: N)->getGlobal();
3098 if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
3099 return CR->getUnsignedMax().ult(RHS: 1ull << 32);
3100
3101 return !TM.isLargeGlobalValue(GV);
3102}
3103
3104bool X86DAGToDAGISel::selectLEA64_Addr(SDValue N, SDValue &Base, SDValue &Scale,
3105 SDValue &Index, SDValue &Disp,
3106 SDValue &Segment) {
3107 // Save the debug loc before calling selectLEAAddr, in case it invalidates N.
3108 SDLoc DL(N);
3109
3110 if (!selectLEAAddr(N, Base, Scale, Index, Disp, Segment))
3111 return false;
3112
3113 EVT BaseType = Base.getValueType();
3114 unsigned SubReg;
3115 if (BaseType == MVT::i8)
3116 SubReg = X86::sub_8bit;
3117 else if (BaseType == MVT::i16)
3118 SubReg = X86::sub_16bit;
3119 else
3120 SubReg = X86::sub_32bit;
3121
3122 auto *RN = dyn_cast<RegisterSDNode>(Val&: Base);
3123 if (RN && RN->getReg() == 0)
3124 Base = CurDAG->getRegister(Reg: 0, VT: MVT::i64);
3125 else if ((BaseType == MVT::i8 || BaseType == MVT::i16 ||
3126 BaseType == MVT::i32) &&
3127 !isa<FrameIndexSDNode>(Val: Base)) {
3128 // Base could already be %rip, particularly in the x32 ABI.
3129 SDValue ImplDef = SDValue(CurDAG->getMachineNode(Opcode: X86::IMPLICIT_DEF, dl: DL,
3130 VT: MVT::i64), 0);
3131 Base = CurDAG->getTargetInsertSubreg(SRIdx: SubReg, DL, VT: MVT::i64, Operand: ImplDef, Subreg: Base);
3132 }
3133
3134 [[maybe_unused]] EVT IndexType = Index.getValueType();
3135 RN = dyn_cast<RegisterSDNode>(Val&: Index);
3136 if (RN && RN->getReg() == 0)
3137 Index = CurDAG->getRegister(Reg: 0, VT: MVT::i64);
3138 else {
3139 assert((IndexType == BaseType) &&
3140 "Expect to be extending 8/16/32-bit registers for use in LEA");
3141 SDValue ImplDef = SDValue(CurDAG->getMachineNode(Opcode: X86::IMPLICIT_DEF, dl: DL,
3142 VT: MVT::i64), 0);
3143 Index = CurDAG->getTargetInsertSubreg(SRIdx: SubReg, DL, VT: MVT::i64, Operand: ImplDef, Subreg: Index);
3144 }
3145
3146 return true;
3147}
3148
3149/// Calls SelectAddr and determines if the maximal addressing
3150/// mode it matches can be cost effectively emitted as an LEA instruction.
3151bool X86DAGToDAGISel::selectLEAAddr(SDValue N,
3152 SDValue &Base, SDValue &Scale,
3153 SDValue &Index, SDValue &Disp,
3154 SDValue &Segment) {
3155 X86ISelAddressMode AM;
3156
3157 // Save the DL and VT before calling matchAddress, it can invalidate N.
3158 SDLoc DL(N);
3159 MVT VT = N.getSimpleValueType();
3160
3161 // Set AM.Segment to prevent MatchAddress from using one. LEA doesn't support
3162 // segments.
3163 SDValue Copy = AM.Segment;
3164 SDValue T = CurDAG->getRegister(Reg: 0, VT: MVT::i32);
3165 AM.Segment = T;
3166 if (matchAddress(N, AM))
3167 return false;
3168 assert (T == AM.Segment);
3169 AM.Segment = Copy;
3170
3171 unsigned Complexity = 0;
3172 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode())
3173 Complexity = 1;
3174 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
3175 Complexity = 4;
3176
3177 if (AM.IndexReg.getNode())
3178 Complexity++;
3179
3180 // Don't match just leal(,%reg,2). It's cheaper to do addl %reg, %reg, or with
3181 // a simple shift.
3182 if (AM.Scale > 1)
3183 Complexity++;
3184
3185 // FIXME: We are artificially lowering the criteria to turn ADD %reg, $GA
3186 // to a LEA. This is determined with some experimentation but is by no means
3187 // optimal (especially for code size consideration). LEA is nice because of
3188 // its three-address nature. Tweak the cost function again when we can run
3189 // convertToThreeAddress() at register allocation time.
3190 if (AM.hasSymbolicDisplacement()) {
3191 // For X86-64, always use LEA to materialize RIP-relative addresses.
3192 if (Subtarget->is64Bit())
3193 Complexity = 4;
3194 else
3195 Complexity += 2;
3196 }
3197
3198 // Heuristic: try harder to form an LEA from ADD if the operands set flags.
3199 // Unlike ADD, LEA does not affect flags, so we will be less likely to require
3200 // duplicating flag-producing instructions later in the pipeline.
3201 if (N.getOpcode() == ISD::ADD) {
3202 auto isMathWithFlags = [](SDValue V) {
3203 switch (V.getOpcode()) {
3204 case X86ISD::ADD:
3205 case X86ISD::SUB:
3206 case X86ISD::ADC:
3207 case X86ISD::SBB:
3208 case X86ISD::SMUL:
3209 case X86ISD::UMUL:
3210 /* TODO: These opcodes can be added safely, but we may want to justify
3211 their inclusion for different reasons (better for reg-alloc).
3212 case X86ISD::OR:
3213 case X86ISD::XOR:
3214 case X86ISD::AND:
3215 */
3216 // Value 1 is the flag output of the node - verify it's not dead.
3217 return !SDValue(V.getNode(), 1).use_empty();
3218 default:
3219 return false;
3220 }
3221 };
3222 // TODO: We might want to factor in whether there's a load folding
3223 // opportunity for the math op that disappears with LEA.
3224 if (isMathWithFlags(N.getOperand(i: 0)) || isMathWithFlags(N.getOperand(i: 1)))
3225 Complexity++;
3226 }
3227
3228 if (AM.Disp)
3229 Complexity++;
3230
3231 // If it isn't worth using an LEA, reject it.
3232 if (Complexity <= 2)
3233 return false;
3234
3235 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment);
3236 return true;
3237}
3238
3239/// This is only run on TargetGlobalTLSAddress nodes.
3240bool X86DAGToDAGISel::selectTLSADDRAddr(SDValue N, SDValue &Base,
3241 SDValue &Scale, SDValue &Index,
3242 SDValue &Disp, SDValue &Segment) {
3243 assert(N.getOpcode() == ISD::TargetGlobalTLSAddress ||
3244 N.getOpcode() == ISD::TargetExternalSymbol);
3245
3246 X86ISelAddressMode AM;
3247 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N)) {
3248 AM.GV = GA->getGlobal();
3249 AM.Disp += GA->getOffset();
3250 AM.SymbolFlags = GA->getTargetFlags();
3251 } else {
3252 auto *SA = cast<ExternalSymbolSDNode>(Val&: N);
3253 AM.ES = SA->getSymbol();
3254 AM.SymbolFlags = SA->getTargetFlags();
3255 }
3256
3257 if (Subtarget->is32Bit()) {
3258 AM.Scale = 1;
3259 AM.IndexReg = CurDAG->getRegister(Reg: X86::EBX, VT: MVT::i32);
3260 }
3261
3262 MVT VT = N.getSimpleValueType();
3263 getAddressOperands(AM, DL: SDLoc(N), VT, Base, Scale, Index, Disp, Segment);
3264 return true;
3265}
3266
3267bool X86DAGToDAGISel::selectRelocImm(SDValue N, SDValue &Op) {
3268 // Keep track of the original value type and whether this value was
3269 // truncated. If we see a truncation from pointer type to VT that truncates
3270 // bits that are known to be zero, we can use a narrow reference.
3271 EVT VT = N.getValueType();
3272 bool WasTruncated = false;
3273 if (N.getOpcode() == ISD::TRUNCATE) {
3274 WasTruncated = true;
3275 N = N.getOperand(i: 0);
3276 }
3277
3278 if (N.getOpcode() != X86ISD::Wrapper)
3279 return false;
3280
3281 // We can only use non-GlobalValues as immediates if they were not truncated,
3282 // as we do not have any range information. If we have a GlobalValue and the
3283 // address was not truncated, we can select it as an operand directly.
3284 unsigned Opc = N.getOperand(i: 0)->getOpcode();
3285 if (Opc != ISD::TargetGlobalAddress || !WasTruncated) {
3286 Op = N.getOperand(i: 0);
3287 // We can only select the operand directly if we didn't have to look past a
3288 // truncate.
3289 return !WasTruncated;
3290 }
3291
3292 // Check that the global's range fits into VT.
3293 auto *GA = cast<GlobalAddressSDNode>(Val: N.getOperand(i: 0));
3294 std::optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
3295 if (!CR || CR->getUnsignedMax().uge(RHS: 1ull << VT.getSizeInBits()))
3296 return false;
3297
3298 // Okay, we can use a narrow reference.
3299 Op = CurDAG->getTargetGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(N), VT,
3300 offset: GA->getOffset(), TargetFlags: GA->getTargetFlags());
3301 return true;
3302}
3303
3304bool X86DAGToDAGISel::tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
3305 SDValue &Base, SDValue &Scale,
3306 SDValue &Index, SDValue &Disp,
3307 SDValue &Segment) {
3308 assert(Root && P && "Unknown root/parent nodes");
3309 if (!ISD::isNON_EXTLoad(N: N.getNode()) ||
3310 !IsProfitableToFold(N, U: P, Root) ||
3311 !IsLegalToFold(N, U: P, Root, OptLevel))
3312 return false;
3313
3314 return selectAddr(Parent: N.getNode(),
3315 N: N.getOperand(i: 1), Base, Scale, Index, Disp, Segment);
3316}
3317
3318bool X86DAGToDAGISel::tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
3319 SDValue &Base, SDValue &Scale,
3320 SDValue &Index, SDValue &Disp,
3321 SDValue &Segment) {
3322 assert(Root && P && "Unknown root/parent nodes");
3323 if (N->getOpcode() != X86ISD::VBROADCAST_LOAD ||
3324 !IsProfitableToFold(N, U: P, Root) ||
3325 !IsLegalToFold(N, U: P, Root, OptLevel))
3326 return false;
3327
3328 return selectAddr(Parent: N.getNode(),
3329 N: N.getOperand(i: 1), Base, Scale, Index, Disp, Segment);
3330}
3331
3332/// Return an SDNode that returns the value of the global base register.
3333/// Output instructions required to initialize the global base register,
3334/// if necessary.
3335SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
3336 Register GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF);
3337 auto &DL = MF->getDataLayout();
3338 return CurDAG->getRegister(Reg: GlobalBaseReg, VT: TLI->getPointerTy(DL)).getNode();
3339}
3340
3341bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(unsigned Width, SDNode *N) const {
3342 if (N->getOpcode() == ISD::TRUNCATE)
3343 N = N->getOperand(Num: 0).getNode();
3344 if (N->getOpcode() != X86ISD::Wrapper)
3345 return false;
3346
3347 auto *GA = dyn_cast<GlobalAddressSDNode>(Val: N->getOperand(Num: 0));
3348 if (!GA)
3349 return false;
3350
3351 auto *GV = GA->getGlobal();
3352 std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange();
3353 if (CR)
3354 return CR->getSignedMin().sge(RHS: -1ull << Width) &&
3355 CR->getSignedMax().slt(RHS: 1ull << Width);
3356 // In the kernel code model, globals are in the negative 2GB of the address
3357 // space, so globals can be a sign extended 32-bit immediate.
3358 // In other code models, small globals are in the low 2GB of the address
3359 // space, so sign extending them is equivalent to zero extending them.
3360 return TM.getCodeModel() != CodeModel::Large && Width == 32 &&
3361 !TM.isLargeGlobalValue(GV);
3362}
3363
3364X86::CondCode X86DAGToDAGISel::getCondFromNode(SDNode *N) const {
3365 assert(N->isMachineOpcode() && "Unexpected node");
3366 unsigned Opc = N->getMachineOpcode();
3367 const MCInstrDesc &MCID = getInstrInfo()->get(Opcode: Opc);
3368 int CondNo = X86::getCondSrcNoFromDesc(MCID);
3369 if (CondNo < 0)
3370 return X86::COND_INVALID;
3371
3372 return static_cast<X86::CondCode>(N->getConstantOperandVal(Num: CondNo));
3373}
3374
3375/// Test whether the given X86ISD::CMP node has any users that use a flag
3376/// other than ZF.
3377bool X86DAGToDAGISel::onlyUsesZeroFlag(SDValue Flags) const {
3378 // Examine each user of the node.
3379 for (SDUse &Use : Flags->uses()) {
3380 // Only check things that use the flags.
3381 if (Use.getResNo() != Flags.getResNo())
3382 continue;
3383 SDNode *User = Use.getUser();
3384 // Only examine CopyToReg uses that copy to EFLAGS.
3385 if (User->getOpcode() != ISD::CopyToReg ||
3386 cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg() != X86::EFLAGS)
3387 return false;
3388 // Examine each user of the CopyToReg use.
3389 for (SDUse &FlagUse : User->uses()) {
3390 // Only examine the Flag result.
3391 if (FlagUse.getResNo() != 1)
3392 continue;
3393 // Anything unusual: assume conservatively.
3394 if (!FlagUse.getUser()->isMachineOpcode())
3395 return false;
3396 // Examine the condition code of the user.
3397 X86::CondCode CC = getCondFromNode(N: FlagUse.getUser());
3398
3399 switch (CC) {
3400 // Comparisons which only use the zero flag.
3401 case X86::COND_E: case X86::COND_NE:
3402 continue;
3403 // Anything else: assume conservatively.
3404 default:
3405 return false;
3406 }
3407 }
3408 }
3409 return true;
3410}
3411
3412/// Test whether the given X86ISD::CMP node has any uses which require the SF
3413/// flag to be accurate.
3414bool X86DAGToDAGISel::hasNoSignFlagUses(SDValue Flags) const {
3415 // Examine each user of the node.
3416 for (SDUse &Use : Flags->uses()) {
3417 // Only check things that use the flags.
3418 if (Use.getResNo() != Flags.getResNo())
3419 continue;
3420 SDNode *User = Use.getUser();
3421 // Only examine CopyToReg uses that copy to EFLAGS.
3422 if (User->getOpcode() != ISD::CopyToReg ||
3423 cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg() != X86::EFLAGS)
3424 return false;
3425 // Examine each user of the CopyToReg use.
3426 for (SDUse &FlagUse : User->uses()) {
3427 // Only examine the Flag result.
3428 if (FlagUse.getResNo() != 1)
3429 continue;
3430 // Anything unusual: assume conservatively.
3431 if (!FlagUse.getUser()->isMachineOpcode())
3432 return false;
3433 // Examine the condition code of the user.
3434 X86::CondCode CC = getCondFromNode(N: FlagUse.getUser());
3435
3436 switch (CC) {
3437 // Comparisons which don't examine the SF flag.
3438 case X86::COND_A: case X86::COND_AE:
3439 case X86::COND_B: case X86::COND_BE:
3440 case X86::COND_E: case X86::COND_NE:
3441 case X86::COND_O: case X86::COND_NO:
3442 case X86::COND_P: case X86::COND_NP:
3443 continue;
3444 // Anything else: assume conservatively.
3445 default:
3446 return false;
3447 }
3448 }
3449 }
3450 return true;
3451}
3452
3453static bool mayUseCarryFlag(X86::CondCode CC) {
3454 switch (CC) {
3455 // Comparisons which don't examine the CF flag.
3456 case X86::COND_O: case X86::COND_NO:
3457 case X86::COND_E: case X86::COND_NE:
3458 case X86::COND_S: case X86::COND_NS:
3459 case X86::COND_P: case X86::COND_NP:
3460 case X86::COND_L: case X86::COND_GE:
3461 case X86::COND_G: case X86::COND_LE:
3462 return false;
3463 // Anything else: assume conservatively.
3464 default:
3465 return true;
3466 }
3467}
3468
3469/// Test whether the given node which sets flags has any uses which require the
3470/// CF flag to be accurate.
3471 bool X86DAGToDAGISel::hasNoCarryFlagUses(SDValue Flags) const {
3472 // Examine each user of the node.
3473 for (SDUse &Use : Flags->uses()) {
3474 // Only check things that use the flags.
3475 if (Use.getResNo() != Flags.getResNo())
3476 continue;
3477
3478 SDNode *User = Use.getUser();
3479 unsigned UserOpc = User->getOpcode();
3480
3481 if (UserOpc == ISD::CopyToReg) {
3482 // Only examine CopyToReg uses that copy to EFLAGS.
3483 if (cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg() != X86::EFLAGS)
3484 return false;
3485 // Examine each user of the CopyToReg use.
3486 for (SDUse &FlagUse : User->uses()) {
3487 // Only examine the Flag result.
3488 if (FlagUse.getResNo() != 1)
3489 continue;
3490 // Anything unusual: assume conservatively.
3491 if (!FlagUse.getUser()->isMachineOpcode())
3492 return false;
3493 // Examine the condition code of the user.
3494 X86::CondCode CC = getCondFromNode(N: FlagUse.getUser());
3495
3496 if (mayUseCarryFlag(CC))
3497 return false;
3498 }
3499
3500 // This CopyToReg is ok. Move on to the next user.
3501 continue;
3502 }
3503
3504 // This might be an unselected node. So look for the pre-isel opcodes that
3505 // use flags.
3506 unsigned CCOpNo;
3507 switch (UserOpc) {
3508 default:
3509 // Something unusual. Be conservative.
3510 return false;
3511 case X86ISD::SETCC: CCOpNo = 0; break;
3512 case X86ISD::SETCC_CARRY: CCOpNo = 0; break;
3513 case X86ISD::CMOV: CCOpNo = 2; break;
3514 case X86ISD::BRCOND: CCOpNo = 2; break;
3515 }
3516
3517 X86::CondCode CC = (X86::CondCode)User->getConstantOperandVal(Num: CCOpNo);
3518 if (mayUseCarryFlag(CC))
3519 return false;
3520 }
3521 return true;
3522}
3523
3524bool X86DAGToDAGISel::checkTCRetEnoughRegs(SDNode *N) const {
3525 // Check that there is enough volatile registers to load the callee address.
3526
3527 const X86RegisterInfo *RI = Subtarget->getRegisterInfo();
3528 unsigned AvailGPRs;
3529 // The register classes below must stay in sync with what's used for
3530 // TCRETURNri, TCRETURN_HIPE32ri, TCRETURN_WIN64ri, etc).
3531 if (Subtarget->is64Bit()) {
3532 const TargetRegisterClass *TCGPRs =
3533 Subtarget->isCallingConvWin64(CC: MF->getFunction().getCallingConv())
3534 ? &X86::GR64_TCW64RegClass
3535 : &X86::GR64_TCRegClass;
3536 // Can't use RSP or RIP for the load in general.
3537 assert(TCGPRs->contains(X86::RSP));
3538 assert(TCGPRs->contains(X86::RIP));
3539 AvailGPRs = TCGPRs->getNumRegs() - 2;
3540 } else {
3541 const TargetRegisterClass *TCGPRs =
3542 MF->getFunction().getCallingConv() == CallingConv::HiPE
3543 ? &X86::GR32RegClass
3544 : &X86::GR32_TCRegClass;
3545 // Can't use ESP for the address in general.
3546 assert(TCGPRs->contains(X86::ESP));
3547 AvailGPRs = TCGPRs->getNumRegs() - 1;
3548 }
3549
3550 // The load's base and index need up to two registers.
3551 unsigned LoadGPRs = 2;
3552
3553 assert(N->getOpcode() == X86ISD::TC_RETURN);
3554 // X86tcret args: (*chain, ptr, imm, regs..., glue)
3555
3556 if (Subtarget->is32Bit()) {
3557 // FIXME: This was carried from X86tcret_1reg which was used for 32-bit,
3558 // but it could apply to 64-bit too.
3559 const SDValue &BasePtr = cast<LoadSDNode>(Val: N->getOperand(Num: 1))->getBasePtr();
3560 if (isa<FrameIndexSDNode>(Val: BasePtr)) {
3561 LoadGPRs -= 2; // Base is fixed index off ESP; no regs needed.
3562 } else if (BasePtr.getOpcode() == X86ISD::Wrapper &&
3563 isa<GlobalAddressSDNode>(Val: BasePtr->getOperand(Num: 0))) {
3564 if (getTargetMachine().isPositionIndependent())
3565 return false;
3566 LoadGPRs -= 1; // Base is a global (immediate since this is non-PIC), no
3567 // reg needed.
3568 }
3569 }
3570
3571 unsigned ArgGPRs = 0;
3572 for (unsigned I = 3, E = N->getNumOperands(); I != E; ++I) {
3573 if (const auto *RN = dyn_cast<RegisterSDNode>(Val: N->getOperand(Num: I))) {
3574 if (!RI->isGeneralPurposeRegister(*MF, RN->getReg()))
3575 continue;
3576 if (++ArgGPRs + LoadGPRs > AvailGPRs)
3577 return false;
3578 }
3579 }
3580
3581 return true;
3582}
3583
3584/// Check whether or not the chain ending in StoreNode is suitable for doing
3585/// the {load; op; store} to modify transformation.
3586static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode,
3587 SDValue StoredVal, SelectionDAG *CurDAG,
3588 unsigned LoadOpNo,
3589 LoadSDNode *&LoadNode,
3590 SDValue &InputChain) {
3591 // Is the stored value result 0 of the operation?
3592 if (StoredVal.getResNo() != 0) return false;
3593
3594 // Are there other uses of the operation other than the store?
3595 if (!StoredVal.getNode()->hasNUsesOfValue(NUses: 1, Value: 0)) return false;
3596
3597 // Is the store non-extending and non-indexed?
3598 if (!ISD::isNormalStore(N: StoreNode) || StoreNode->isNonTemporal())
3599 return false;
3600
3601 SDValue Load = StoredVal->getOperand(Num: LoadOpNo);
3602 // Is the stored value a non-extending and non-indexed load?
3603 if (!ISD::isNormalLoad(N: Load.getNode())) return false;
3604
3605 // Return LoadNode by reference.
3606 LoadNode = cast<LoadSDNode>(Val&: Load);
3607
3608 // Is store the only read of the loaded value?
3609 if (!Load.hasOneUse())
3610 return false;
3611
3612 // Is the address of the store the same as the load?
3613 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
3614 LoadNode->getOffset() != StoreNode->getOffset())
3615 return false;
3616
3617 bool FoundLoad = false;
3618 SmallVector<SDValue, 4> ChainOps;
3619 SmallVector<const SDNode *, 4> LoopWorklist;
3620 SmallPtrSet<const SDNode *, 16> Visited;
3621 const unsigned int Max = 1024;
3622
3623 // Visualization of Load-Op-Store fusion:
3624 // -------------------------
3625 // Legend:
3626 // *-lines = Chain operand dependencies.
3627 // |-lines = Normal operand dependencies.
3628 // Dependencies flow down and right. n-suffix references multiple nodes.
3629 //
3630 // C Xn C
3631 // * * *
3632 // * * *
3633 // Xn A-LD Yn TF Yn
3634 // * * \ | * |
3635 // * * \ | * |
3636 // * * \ | => A--LD_OP_ST
3637 // * * \| \
3638 // TF OP \
3639 // * | \ Zn
3640 // * | \
3641 // A-ST Zn
3642 //
3643
3644 // This merge induced dependences from: #1: Xn -> LD, OP, Zn
3645 // #2: Yn -> LD
3646 // #3: ST -> Zn
3647
3648 // Ensure the transform is safe by checking for the dual
3649 // dependencies to make sure we do not induce a loop.
3650
3651 // As LD is a predecessor to both OP and ST we can do this by checking:
3652 // a). if LD is a predecessor to a member of Xn or Yn.
3653 // b). if a Zn is a predecessor to ST.
3654
3655 // However, (b) can only occur through being a chain predecessor to
3656 // ST, which is the same as Zn being a member or predecessor of Xn,
3657 // which is a subset of LD being a predecessor of Xn. So it's
3658 // subsumed by check (a).
3659
3660 SDValue Chain = StoreNode->getChain();
3661
3662 // Gather X elements in ChainOps.
3663 if (Chain == Load.getValue(R: 1)) {
3664 FoundLoad = true;
3665 ChainOps.push_back(Elt: Load.getOperand(i: 0));
3666 } else if (Chain.getOpcode() == ISD::TokenFactor) {
3667 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
3668 SDValue Op = Chain.getOperand(i);
3669 if (Op == Load.getValue(R: 1)) {
3670 FoundLoad = true;
3671 // Drop Load, but keep its chain. No cycle check necessary.
3672 ChainOps.push_back(Elt: Load.getOperand(i: 0));
3673 continue;
3674 }
3675 LoopWorklist.push_back(Elt: Op.getNode());
3676 ChainOps.push_back(Elt: Op);
3677 }
3678 }
3679
3680 if (!FoundLoad)
3681 return false;
3682
3683 // Worklist is currently Xn. Add Yn to worklist.
3684 for (SDValue Op : StoredVal->ops())
3685 if (Op.getNode() != LoadNode)
3686 LoopWorklist.push_back(Elt: Op.getNode());
3687
3688 // Check (a) if Load is a predecessor to Xn + Yn
3689 if (SDNode::hasPredecessorHelper(N: Load.getNode(), Visited, Worklist&: LoopWorklist, MaxSteps: Max,
3690 TopologicalPrune: true))
3691 return false;
3692
3693 InputChain =
3694 CurDAG->getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ChainOps);
3695 return true;
3696}
3697
3698// Change a chain of {load; op; store} of the same value into a simple op
3699// through memory of that value, if the uses of the modified value and its
3700// address are suitable.
3701//
3702// The tablegen pattern memory operand pattern is currently not able to match
3703// the case where the EFLAGS on the original operation are used.
3704//
3705// To move this to tablegen, we'll need to improve tablegen to allow flags to
3706// be transferred from a node in the pattern to the result node, probably with
3707// a new keyword. For example, we have this
3708// def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst",
3709// [(store (add (loadi64 addr:$dst), -1), addr:$dst)]>;
3710// but maybe need something like this
3711// def DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst), "dec{q}\t$dst",
3712// [(store (X86add_flag (loadi64 addr:$dst), -1), addr:$dst),
3713// (transferrable EFLAGS)]>;
3714//
3715// Until then, we manually fold these and instruction select the operation
3716// here.
3717bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) {
3718 auto *StoreNode = cast<StoreSDNode>(Val: Node);
3719 SDValue StoredVal = StoreNode->getOperand(Num: 1);
3720 unsigned Opc = StoredVal->getOpcode();
3721
3722 // Before we try to select anything, make sure this is memory operand size
3723 // and opcode we can handle. Note that this must match the code below that
3724 // actually lowers the opcodes.
3725 EVT MemVT = StoreNode->getMemoryVT();
3726 if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 &&
3727 MemVT != MVT::i8)
3728 return false;
3729
3730 bool IsCommutable = false;
3731 bool IsNegate = false;
3732 switch (Opc) {
3733 default:
3734 return false;
3735 case X86ISD::SUB:
3736 IsNegate = isNullConstant(V: StoredVal.getOperand(i: 0));
3737 break;
3738 case X86ISD::SBB:
3739 break;
3740 case X86ISD::ADD:
3741 case X86ISD::ADC:
3742 case X86ISD::AND:
3743 case X86ISD::OR:
3744 case X86ISD::XOR:
3745 IsCommutable = true;
3746 break;
3747 }
3748
3749 unsigned LoadOpNo = IsNegate ? 1 : 0;
3750 LoadSDNode *LoadNode = nullptr;
3751 SDValue InputChain;
3752 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadOpNo,
3753 LoadNode, InputChain)) {
3754 if (!IsCommutable)
3755 return false;
3756
3757 // This operation is commutable, try the other operand.
3758 LoadOpNo = 1;
3759 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadOpNo,
3760 LoadNode, InputChain))
3761 return false;
3762 }
3763
3764 SDValue Base, Scale, Index, Disp, Segment;
3765 if (!selectAddr(Parent: LoadNode, N: LoadNode->getBasePtr(), Base, Scale, Index, Disp,
3766 Segment))
3767 return false;
3768
3769 auto SelectOpcode = [&](unsigned Opc64, unsigned Opc32, unsigned Opc16,
3770 unsigned Opc8) {
3771 switch (MemVT.getSimpleVT().SimpleTy) {
3772 case MVT::i64:
3773 return Opc64;
3774 case MVT::i32:
3775 return Opc32;
3776 case MVT::i16:
3777 return Opc16;
3778 case MVT::i8:
3779 return Opc8;
3780 default:
3781 llvm_unreachable("Invalid size!");
3782 }
3783 };
3784
3785 MachineSDNode *Result;
3786 switch (Opc) {
3787 case X86ISD::SUB:
3788 // Handle negate.
3789 if (IsNegate) {
3790 unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m,
3791 X86::NEG8m);
3792 const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain};
3793 Result = CurDAG->getMachineNode(Opcode: NewOpc, dl: SDLoc(Node), VT1: MVT::i32,
3794 VT2: MVT::Other, Ops);
3795 break;
3796 }
3797 [[fallthrough]];
3798 case X86ISD::ADD:
3799 // Try to match inc/dec.
3800 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) {
3801 bool IsOne = isOneConstant(V: StoredVal.getOperand(i: 1));
3802 bool IsNegOne = isAllOnesConstant(V: StoredVal.getOperand(i: 1));
3803 // ADD/SUB with 1/-1 and carry flag isn't used can use inc/dec.
3804 if ((IsOne || IsNegOne) && hasNoCarryFlagUses(Flags: StoredVal.getValue(R: 1))) {
3805 unsigned NewOpc =
3806 ((Opc == X86ISD::ADD) == IsOne)
3807 ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
3808 : SelectOpcode(X86::DEC64m, X86::DEC32m, X86::DEC16m, X86::DEC8m);
3809 const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain};
3810 Result = CurDAG->getMachineNode(Opcode: NewOpc, dl: SDLoc(Node), VT1: MVT::i32,
3811 VT2: MVT::Other, Ops);
3812 break;
3813 }
3814 }
3815 [[fallthrough]];
3816 case X86ISD::ADC:
3817 case X86ISD::SBB:
3818 case X86ISD::AND:
3819 case X86ISD::OR:
3820 case X86ISD::XOR: {
3821 auto SelectRegOpcode = [SelectOpcode](unsigned Opc) {
3822 switch (Opc) {
3823 case X86ISD::ADD:
3824 return SelectOpcode(X86::ADD64mr, X86::ADD32mr, X86::ADD16mr,
3825 X86::ADD8mr);
3826 case X86ISD::ADC:
3827 return SelectOpcode(X86::ADC64mr, X86::ADC32mr, X86::ADC16mr,
3828 X86::ADC8mr);
3829 case X86ISD::SUB:
3830 return SelectOpcode(X86::SUB64mr, X86::SUB32mr, X86::SUB16mr,
3831 X86::SUB8mr);
3832 case X86ISD::SBB:
3833 return SelectOpcode(X86::SBB64mr, X86::SBB32mr, X86::SBB16mr,
3834 X86::SBB8mr);
3835 case X86ISD::AND:
3836 return SelectOpcode(X86::AND64mr, X86::AND32mr, X86::AND16mr,
3837 X86::AND8mr);
3838 case X86ISD::OR:
3839 return SelectOpcode(X86::OR64mr, X86::OR32mr, X86::OR16mr, X86::OR8mr);
3840 case X86ISD::XOR:
3841 return SelectOpcode(X86::XOR64mr, X86::XOR32mr, X86::XOR16mr,
3842 X86::XOR8mr);
3843 default:
3844 llvm_unreachable("Invalid opcode!");
3845 }
3846 };
3847 auto SelectImmOpcode = [SelectOpcode](unsigned Opc) {
3848 switch (Opc) {
3849 case X86ISD::ADD:
3850 return SelectOpcode(X86::ADD64mi32, X86::ADD32mi, X86::ADD16mi,
3851 X86::ADD8mi);
3852 case X86ISD::ADC:
3853 return SelectOpcode(X86::ADC64mi32, X86::ADC32mi, X86::ADC16mi,
3854 X86::ADC8mi);
3855 case X86ISD::SUB:
3856 return SelectOpcode(X86::SUB64mi32, X86::SUB32mi, X86::SUB16mi,
3857 X86::SUB8mi);
3858 case X86ISD::SBB:
3859 return SelectOpcode(X86::SBB64mi32, X86::SBB32mi, X86::SBB16mi,
3860 X86::SBB8mi);
3861 case X86ISD::AND:
3862 return SelectOpcode(X86::AND64mi32, X86::AND32mi, X86::AND16mi,
3863 X86::AND8mi);
3864 case X86ISD::OR:
3865 return SelectOpcode(X86::OR64mi32, X86::OR32mi, X86::OR16mi,
3866 X86::OR8mi);
3867 case X86ISD::XOR:
3868 return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi,
3869 X86::XOR8mi);
3870 default:
3871 llvm_unreachable("Invalid opcode!");
3872 }
3873 };
3874
3875 unsigned NewOpc = SelectRegOpcode(Opc);
3876 SDValue Operand = StoredVal->getOperand(Num: 1-LoadOpNo);
3877
3878 // See if the operand is a constant that we can fold into an immediate
3879 // operand.
3880 if (auto *OperandC = dyn_cast<ConstantSDNode>(Val&: Operand)) {
3881 int64_t OperandV = OperandC->getSExtValue();
3882
3883 // Check if we can shrink the operand enough to fit in an immediate (or
3884 // fit into a smaller immediate) by negating it and switching the
3885 // operation.
3886 if ((Opc == X86ISD::ADD || Opc == X86ISD::SUB) &&
3887 ((MemVT != MVT::i8 && !isInt<8>(x: OperandV) && isInt<8>(x: -OperandV)) ||
3888 (MemVT == MVT::i64 && !isInt<32>(x: OperandV) &&
3889 isInt<32>(x: -OperandV))) &&
3890 hasNoCarryFlagUses(Flags: StoredVal.getValue(R: 1))) {
3891 OperandV = -OperandV;
3892 Opc = Opc == X86ISD::ADD ? X86ISD::SUB : X86ISD::ADD;
3893 }
3894
3895 if (MemVT != MVT::i64 || isInt<32>(x: OperandV)) {
3896 Operand = CurDAG->getSignedTargetConstant(Val: OperandV, DL: SDLoc(Node), VT: MemVT);
3897 NewOpc = SelectImmOpcode(Opc);
3898 }
3899 }
3900
3901 if (Opc == X86ISD::ADC || Opc == X86ISD::SBB) {
3902 SDValue CopyTo =
3903 CurDAG->getCopyToReg(Chain: InputChain, dl: SDLoc(Node), Reg: X86::EFLAGS,
3904 N: StoredVal.getOperand(i: 2), Glue: SDValue());
3905
3906 const SDValue Ops[] = {Base, Scale, Index, Disp,
3907 Segment, Operand, CopyTo, CopyTo.getValue(R: 1)};
3908 Result = CurDAG->getMachineNode(Opcode: NewOpc, dl: SDLoc(Node), VT1: MVT::i32, VT2: MVT::Other,
3909 Ops);
3910 } else {
3911 const SDValue Ops[] = {Base, Scale, Index, Disp,
3912 Segment, Operand, InputChain};
3913 Result = CurDAG->getMachineNode(Opcode: NewOpc, dl: SDLoc(Node), VT1: MVT::i32, VT2: MVT::Other,
3914 Ops);
3915 }
3916 break;
3917 }
3918 default:
3919 llvm_unreachable("Invalid opcode!");
3920 }
3921
3922 MachineMemOperand *MemOps[] = {StoreNode->getMemOperand(),
3923 LoadNode->getMemOperand()};
3924 CurDAG->setNodeMemRefs(N: Result, NewMemRefs: MemOps);
3925
3926 // Update Load Chain uses as well.
3927 ReplaceUses(F: SDValue(LoadNode, 1), T: SDValue(Result, 1));
3928 ReplaceUses(F: SDValue(StoreNode, 0), T: SDValue(Result, 1));
3929 ReplaceUses(F: SDValue(StoredVal.getNode(), 1), T: SDValue(Result, 0));
3930 CurDAG->RemoveDeadNode(N: Node);
3931 return true;
3932}
3933
3934// See if this is an X & Mask that we can match to BEXTR/BZHI.
3935// Where Mask is one of the following patterns:
3936// a) x & (1 << nbits) - 1
3937// b) x & ~(-1 << nbits)
3938// c) x & (-1 >> (32 - y))
3939// d) x << (32 - y) >> (32 - y)
3940// e) (1 << nbits) - 1
3941bool X86DAGToDAGISel::matchBitExtract(SDNode *Node) {
3942 assert(
3943 (Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::AND ||
3944 Node->getOpcode() == ISD::SRL) &&
3945 "Should be either an and-mask, or right-shift after clearing high bits.");
3946
3947 // BEXTR is BMI instruction, BZHI is BMI2 instruction. We need at least one.
3948 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2())
3949 return false;
3950
3951 MVT NVT = Node->getSimpleValueType(ResNo: 0);
3952
3953 // Only supported for 32 and 64 bits.
3954 if (NVT != MVT::i32 && NVT != MVT::i64)
3955 return false;
3956
3957 SDValue NBits;
3958 bool NegateNBits;
3959
3960 // If we have BMI2's BZHI, we are ok with muti-use patterns.
3961 // Else, if we only have BMI1's BEXTR, we require one-use.
3962 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2();
3963 auto checkUses = [AllowExtraUsesByDefault](
3964 SDValue Op, unsigned NUses,
3965 std::optional<bool> AllowExtraUses) {
3966 return AllowExtraUses.value_or(u: AllowExtraUsesByDefault) ||
3967 Op.getNode()->hasNUsesOfValue(NUses, Value: Op.getResNo());
3968 };
3969 auto checkOneUse = [checkUses](SDValue Op,
3970 std::optional<bool> AllowExtraUses =
3971 std::nullopt) {
3972 return checkUses(Op, 1, AllowExtraUses);
3973 };
3974 auto checkTwoUse = [checkUses](SDValue Op,
3975 std::optional<bool> AllowExtraUses =
3976 std::nullopt) {
3977 return checkUses(Op, 2, AllowExtraUses);
3978 };
3979
3980 auto peekThroughOneUseTruncation = [checkOneUse](SDValue V) {
3981 if (V->getOpcode() == ISD::TRUNCATE && checkOneUse(V)) {
3982 assert(V.getSimpleValueType() == MVT::i32 &&
3983 V.getOperand(0).getSimpleValueType() == MVT::i64 &&
3984 "Expected i64 -> i32 truncation");
3985 V = V.getOperand(i: 0);
3986 }
3987 return V;
3988 };
3989
3990 // a) x & ((1 << nbits) + (-1))
3991 auto matchPatternA = [checkOneUse, peekThroughOneUseTruncation, &NBits,
3992 &NegateNBits](SDValue Mask) -> bool {
3993 // Match `add`. Must only have one use!
3994 if (Mask->getOpcode() != ISD::ADD || !checkOneUse(Mask))
3995 return false;
3996 // We should be adding all-ones constant (i.e. subtracting one.)
3997 if (!isAllOnesConstant(V: Mask->getOperand(Num: 1)))
3998 return false;
3999 // Match `1 << nbits`. Might be truncated. Must only have one use!
4000 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(Num: 0));
4001 if (M0->getOpcode() != ISD::SHL || !checkOneUse(M0))
4002 return false;
4003 if (!isOneConstant(V: M0->getOperand(Num: 0)))
4004 return false;
4005 NBits = M0->getOperand(Num: 1);
4006 NegateNBits = false;
4007 return true;
4008 };
4009
4010 auto isAllOnes = [this, peekThroughOneUseTruncation, NVT](SDValue V) {
4011 V = peekThroughOneUseTruncation(V);
4012 return CurDAG->MaskedValueIsAllOnes(
4013 Op: V, Mask: APInt::getLowBitsSet(numBits: V.getSimpleValueType().getSizeInBits(),
4014 loBitsSet: NVT.getSizeInBits()));
4015 };
4016
4017 // b) x & ~(-1 << nbits)
4018 auto matchPatternB = [checkOneUse, isAllOnes, peekThroughOneUseTruncation,
4019 &NBits, &NegateNBits](SDValue Mask) -> bool {
4020 // Match `~()`. Must only have one use!
4021 if (Mask.getOpcode() != ISD::XOR || !checkOneUse(Mask))
4022 return false;
4023 // The -1 only has to be all-ones for the final Node's NVT.
4024 if (!isAllOnes(Mask->getOperand(Num: 1)))
4025 return false;
4026 // Match `-1 << nbits`. Might be truncated. Must only have one use!
4027 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(Num: 0));
4028 if (M0->getOpcode() != ISD::SHL || !checkOneUse(M0))
4029 return false;
4030 // The -1 only has to be all-ones for the final Node's NVT.
4031 if (!isAllOnes(M0->getOperand(Num: 0)))
4032 return false;
4033 NBits = M0->getOperand(Num: 1);
4034 NegateNBits = false;
4035 return true;
4036 };
4037
4038 // Try to match potentially-truncated shift amount as `(bitwidth - y)`,
4039 // or leave the shift amount as-is, but then we'll have to negate it.
4040 auto canonicalizeShiftAmt = [&NBits, &NegateNBits](SDValue ShiftAmt,
4041 unsigned Bitwidth) {
4042 NBits = ShiftAmt;
4043 NegateNBits = true;
4044 // Skip over a truncate of the shift amount, if any.
4045 if (NBits.getOpcode() == ISD::TRUNCATE)
4046 NBits = NBits.getOperand(i: 0);
4047 // Try to match the shift amount as (bitwidth - y). It should go away, too.
4048 // If it doesn't match, that's fine, we'll just negate it ourselves.
4049 if (NBits.getOpcode() != ISD::SUB)
4050 return;
4051 auto *V0 = dyn_cast<ConstantSDNode>(Val: NBits.getOperand(i: 0));
4052 if (!V0 || V0->getZExtValue() != Bitwidth)
4053 return;
4054 NBits = NBits.getOperand(i: 1);
4055 NegateNBits = false;
4056 };
4057
4058 // c) x & (-1 >> z) but then we'll have to subtract z from bitwidth
4059 // or
4060 // c) x & (-1 >> (32 - y))
4061 auto matchPatternC = [checkOneUse, peekThroughOneUseTruncation, &NegateNBits,
4062 canonicalizeShiftAmt](SDValue Mask) -> bool {
4063 // The mask itself may be truncated.
4064 Mask = peekThroughOneUseTruncation(Mask);
4065 unsigned Bitwidth = Mask.getSimpleValueType().getSizeInBits();
4066 // Match `l>>`. Must only have one use!
4067 if (Mask.getOpcode() != ISD::SRL || !checkOneUse(Mask))
4068 return false;
4069 // We should be shifting truly all-ones constant.
4070 if (!isAllOnesConstant(V: Mask.getOperand(i: 0)))
4071 return false;
4072 SDValue M1 = Mask.getOperand(i: 1);
4073 // The shift amount should not be used externally.
4074 if (!checkOneUse(M1))
4075 return false;
4076 canonicalizeShiftAmt(M1, Bitwidth);
4077 // Pattern c. is non-canonical, and is expanded into pattern d. iff there
4078 // is no extra use of the mask. Clearly, there was one since we are here.
4079 // But at the same time, if we need to negate the shift amount,
4080 // then we don't want the mask to stick around, else it's unprofitable.
4081 return !NegateNBits;
4082 };
4083
4084 SDValue X;
4085
4086 // d) x << z >> z but then we'll have to subtract z from bitwidth
4087 // or
4088 // d) x << (32 - y) >> (32 - y)
4089 auto matchPatternD = [checkOneUse, checkTwoUse, canonicalizeShiftAmt,
4090 AllowExtraUsesByDefault, &NegateNBits,
4091 &X](SDNode *Node) -> bool {
4092 if (Node->getOpcode() != ISD::SRL)
4093 return false;
4094 SDValue N0 = Node->getOperand(Num: 0);
4095 if (N0->getOpcode() != ISD::SHL)
4096 return false;
4097 unsigned Bitwidth = N0.getSimpleValueType().getSizeInBits();
4098 SDValue N1 = Node->getOperand(Num: 1);
4099 SDValue N01 = N0->getOperand(Num: 1);
4100 // Both of the shifts must be by the exact same value.
4101 if (N1 != N01)
4102 return false;
4103 canonicalizeShiftAmt(N1, Bitwidth);
4104 // There should not be any external uses of the inner shift / shift amount.
4105 // Note that while we are generally okay with external uses given BMI2,
4106 // iff we need to negate the shift amount, we are not okay with extra uses.
4107 const bool AllowExtraUses = AllowExtraUsesByDefault && !NegateNBits;
4108 if (!checkOneUse(N0, AllowExtraUses) || !checkTwoUse(N1, AllowExtraUses))
4109 return false;
4110 X = N0->getOperand(Num: 0);
4111 return true;
4112 };
4113
4114 auto matchLowBitMask = [matchPatternA, matchPatternB,
4115 matchPatternC](SDValue Mask) -> bool {
4116 return matchPatternA(Mask) || matchPatternB(Mask) || matchPatternC(Mask);
4117 };
4118
4119 if (Node->getOpcode() == ISD::AND) {
4120 X = Node->getOperand(Num: 0);
4121 SDValue Mask = Node->getOperand(Num: 1);
4122
4123 if (matchLowBitMask(Mask)) {
4124 // Great.
4125 } else {
4126 std::swap(a&: X, b&: Mask);
4127 if (!matchLowBitMask(Mask))
4128 return false;
4129 }
4130 } else if (matchLowBitMask(SDValue(Node, 0))) {
4131 X = CurDAG->getAllOnesConstant(DL: SDLoc(Node), VT: NVT);
4132 } else if (!matchPatternD(Node))
4133 return false;
4134
4135 // If we need to negate the shift amount, require BMI2 BZHI support.
4136 // It's just too unprofitable for BMI1 BEXTR.
4137 if (NegateNBits && !Subtarget->hasBMI2())
4138 return false;
4139
4140 SDLoc DL(Node);
4141
4142 if (NBits.getSimpleValueType() != MVT::i8) {
4143 // Truncate the shift amount.
4144 NBits = CurDAG->getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8, Operand: NBits);
4145 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: NBits);
4146 }
4147
4148 // Turn (i32)(x & imm8) into (i32)x & imm32.
4149 ConstantSDNode *Imm = nullptr;
4150 if (NBits->getOpcode() == ISD::AND)
4151 if ((Imm = dyn_cast<ConstantSDNode>(Val: NBits->getOperand(Num: 1))))
4152 NBits = NBits->getOperand(Num: 0);
4153
4154 // Insert 8-bit NBits into lowest 8 bits of 32-bit register.
4155 // All the other bits are undefined, we do not care about them.
4156 SDValue ImplDef = SDValue(
4157 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::i32), 0);
4158 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: ImplDef);
4159
4160 SDValue SRIdxVal = CurDAG->getTargetConstant(Val: X86::sub_8bit, DL, VT: MVT::i32);
4161 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: SRIdxVal);
4162 NBits = SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
4163 VT: MVT::i32, Op1: ImplDef, Op2: NBits, Op3: SRIdxVal),
4164 0);
4165 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: NBits);
4166
4167 if (Imm) {
4168 NBits =
4169 CurDAG->getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: NBits,
4170 N2: CurDAG->getConstant(Val: Imm->getZExtValue(), DL, VT: MVT::i32));
4171 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: NBits);
4172 }
4173
4174 // We might have matched the amount of high bits to be cleared,
4175 // but we want the amount of low bits to be kept, so negate it then.
4176 if (NegateNBits) {
4177 SDValue BitWidthC = CurDAG->getConstant(Val: NVT.getSizeInBits(), DL, VT: MVT::i32);
4178 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: BitWidthC);
4179
4180 NBits = CurDAG->getNode(Opcode: ISD::SUB, DL, VT: MVT::i32, N1: BitWidthC, N2: NBits);
4181 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: NBits);
4182 }
4183
4184 if (Subtarget->hasBMI2()) {
4185 // Great, just emit the BZHI..
4186 if (NVT != MVT::i32) {
4187 // But have to place the bit count into the wide-enough register first.
4188 NBits = CurDAG->getNode(Opcode: ISD::ANY_EXTEND, DL, VT: NVT, Operand: NBits);
4189 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: NBits);
4190 }
4191
4192 SDValue Extract = CurDAG->getNode(Opcode: X86ISD::BZHI, DL, VT: NVT, N1: X, N2: NBits);
4193 ReplaceNode(F: Node, T: Extract.getNode());
4194 SelectCode(N: Extract.getNode());
4195 return true;
4196 }
4197
4198 // Else, if we do *NOT* have BMI2, let's find out if the if the 'X' is
4199 // *logically* shifted (potentially with one-use trunc inbetween),
4200 // and the truncation was the only use of the shift,
4201 // and if so look past one-use truncation.
4202 {
4203 SDValue RealX = peekThroughOneUseTruncation(X);
4204 // FIXME: only if the shift is one-use?
4205 if (RealX != X && RealX.getOpcode() == ISD::SRL)
4206 X = RealX;
4207 }
4208
4209 MVT XVT = X.getSimpleValueType();
4210
4211 // Else, emitting BEXTR requires one more step.
4212 // The 'control' of BEXTR has the pattern of:
4213 // [15...8 bit][ 7...0 bit] location
4214 // [ bit count][ shift] name
4215 // I.e. 0b000000011'00000001 means (x >> 0b1) & 0b11
4216
4217 // Shift NBits left by 8 bits, thus producing 'control'.
4218 // This makes the low 8 bits to be zero.
4219 SDValue C8 = CurDAG->getConstant(Val: 8, DL, VT: MVT::i8);
4220 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: C8);
4221 SDValue Control = CurDAG->getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: NBits, N2: C8);
4222 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: Control);
4223
4224 // If the 'X' is *logically* shifted, we can fold that shift into 'control'.
4225 // FIXME: only if the shift is one-use?
4226 if (X.getOpcode() == ISD::SRL) {
4227 SDValue ShiftAmt = X.getOperand(i: 1);
4228 X = X.getOperand(i: 0);
4229
4230 assert(ShiftAmt.getValueType() == MVT::i8 &&
4231 "Expected shift amount to be i8");
4232
4233 // Now, *zero*-extend the shift amount. The bits 8...15 *must* be zero!
4234 // We could zext to i16 in some form, but we intentionally don't do that.
4235 SDValue OrigShiftAmt = ShiftAmt;
4236 ShiftAmt = CurDAG->getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: ShiftAmt);
4237 insertDAGNode(DAG&: *CurDAG, Pos: OrigShiftAmt, N: ShiftAmt);
4238
4239 // And now 'or' these low 8 bits of shift amount into the 'control'.
4240 Control = CurDAG->getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Control, N2: ShiftAmt);
4241 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: Control);
4242 }
4243
4244 // But have to place the 'control' into the wide-enough register first.
4245 if (XVT != MVT::i32) {
4246 Control = CurDAG->getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XVT, Operand: Control);
4247 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: Control);
4248 }
4249
4250 // And finally, form the BEXTR itself.
4251 SDValue Extract = CurDAG->getNode(Opcode: X86ISD::BEXTR, DL, VT: XVT, N1: X, N2: Control);
4252
4253 // The 'X' was originally truncated. Do that now.
4254 if (XVT != NVT) {
4255 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(Node, 0), N: Extract);
4256 Extract = CurDAG->getNode(Opcode: ISD::TRUNCATE, DL, VT: NVT, Operand: Extract);
4257 }
4258
4259 ReplaceNode(F: Node, T: Extract.getNode());
4260 SelectCode(N: Extract.getNode());
4261
4262 return true;
4263}
4264
4265// See if this is an (X >> C1) & C2 that we can match to BEXTR/BEXTRI.
4266MachineSDNode *X86DAGToDAGISel::matchBEXTRFromAndImm(SDNode *Node) {
4267 MVT NVT = Node->getSimpleValueType(ResNo: 0);
4268 SDLoc dl(Node);
4269
4270 SDValue N0 = Node->getOperand(Num: 0);
4271 SDValue N1 = Node->getOperand(Num: 1);
4272
4273 // If we have TBM we can use an immediate for the control. If we have BMI
4274 // we should only do this if the BEXTR instruction is implemented well.
4275 // Otherwise moving the control into a register makes this more costly.
4276 // TODO: Maybe load folding, greater than 32-bit masks, or a guarantee of LICM
4277 // hoisting the move immediate would make it worthwhile with a less optimal
4278 // BEXTR?
4279 bool PreferBEXTR =
4280 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR());
4281 if (!PreferBEXTR && !Subtarget->hasBMI2())
4282 return nullptr;
4283
4284 // Must have a shift right.
4285 if (N0->getOpcode() != ISD::SRL && N0->getOpcode() != ISD::SRA)
4286 return nullptr;
4287
4288 // Shift can't have additional users.
4289 if (!N0->hasOneUse())
4290 return nullptr;
4291
4292 // Only supported for 32 and 64 bits.
4293 if (NVT != MVT::i32 && NVT != MVT::i64)
4294 return nullptr;
4295
4296 // Shift amount and RHS of and must be constant.
4297 auto *MaskCst = dyn_cast<ConstantSDNode>(Val&: N1);
4298 auto *ShiftCst = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
4299 if (!MaskCst || !ShiftCst)
4300 return nullptr;
4301
4302 // And RHS must be a mask.
4303 uint64_t Mask = MaskCst->getZExtValue();
4304 if (!isMask_64(Value: Mask))
4305 return nullptr;
4306
4307 uint64_t Shift = ShiftCst->getZExtValue();
4308 uint64_t MaskSize = llvm::popcount(Value: Mask);
4309
4310 // Don't interfere with something that can be handled by extracting AH.
4311 // TODO: If we are able to fold a load, BEXTR might still be better than AH.
4312 if (Shift == 8 && MaskSize == 8)
4313 return nullptr;
4314
4315 // Make sure we are only using bits that were in the original value, not
4316 // shifted in.
4317 if (Shift + MaskSize > NVT.getSizeInBits())
4318 return nullptr;
4319
4320 // BZHI, if available, is always fast, unlike BEXTR. But even if we decide
4321 // that we can't use BEXTR, it is only worthwhile using BZHI if the mask
4322 // does not fit into 32 bits. Load folding is not a sufficient reason.
4323 if (!PreferBEXTR && MaskSize <= 32)
4324 return nullptr;
4325
4326 SDValue Control;
4327 unsigned ROpc, MOpc;
4328
4329#define GET_EGPR_IF_ENABLED(OPC) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC)
4330 if (!PreferBEXTR) {
4331 assert(Subtarget->hasBMI2() && "We must have BMI2's BZHI then.");
4332 // If we can't make use of BEXTR then we can't fuse shift+mask stages.
4333 // Let's perform the mask first, and apply shift later. Note that we need to
4334 // widen the mask to account for the fact that we'll apply shift afterwards!
4335 Control = CurDAG->getTargetConstant(Val: Shift + MaskSize, DL: dl, VT: NVT);
4336 ROpc = NVT == MVT::i64 ? GET_EGPR_IF_ENABLED(X86::BZHI64rr)
4337 : GET_EGPR_IF_ENABLED(X86::BZHI32rr);
4338 MOpc = NVT == MVT::i64 ? GET_EGPR_IF_ENABLED(X86::BZHI64rm)
4339 : GET_EGPR_IF_ENABLED(X86::BZHI32rm);
4340 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4341 Control = SDValue(CurDAG->getMachineNode(Opcode: NewOpc, dl, VT: NVT, Op1: Control), 0);
4342 } else {
4343 // The 'control' of BEXTR has the pattern of:
4344 // [15...8 bit][ 7...0 bit] location
4345 // [ bit count][ shift] name
4346 // I.e. 0b000000011'00000001 means (x >> 0b1) & 0b11
4347 Control = CurDAG->getTargetConstant(Val: Shift | (MaskSize << 8), DL: dl, VT: NVT);
4348 if (Subtarget->hasTBM()) {
4349 ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
4350 MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
4351 } else {
4352 assert(Subtarget->hasBMI() && "We must have BMI1's BEXTR then.");
4353 // BMI requires the immediate to placed in a register.
4354 ROpc = NVT == MVT::i64 ? GET_EGPR_IF_ENABLED(X86::BEXTR64rr)
4355 : GET_EGPR_IF_ENABLED(X86::BEXTR32rr);
4356 MOpc = NVT == MVT::i64 ? GET_EGPR_IF_ENABLED(X86::BEXTR64rm)
4357 : GET_EGPR_IF_ENABLED(X86::BEXTR32rm);
4358 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4359 Control = SDValue(CurDAG->getMachineNode(Opcode: NewOpc, dl, VT: NVT, Op1: Control), 0);
4360 }
4361 }
4362
4363 MachineSDNode *NewNode;
4364 SDValue Input = N0->getOperand(Num: 0);
4365 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4366 if (tryFoldLoad(Root: Node, P: N0.getNode(), N: Input, Base&: Tmp0, Scale&: Tmp1, Index&: Tmp2, Disp&: Tmp3, Segment&: Tmp4)) {
4367 SDValue Ops[] = {
4368 Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.getOperand(i: 0)};
4369 SDVTList VTs = CurDAG->getVTList(VT1: NVT, VT2: MVT::i32, VT3: MVT::Other);
4370 NewNode = CurDAG->getMachineNode(Opcode: MOpc, dl, VTs, Ops);
4371 // Update the chain.
4372 ReplaceUses(F: Input.getValue(R: 1), T: SDValue(NewNode, 2));
4373 // Record the mem-refs
4374 CurDAG->setNodeMemRefs(N: NewNode, NewMemRefs: {cast<LoadSDNode>(Val&: Input)->getMemOperand()});
4375 } else {
4376 NewNode = CurDAG->getMachineNode(Opcode: ROpc, dl, VT1: NVT, VT2: MVT::i32, Op1: Input, Op2: Control);
4377 }
4378
4379 if (!PreferBEXTR) {
4380 // We still need to apply the shift.
4381 SDValue ShAmt = CurDAG->getTargetConstant(Val: Shift, DL: dl, VT: NVT);
4382 unsigned NewOpc = NVT == MVT::i64 ? GET_ND_IF_ENABLED(X86::SHR64ri)
4383 : GET_ND_IF_ENABLED(X86::SHR32ri);
4384 NewNode =
4385 CurDAG->getMachineNode(Opcode: NewOpc, dl, VT: NVT, Op1: SDValue(NewNode, 0), Op2: ShAmt);
4386 }
4387
4388 return NewNode;
4389}
4390
4391// Emit a PCMISTR(I/M) instruction.
4392MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(unsigned ROpc, unsigned MOpc,
4393 bool MayFoldLoad, const SDLoc &dl,
4394 MVT VT, SDNode *Node) {
4395 SDValue N0 = Node->getOperand(Num: 0);
4396 SDValue N1 = Node->getOperand(Num: 1);
4397 SDValue Imm = Node->getOperand(Num: 2);
4398 auto *Val = cast<ConstantSDNode>(Val&: Imm)->getConstantIntValue();
4399 Imm = CurDAG->getTargetConstant(Val: *Val, DL: SDLoc(Node), VT: Imm.getValueType());
4400
4401 // Try to fold a load. No need to check alignment.
4402 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4403 if (MayFoldLoad && tryFoldLoad(P: Node, N: N1, Base&: Tmp0, Scale&: Tmp1, Index&: Tmp2, Disp&: Tmp3, Segment&: Tmp4)) {
4404 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
4405 N1.getOperand(i: 0) };
4406 SDVTList VTs = CurDAG->getVTList(VT1: VT, VT2: MVT::i32, VT3: MVT::Other);
4407 MachineSDNode *CNode = CurDAG->getMachineNode(Opcode: MOpc, dl, VTs, Ops);
4408 // Update the chain.
4409 ReplaceUses(F: N1.getValue(R: 1), T: SDValue(CNode, 2));
4410 // Record the mem-refs
4411 CurDAG->setNodeMemRefs(N: CNode, NewMemRefs: {cast<LoadSDNode>(Val&: N1)->getMemOperand()});
4412 return CNode;
4413 }
4414
4415 SDValue Ops[] = { N0, N1, Imm };
4416 SDVTList VTs = CurDAG->getVTList(VT1: VT, VT2: MVT::i32);
4417 MachineSDNode *CNode = CurDAG->getMachineNode(Opcode: ROpc, dl, VTs, Ops);
4418 return CNode;
4419}
4420
4421// Emit a PCMESTR(I/M) instruction. Also return the Glue result in case we need
4422// to emit a second instruction after this one. This is needed since we have two
4423// copyToReg nodes glued before this and we need to continue that glue through.
4424MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(unsigned ROpc, unsigned MOpc,
4425 bool MayFoldLoad, const SDLoc &dl,
4426 MVT VT, SDNode *Node,
4427 SDValue &InGlue) {
4428 SDValue N0 = Node->getOperand(Num: 0);
4429 SDValue N2 = Node->getOperand(Num: 2);
4430 SDValue Imm = Node->getOperand(Num: 4);
4431 auto *Val = cast<ConstantSDNode>(Val&: Imm)->getConstantIntValue();
4432 Imm = CurDAG->getTargetConstant(Val: *Val, DL: SDLoc(Node), VT: Imm.getValueType());
4433
4434 // Try to fold a load. No need to check alignment.
4435 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4436 if (MayFoldLoad && tryFoldLoad(P: Node, N: N2, Base&: Tmp0, Scale&: Tmp1, Index&: Tmp2, Disp&: Tmp3, Segment&: Tmp4)) {
4437 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
4438 N2.getOperand(i: 0), InGlue };
4439 SDVTList VTs = CurDAG->getVTList(VT1: VT, VT2: MVT::i32, VT3: MVT::Other, VT4: MVT::Glue);
4440 MachineSDNode *CNode = CurDAG->getMachineNode(Opcode: MOpc, dl, VTs, Ops);
4441 InGlue = SDValue(CNode, 3);
4442 // Update the chain.
4443 ReplaceUses(F: N2.getValue(R: 1), T: SDValue(CNode, 2));
4444 // Record the mem-refs
4445 CurDAG->setNodeMemRefs(N: CNode, NewMemRefs: {cast<LoadSDNode>(Val&: N2)->getMemOperand()});
4446 return CNode;
4447 }
4448
4449 SDValue Ops[] = { N0, N2, Imm, InGlue };
4450 SDVTList VTs = CurDAG->getVTList(VT1: VT, VT2: MVT::i32, VT3: MVT::Glue);
4451 MachineSDNode *CNode = CurDAG->getMachineNode(Opcode: ROpc, dl, VTs, Ops);
4452 InGlue = SDValue(CNode, 2);
4453 return CNode;
4454}
4455
4456bool X86DAGToDAGISel::tryShiftAmountMod(SDNode *N) {
4457 EVT VT = N->getValueType(ResNo: 0);
4458
4459 // Only handle scalar shifts.
4460 if (VT.isVector())
4461 return false;
4462
4463 // Narrower shifts only mask to 5 bits in hardware.
4464 unsigned Size = VT == MVT::i64 ? 64 : 32;
4465
4466 SDValue OrigShiftAmt = N->getOperand(Num: 1);
4467 SDValue ShiftAmt = OrigShiftAmt;
4468 SDLoc DL(N);
4469
4470 // Skip over a truncate of the shift amount.
4471 if (ShiftAmt->getOpcode() == ISD::TRUNCATE)
4472 ShiftAmt = ShiftAmt->getOperand(Num: 0);
4473
4474 // This function is called after X86DAGToDAGISel::matchBitExtract(),
4475 // so we are not afraid that we might mess up BZHI/BEXTR pattern.
4476
4477 SDValue NewShiftAmt;
4478 if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB ||
4479 ShiftAmt->getOpcode() == ISD::XOR) {
4480 SDValue Add0 = ShiftAmt->getOperand(Num: 0);
4481 SDValue Add1 = ShiftAmt->getOperand(Num: 1);
4482 auto *Add0C = dyn_cast<ConstantSDNode>(Val&: Add0);
4483 auto *Add1C = dyn_cast<ConstantSDNode>(Val&: Add1);
4484 // If we are shifting by X+/-/^N where N == 0 mod Size, then just shift by X
4485 // to avoid the ADD/SUB/XOR.
4486 if (Add1C && Add1C->getAPIntValue().urem(RHS: Size) == 0) {
4487 NewShiftAmt = Add0;
4488
4489 } else if (ShiftAmt->getOpcode() != ISD::ADD && ShiftAmt.hasOneUse() &&
4490 ((Add0C && Add0C->getAPIntValue().urem(RHS: Size) == Size - 1) ||
4491 (Add1C && Add1C->getAPIntValue().urem(RHS: Size) == Size - 1))) {
4492 // If we are doing a NOT on just the lower bits with (Size*N-1) -/^ X
4493 // we can replace it with a NOT. In the XOR case it may save some code
4494 // size, in the SUB case it also may save a move.
4495 assert(Add0C == nullptr || Add1C == nullptr);
4496
4497 // We can only do N-X, not X-N
4498 if (ShiftAmt->getOpcode() == ISD::SUB && Add0C == nullptr)
4499 return false;
4500
4501 EVT OpVT = ShiftAmt.getValueType();
4502
4503 SDValue AllOnes = CurDAG->getAllOnesConstant(DL, VT: OpVT);
4504 NewShiftAmt = CurDAG->getNode(Opcode: ISD::XOR, DL, VT: OpVT,
4505 N1: Add0C == nullptr ? Add0 : Add1, N2: AllOnes);
4506 insertDAGNode(DAG&: *CurDAG, Pos: OrigShiftAmt, N: AllOnes);
4507 insertDAGNode(DAG&: *CurDAG, Pos: OrigShiftAmt, N: NewShiftAmt);
4508 // If we are shifting by N-X where N == 0 mod Size, then just shift by
4509 // -X to generate a NEG instead of a SUB of a constant.
4510 } else if (ShiftAmt->getOpcode() == ISD::SUB && Add0C &&
4511 Add0C->getZExtValue() != 0) {
4512 EVT SubVT = ShiftAmt.getValueType();
4513 SDValue X;
4514 if (Add0C->getZExtValue() % Size == 0)
4515 X = Add1;
4516 else if (ShiftAmt.hasOneUse() && Size == 64 &&
4517 Add0C->getZExtValue() % 32 == 0) {
4518 // We have a 64-bit shift by (n*32-x), turn it into -(x+n*32).
4519 // This is mainly beneficial if we already compute (x+n*32).
4520 if (Add1.getOpcode() == ISD::TRUNCATE) {
4521 Add1 = Add1.getOperand(i: 0);
4522 SubVT = Add1.getValueType();
4523 }
4524 if (Add0.getValueType() != SubVT) {
4525 Add0 = CurDAG->getZExtOrTrunc(Op: Add0, DL, VT: SubVT);
4526 insertDAGNode(DAG&: *CurDAG, Pos: OrigShiftAmt, N: Add0);
4527 }
4528
4529 X = CurDAG->getNode(Opcode: ISD::ADD, DL, VT: SubVT, N1: Add1, N2: Add0);
4530 insertDAGNode(DAG&: *CurDAG, Pos: OrigShiftAmt, N: X);
4531 } else
4532 return false;
4533 // Insert a negate op.
4534 // TODO: This isn't guaranteed to replace the sub if there is a logic cone
4535 // that uses it that's not a shift.
4536 SDValue Zero = CurDAG->getConstant(Val: 0, DL, VT: SubVT);
4537 SDValue Neg = CurDAG->getNode(Opcode: ISD::SUB, DL, VT: SubVT, N1: Zero, N2: X);
4538 NewShiftAmt = Neg;
4539
4540 // Insert these operands into a valid topological order so they can
4541 // get selected independently.
4542 insertDAGNode(DAG&: *CurDAG, Pos: OrigShiftAmt, N: Zero);
4543 insertDAGNode(DAG&: *CurDAG, Pos: OrigShiftAmt, N: Neg);
4544 } else
4545 return false;
4546 } else
4547 return false;
4548
4549 if (NewShiftAmt.getValueType() != MVT::i8) {
4550 // Need to truncate the shift amount.
4551 NewShiftAmt = CurDAG->getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8, Operand: NewShiftAmt);
4552 // Add to a correct topological ordering.
4553 insertDAGNode(DAG&: *CurDAG, Pos: OrigShiftAmt, N: NewShiftAmt);
4554 }
4555
4556 // Insert a new mask to keep the shift amount legal. This should be removed
4557 // by isel patterns.
4558 NewShiftAmt = CurDAG->getNode(Opcode: ISD::AND, DL, VT: MVT::i8, N1: NewShiftAmt,
4559 N2: CurDAG->getConstant(Val: Size - 1, DL, VT: MVT::i8));
4560 // Place in a correct topological ordering.
4561 insertDAGNode(DAG&: *CurDAG, Pos: OrigShiftAmt, N: NewShiftAmt);
4562
4563 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
4564 Op2: NewShiftAmt);
4565 if (UpdatedNode != N) {
4566 // If we found an existing node, we should replace ourselves with that node
4567 // and wait for it to be selected after its other users.
4568 ReplaceNode(F: N, T: UpdatedNode);
4569 return true;
4570 }
4571
4572 // If the original shift amount is now dead, delete it so that we don't run
4573 // it through isel.
4574 if (OrigShiftAmt.getNode()->use_empty())
4575 CurDAG->RemoveDeadNode(N: OrigShiftAmt.getNode());
4576
4577 // Now that we've optimized the shift amount, defer to normal isel to get
4578 // load folding and legacy vs BMI2 selection without repeating it here.
4579 SelectCode(N);
4580 return true;
4581}
4582
4583bool X86DAGToDAGISel::tryShrinkShlLogicImm(SDNode *N) {
4584 MVT NVT = N->getSimpleValueType(ResNo: 0);
4585 unsigned Opcode = N->getOpcode();
4586 SDLoc dl(N);
4587
4588 // For operations of the form (x << C1) op C2, check if we can use a smaller
4589 // encoding for C2 by transforming it into (x op (C2>>C1)) << C1.
4590 SDValue Shift = N->getOperand(Num: 0);
4591 SDValue N1 = N->getOperand(Num: 1);
4592
4593 auto *Cst = dyn_cast<ConstantSDNode>(Val&: N1);
4594 if (!Cst)
4595 return false;
4596
4597 int64_t Val = Cst->getSExtValue();
4598
4599 // If we have an any_extend feeding the AND, look through it to see if there
4600 // is a shift behind it. But only if the AND doesn't use the extended bits.
4601 // FIXME: Generalize this to other ANY_EXTEND than i32 to i64?
4602 bool FoundAnyExtend = false;
4603 if (Shift.getOpcode() == ISD::ANY_EXTEND && Shift.hasOneUse() &&
4604 Shift.getOperand(i: 0).getSimpleValueType() == MVT::i32 &&
4605 isUInt<32>(x: Val)) {
4606 FoundAnyExtend = true;
4607 Shift = Shift.getOperand(i: 0);
4608 }
4609
4610 if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse())
4611 return false;
4612
4613 // i8 is unshrinkable, i16 should be promoted to i32.
4614 if (NVT != MVT::i32 && NVT != MVT::i64)
4615 return false;
4616
4617 auto *ShlCst = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1));
4618 if (!ShlCst)
4619 return false;
4620
4621 uint64_t ShAmt = ShlCst->getZExtValue();
4622
4623 // Make sure that we don't change the operation by removing bits.
4624 // This only matters for OR and XOR, AND is unaffected.
4625 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1;
4626 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
4627 return false;
4628
4629 // Check the minimum bitwidth for the new constant.
4630 // TODO: Using 16 and 8 bit operations is also possible for or32 & xor32.
4631 auto CanShrinkImmediate = [&](int64_t &ShiftedVal) {
4632 if (Opcode == ISD::AND) {
4633 // AND32ri is the same as AND64ri32 with zext imm.
4634 // Try this before sign extended immediates below.
4635 ShiftedVal = (uint64_t)Val >> ShAmt;
4636 if (NVT == MVT::i64 && !isUInt<32>(x: Val) && isUInt<32>(x: ShiftedVal))
4637 return true;
4638 // Also swap order when the AND can become MOVZX.
4639 if (ShiftedVal == UINT8_MAX || ShiftedVal == UINT16_MAX)
4640 return true;
4641 }
4642 ShiftedVal = Val >> ShAmt;
4643 if ((!isInt<8>(x: Val) && isInt<8>(x: ShiftedVal)) ||
4644 (!isInt<32>(x: Val) && isInt<32>(x: ShiftedVal)))
4645 return true;
4646 if (Opcode != ISD::AND) {
4647 // MOV32ri+OR64r/XOR64r is cheaper than MOV64ri64+OR64rr/XOR64rr
4648 ShiftedVal = (uint64_t)Val >> ShAmt;
4649 if (NVT == MVT::i64 && !isUInt<32>(x: Val) && isUInt<32>(x: ShiftedVal))
4650 return true;
4651 }
4652 return false;
4653 };
4654
4655 int64_t ShiftedVal;
4656 if (!CanShrinkImmediate(ShiftedVal))
4657 return false;
4658
4659 // Ok, we can reorder to get a smaller immediate.
4660
4661 // But, its possible the original immediate allowed an AND to become MOVZX.
4662 // Doing this late due to avoid the MakedValueIsZero call as late as
4663 // possible.
4664 if (Opcode == ISD::AND) {
4665 // Find the smallest zext this could possibly be.
4666 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits();
4667 ZExtWidth = llvm::bit_ceil(Value: std::max(a: ZExtWidth, b: 8U));
4668
4669 // Figure out which bits need to be zero to achieve that mask.
4670 APInt NeededMask = APInt::getLowBitsSet(numBits: NVT.getSizeInBits(),
4671 loBitsSet: ZExtWidth);
4672 NeededMask &= ~Cst->getAPIntValue();
4673
4674 if (CurDAG->MaskedValueIsZero(Op: N->getOperand(Num: 0), Mask: NeededMask))
4675 return false;
4676 }
4677
4678 SDValue X = Shift.getOperand(i: 0);
4679 if (FoundAnyExtend) {
4680 SDValue NewX = CurDAG->getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: X);
4681 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(N, 0), N: NewX);
4682 X = NewX;
4683 }
4684
4685 SDValue NewCst = CurDAG->getSignedConstant(Val: ShiftedVal, DL: dl, VT: NVT);
4686 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(N, 0), N: NewCst);
4687 SDValue NewBinOp = CurDAG->getNode(Opcode, DL: dl, VT: NVT, N1: X, N2: NewCst);
4688 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(N, 0), N: NewBinOp);
4689 SDValue NewSHL = CurDAG->getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: NewBinOp,
4690 N2: Shift.getOperand(i: 1));
4691 ReplaceNode(F: N, T: NewSHL.getNode());
4692 SelectCode(N: NewSHL.getNode());
4693 return true;
4694}
4695
4696bool X86DAGToDAGISel::matchVPTERNLOG(SDNode *Root, SDNode *ParentA,
4697 SDNode *ParentB, SDNode *ParentC,
4698 SDValue A, SDValue B, SDValue C,
4699 uint8_t Imm) {
4700 assert(A.isOperandOf(ParentA) && B.isOperandOf(ParentB) &&
4701 C.isOperandOf(ParentC) && "Incorrect parent node");
4702
4703 auto tryFoldLoadOrBCast =
4704 [this](SDNode *Root, SDNode *P, SDValue &L, SDValue &Base, SDValue &Scale,
4705 SDValue &Index, SDValue &Disp, SDValue &Segment) {
4706 if (tryFoldLoad(Root, P, N: L, Base, Scale, Index, Disp, Segment))
4707 return true;
4708
4709 // Not a load, check for broadcast which may be behind a bitcast.
4710 if (L.getOpcode() == ISD::BITCAST && L.hasOneUse()) {
4711 P = L.getNode();
4712 L = L.getOperand(i: 0);
4713 }
4714
4715 if (L.getOpcode() != X86ISD::VBROADCAST_LOAD)
4716 return false;
4717
4718 // Only 32 and 64 bit broadcasts are supported.
4719 auto *MemIntr = cast<MemIntrinsicSDNode>(Val&: L);
4720 unsigned Size = MemIntr->getMemoryVT().getSizeInBits();
4721 if (Size != 32 && Size != 64)
4722 return false;
4723
4724 return tryFoldBroadcast(Root, P, N: L, Base, Scale, Index, Disp, Segment);
4725 };
4726
4727 bool FoldedLoad = false;
4728 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4729 if (tryFoldLoadOrBCast(Root, ParentC, C, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4730 FoldedLoad = true;
4731 } else if (tryFoldLoadOrBCast(Root, ParentA, A, Tmp0, Tmp1, Tmp2, Tmp3,
4732 Tmp4)) {
4733 FoldedLoad = true;
4734 std::swap(a&: A, b&: C);
4735 // Swap bits 1/4 and 3/6.
4736 uint8_t OldImm = Imm;
4737 Imm = OldImm & 0xa5;
4738 if (OldImm & 0x02) Imm |= 0x10;
4739 if (OldImm & 0x10) Imm |= 0x02;
4740 if (OldImm & 0x08) Imm |= 0x40;
4741 if (OldImm & 0x40) Imm |= 0x08;
4742 } else if (tryFoldLoadOrBCast(Root, ParentB, B, Tmp0, Tmp1, Tmp2, Tmp3,
4743 Tmp4)) {
4744 FoldedLoad = true;
4745 std::swap(a&: B, b&: C);
4746 // Swap bits 1/2 and 5/6.
4747 uint8_t OldImm = Imm;
4748 Imm = OldImm & 0x99;
4749 if (OldImm & 0x02) Imm |= 0x04;
4750 if (OldImm & 0x04) Imm |= 0x02;
4751 if (OldImm & 0x20) Imm |= 0x40;
4752 if (OldImm & 0x40) Imm |= 0x20;
4753 }
4754
4755 SDLoc DL(Root);
4756
4757 SDValue TImm = CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i8);
4758
4759 MVT NVT = Root->getSimpleValueType(ResNo: 0);
4760
4761 MachineSDNode *MNode;
4762 if (FoldedLoad) {
4763 SDVTList VTs = CurDAG->getVTList(VT1: NVT, VT2: MVT::Other);
4764
4765 unsigned Opc;
4766 if (C.getOpcode() == X86ISD::VBROADCAST_LOAD) {
4767 auto *MemIntr = cast<MemIntrinsicSDNode>(Val&: C);
4768 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits();
4769 assert((EltSize == 32 || EltSize == 64) && "Unexpected broadcast size!");
4770
4771 bool UseD = EltSize == 32;
4772 if (NVT.is128BitVector())
4773 Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi;
4774 else if (NVT.is256BitVector())
4775 Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi;
4776 else if (NVT.is512BitVector())
4777 Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi;
4778 else
4779 llvm_unreachable("Unexpected vector size!");
4780 } else {
4781 bool UseD = NVT.getVectorElementType() == MVT::i32;
4782 if (NVT.is128BitVector())
4783 Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi;
4784 else if (NVT.is256BitVector())
4785 Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi;
4786 else if (NVT.is512BitVector())
4787 Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi;
4788 else
4789 llvm_unreachable("Unexpected vector size!");
4790 }
4791
4792 SDValue Ops[] = {A, B, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, TImm, C.getOperand(i: 0)};
4793 MNode = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VTs, Ops);
4794
4795 // Update the chain.
4796 ReplaceUses(F: C.getValue(R: 1), T: SDValue(MNode, 1));
4797 // Record the mem-refs
4798 CurDAG->setNodeMemRefs(N: MNode, NewMemRefs: {cast<MemSDNode>(Val&: C)->getMemOperand()});
4799 } else {
4800 bool UseD = NVT.getVectorElementType() == MVT::i32;
4801 unsigned Opc;
4802 if (NVT.is128BitVector())
4803 Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri;
4804 else if (NVT.is256BitVector())
4805 Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri;
4806 else if (NVT.is512BitVector())
4807 Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri;
4808 else
4809 llvm_unreachable("Unexpected vector size!");
4810
4811 MNode = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: NVT, Ops: {A, B, C, TImm});
4812 }
4813
4814 ReplaceUses(F: SDValue(Root, 0), T: SDValue(MNode, 0));
4815 CurDAG->RemoveDeadNode(N: Root);
4816 return true;
4817}
4818
4819// Try to match two logic ops to a VPTERNLOG.
4820// FIXME: Handle more complex patterns that use an operand more than once?
4821bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
4822 MVT NVT = N->getSimpleValueType(ResNo: 0);
4823
4824 // Make sure we support VPTERNLOG.
4825 if (!NVT.isVector() || !Subtarget->hasAVX512() ||
4826 NVT.getVectorElementType() == MVT::i1)
4827 return false;
4828
4829 // We need VLX for 128/256-bit.
4830 if (!(Subtarget->hasVLX() || NVT.is512BitVector()))
4831 return false;
4832
4833 auto getFoldableLogicOp = [](SDValue Op) {
4834 // Peek through single use bitcast.
4835 if (Op.getOpcode() == ISD::BITCAST && Op.hasOneUse())
4836 Op = Op.getOperand(i: 0);
4837
4838 if (!Op.hasOneUse())
4839 return SDValue();
4840
4841 unsigned Opc = Op.getOpcode();
4842 if (Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR ||
4843 Opc == X86ISD::ANDNP)
4844 return Op;
4845
4846 return SDValue();
4847 };
4848
4849 SDValue N0, N1, A, FoldableOp;
4850
4851 // Identify and (optionally) peel an outer NOT that wraps a pure logic tree
4852 auto tryPeelOuterNotWrappingLogic = [&](SDNode *Op) {
4853 if (Op->getOpcode() == ISD::XOR && Op->hasOneUse() &&
4854 ISD::isBuildVectorAllOnes(N: Op->getOperand(Num: 1).getNode())) {
4855 SDValue InnerOp = getFoldableLogicOp(Op->getOperand(Num: 0));
4856
4857 if (!InnerOp)
4858 return SDValue();
4859
4860 N0 = InnerOp.getOperand(i: 0);
4861 N1 = InnerOp.getOperand(i: 1);
4862 if ((FoldableOp = getFoldableLogicOp(N1))) {
4863 A = N0;
4864 return InnerOp;
4865 }
4866 if ((FoldableOp = getFoldableLogicOp(N0))) {
4867 A = N1;
4868 return InnerOp;
4869 }
4870 }
4871 return SDValue();
4872 };
4873
4874 bool PeeledOuterNot = false;
4875 SDNode *OriN = N;
4876 if (SDValue InnerOp = tryPeelOuterNotWrappingLogic(N)) {
4877 PeeledOuterNot = true;
4878 N = InnerOp.getNode();
4879 } else {
4880 N0 = N->getOperand(Num: 0);
4881 N1 = N->getOperand(Num: 1);
4882
4883 if ((FoldableOp = getFoldableLogicOp(N1)))
4884 A = N0;
4885 else if ((FoldableOp = getFoldableLogicOp(N0)))
4886 A = N1;
4887 else
4888 return false;
4889 }
4890
4891 SDValue B = FoldableOp.getOperand(i: 0);
4892 SDValue C = FoldableOp.getOperand(i: 1);
4893 SDNode *ParentA = N;
4894 SDNode *ParentB = FoldableOp.getNode();
4895 SDNode *ParentC = FoldableOp.getNode();
4896
4897 // We can build the appropriate control immediate by performing the logic
4898 // operation we're matching using these constants for A, B, and C.
4899 uint8_t TernlogMagicA = 0xf0;
4900 uint8_t TernlogMagicB = 0xcc;
4901 uint8_t TernlogMagicC = 0xaa;
4902
4903 // Some of the inputs may be inverted, peek through them and invert the
4904 // magic values accordingly.
4905 // TODO: There may be a bitcast before the xor that we should peek through.
4906 auto PeekThroughNot = [](SDValue &Op, SDNode *&Parent, uint8_t &Magic) {
4907 if (Op.getOpcode() == ISD::XOR && Op.hasOneUse() &&
4908 ISD::isBuildVectorAllOnes(N: Op.getOperand(i: 1).getNode())) {
4909 Magic = ~Magic;
4910 Parent = Op.getNode();
4911 Op = Op.getOperand(i: 0);
4912 }
4913 };
4914
4915 PeekThroughNot(A, ParentA, TernlogMagicA);
4916 PeekThroughNot(B, ParentB, TernlogMagicB);
4917 PeekThroughNot(C, ParentC, TernlogMagicC);
4918
4919 uint8_t Imm;
4920 switch (FoldableOp.getOpcode()) {
4921 default: llvm_unreachable("Unexpected opcode!");
4922 case ISD::AND: Imm = TernlogMagicB & TernlogMagicC; break;
4923 case ISD::OR: Imm = TernlogMagicB | TernlogMagicC; break;
4924 case ISD::XOR: Imm = TernlogMagicB ^ TernlogMagicC; break;
4925 case X86ISD::ANDNP: Imm = ~(TernlogMagicB) & TernlogMagicC; break;
4926 }
4927
4928 switch (N->getOpcode()) {
4929 default: llvm_unreachable("Unexpected opcode!");
4930 case X86ISD::ANDNP:
4931 if (A == N0)
4932 Imm &= ~TernlogMagicA;
4933 else
4934 Imm = ~(Imm) & TernlogMagicA;
4935 break;
4936 case ISD::AND: Imm &= TernlogMagicA; break;
4937 case ISD::OR: Imm |= TernlogMagicA; break;
4938 case ISD::XOR: Imm ^= TernlogMagicA; break;
4939 }
4940
4941 if (PeeledOuterNot)
4942 Imm = ~Imm;
4943
4944 return matchVPTERNLOG(Root: OriN, ParentA, ParentB, ParentC, A, B, C, Imm);
4945}
4946
4947/// If the high bits of an 'and' operand are known zero, try setting the
4948/// high bits of an 'and' constant operand to produce a smaller encoding by
4949/// creating a small, sign-extended negative immediate rather than a large
4950/// positive one. This reverses a transform in SimplifyDemandedBits that
4951/// shrinks mask constants by clearing bits. There is also a possibility that
4952/// the 'and' mask can be made -1, so the 'and' itself is unnecessary. In that
4953/// case, just replace the 'and'. Return 'true' if the node is replaced.
4954bool X86DAGToDAGISel::shrinkAndImmediate(SDNode *And) {
4955 // i8 is unshrinkable, i16 should be promoted to i32, and vector ops don't
4956 // have immediate operands.
4957 MVT VT = And->getSimpleValueType(ResNo: 0);
4958 if (VT != MVT::i32 && VT != MVT::i64)
4959 return false;
4960
4961 auto *And1C = dyn_cast<ConstantSDNode>(Val: And->getOperand(Num: 1));
4962 if (!And1C)
4963 return false;
4964
4965 // Bail out if the mask constant is already negative. It's can't shrink more.
4966 // If the upper 32 bits of a 64 bit mask are all zeros, we have special isel
4967 // patterns to use a 32-bit and instead of a 64-bit and by relying on the
4968 // implicit zeroing of 32 bit ops. So we should check if the lower 32 bits
4969 // are negative too.
4970 APInt MaskVal = And1C->getAPIntValue();
4971 unsigned MaskLZ = MaskVal.countl_zero();
4972 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32))
4973 return false;
4974
4975 // Don't extend into the upper 32 bits of a 64 bit mask.
4976 if (VT == MVT::i64 && MaskLZ >= 32) {
4977 MaskLZ -= 32;
4978 MaskVal = MaskVal.trunc(width: 32);
4979 }
4980
4981 SDValue And0 = And->getOperand(Num: 0);
4982 APInt HighZeros = APInt::getHighBitsSet(numBits: MaskVal.getBitWidth(), hiBitsSet: MaskLZ);
4983 APInt NegMaskVal = MaskVal | HighZeros;
4984
4985 // If a negative constant would not allow a smaller encoding, there's no need
4986 // to continue. Only change the constant when we know it's a win.
4987 unsigned MinWidth = NegMaskVal.getSignificantBits();
4988 if (MinWidth > 32 || (MinWidth > 8 && MaskVal.getSignificantBits() <= 32))
4989 return false;
4990
4991 // Extend masks if we truncated above.
4992 if (VT == MVT::i64 && MaskVal.getBitWidth() < 64) {
4993 NegMaskVal = NegMaskVal.zext(width: 64);
4994 HighZeros = HighZeros.zext(width: 64);
4995 }
4996
4997 // The variable operand must be all zeros in the top bits to allow using the
4998 // new, negative constant as the mask.
4999 // TODO: Handle constant folding?
5000 KnownBits Known0 = CurDAG->computeKnownBits(Op: And0);
5001 if (Known0.isConstant() || !HighZeros.isSubsetOf(RHS: Known0.Zero))
5002 return false;
5003
5004 // Check if the mask is -1. In that case, this is an unnecessary instruction
5005 // that escaped earlier analysis.
5006 if (NegMaskVal.isAllOnes()) {
5007 ReplaceNode(F: And, T: And0.getNode());
5008 return true;
5009 }
5010
5011 // A negative mask allows a smaller encoding. Create a new 'and' node.
5012 SDValue NewMask = CurDAG->getConstant(Val: NegMaskVal, DL: SDLoc(And), VT);
5013 insertDAGNode(DAG&: *CurDAG, Pos: SDValue(And, 0), N: NewMask);
5014 SDValue NewAnd = CurDAG->getNode(Opcode: ISD::AND, DL: SDLoc(And), VT, N1: And0, N2: NewMask);
5015 ReplaceNode(F: And, T: NewAnd.getNode());
5016 SelectCode(N: NewAnd.getNode());
5017 return true;
5018}
5019
5020static unsigned getVPTESTMOpc(MVT TestVT, bool IsTestN, bool FoldedLoad,
5021 bool FoldedBCast, bool Masked) {
5022#define VPTESTM_CASE(VT, SUFFIX) \
5023case MVT::VT: \
5024 if (Masked) \
5025 return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
5026 return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
5027
5028
5029#define VPTESTM_BROADCAST_CASES(SUFFIX) \
5030default: llvm_unreachable("Unexpected VT!"); \
5031VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
5032VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
5033VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
5034VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
5035VPTESTM_CASE(v16i32, DZ##SUFFIX) \
5036VPTESTM_CASE(v8i64, QZ##SUFFIX)
5037
5038#define VPTESTM_FULL_CASES(SUFFIX) \
5039VPTESTM_BROADCAST_CASES(SUFFIX) \
5040VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
5041VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
5042VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
5043VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
5044VPTESTM_CASE(v64i8, BZ##SUFFIX) \
5045VPTESTM_CASE(v32i16, WZ##SUFFIX)
5046
5047 if (FoldedBCast) {
5048 switch (TestVT.SimpleTy) {
5049 VPTESTM_BROADCAST_CASES(rmb)
5050 }
5051 }
5052
5053 if (FoldedLoad) {
5054 switch (TestVT.SimpleTy) {
5055 VPTESTM_FULL_CASES(rm)
5056 }
5057 }
5058
5059 switch (TestVT.SimpleTy) {
5060 VPTESTM_FULL_CASES(rr)
5061 }
5062
5063#undef VPTESTM_FULL_CASES
5064#undef VPTESTM_BROADCAST_CASES
5065#undef VPTESTM_CASE
5066}
5067
5068static void orderRegForMul(SDValue &N0, SDValue &N1, const unsigned LoReg,
5069 const MachineRegisterInfo &MRI) {
5070 auto GetPhysReg = [&](SDValue V) -> Register {
5071 if (V.getOpcode() != ISD::CopyFromReg)
5072 return Register();
5073 Register Reg = cast<RegisterSDNode>(Val: V.getOperand(i: 1))->getReg();
5074 if (Reg.isVirtual())
5075 return MRI.getLiveInPhysReg(VReg: Reg);
5076 return Reg;
5077 };
5078
5079 if (GetPhysReg(N1) == LoReg && GetPhysReg(N0) != LoReg)
5080 std::swap(a&: N0, b&: N1);
5081}
5082
5083// Try to create VPTESTM instruction. If InMask is not null, it will be used
5084// to form a masked operation.
5085bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
5086 SDValue InMask) {
5087 assert(Subtarget->hasAVX512() && "Expected AVX512!");
5088 assert(Setcc.getSimpleValueType().getVectorElementType() == MVT::i1 &&
5089 "Unexpected VT!");
5090
5091 // Look for equal and not equal compares.
5092 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Setcc.getOperand(i: 2))->get();
5093 if (CC != ISD::SETEQ && CC != ISD::SETNE)
5094 return false;
5095
5096 SDValue SetccOp0 = Setcc.getOperand(i: 0);
5097 SDValue SetccOp1 = Setcc.getOperand(i: 1);
5098
5099 // Canonicalize the all zero vector to the RHS.
5100 if (ISD::isBuildVectorAllZeros(N: SetccOp0.getNode()))
5101 std::swap(a&: SetccOp0, b&: SetccOp1);
5102
5103 // See if we're comparing against zero.
5104 if (!ISD::isBuildVectorAllZeros(N: SetccOp1.getNode()))
5105 return false;
5106
5107 SDValue N0 = SetccOp0;
5108
5109 MVT CmpVT = N0.getSimpleValueType();
5110 MVT CmpSVT = CmpVT.getVectorElementType();
5111
5112 // Start with both operands the same. We'll try to refine this.
5113 SDValue Src0 = N0;
5114 SDValue Src1 = N0;
5115
5116 {
5117 // Look through single use bitcasts.
5118 SDValue N0Temp = N0;
5119 if (N0Temp.getOpcode() == ISD::BITCAST && N0Temp.hasOneUse())
5120 N0Temp = N0.getOperand(i: 0);
5121
5122 // Look for single use AND.
5123 if (N0Temp.getOpcode() == ISD::AND && N0Temp.hasOneUse()) {
5124 Src0 = N0Temp.getOperand(i: 0);
5125 Src1 = N0Temp.getOperand(i: 1);
5126 }
5127 }
5128
5129 // Without VLX we need to widen the operation.
5130 bool Widen = !Subtarget->hasVLX() && !CmpVT.is512BitVector();
5131
5132 auto tryFoldLoadOrBCast = [&](SDNode *Root, SDNode *P, SDValue &L,
5133 SDValue &Base, SDValue &Scale, SDValue &Index,
5134 SDValue &Disp, SDValue &Segment) {
5135 // If we need to widen, we can't fold the load.
5136 if (!Widen)
5137 if (tryFoldLoad(Root, P, N: L, Base, Scale, Index, Disp, Segment))
5138 return true;
5139
5140 // If we didn't fold a load, try to match broadcast. No widening limitation
5141 // for this. But only 32 and 64 bit types are supported.
5142 if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64)
5143 return false;
5144
5145 // Look through single use bitcasts.
5146 if (L.getOpcode() == ISD::BITCAST && L.hasOneUse()) {
5147 P = L.getNode();
5148 L = L.getOperand(i: 0);
5149 }
5150
5151 if (L.getOpcode() != X86ISD::VBROADCAST_LOAD)
5152 return false;
5153
5154 auto *MemIntr = cast<MemIntrinsicSDNode>(Val&: L);
5155 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.getSizeInBits())
5156 return false;
5157
5158 return tryFoldBroadcast(Root, P, N: L, Base, Scale, Index, Disp, Segment);
5159 };
5160
5161 // We can only fold loads if the sources are unique.
5162 bool CanFoldLoads = Src0 != Src1;
5163
5164 bool FoldedLoad = false;
5165 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5166 if (CanFoldLoads) {
5167 FoldedLoad = tryFoldLoadOrBCast(Root, N0.getNode(), Src1, Tmp0, Tmp1, Tmp2,
5168 Tmp3, Tmp4);
5169 if (!FoldedLoad) {
5170 // And is commutative.
5171 FoldedLoad = tryFoldLoadOrBCast(Root, N0.getNode(), Src0, Tmp0, Tmp1,
5172 Tmp2, Tmp3, Tmp4);
5173 if (FoldedLoad)
5174 std::swap(a&: Src0, b&: Src1);
5175 }
5176 }
5177
5178 bool FoldedBCast = FoldedLoad && Src1.getOpcode() == X86ISD::VBROADCAST_LOAD;
5179
5180 bool IsMasked = InMask.getNode() != nullptr;
5181
5182 SDLoc dl(Root);
5183
5184 MVT ResVT = Setcc.getSimpleValueType();
5185 MVT MaskVT = ResVT;
5186 if (Widen) {
5187 // Widen the inputs using insert_subreg or copy_to_regclass.
5188 unsigned Scale = CmpVT.is128BitVector() ? 4 : 2;
5189 unsigned SubReg = CmpVT.is128BitVector() ? X86::sub_xmm : X86::sub_ymm;
5190 unsigned NumElts = CmpVT.getVectorNumElements() * Scale;
5191 CmpVT = MVT::getVectorVT(VT: CmpSVT, NumElements: NumElts);
5192 MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
5193 SDValue ImplDef = SDValue(CurDAG->getMachineNode(Opcode: X86::IMPLICIT_DEF, dl,
5194 VT: CmpVT), 0);
5195 Src0 = CurDAG->getTargetInsertSubreg(SRIdx: SubReg, DL: dl, VT: CmpVT, Operand: ImplDef, Subreg: Src0);
5196
5197 if (!FoldedBCast)
5198 Src1 = CurDAG->getTargetInsertSubreg(SRIdx: SubReg, DL: dl, VT: CmpVT, Operand: ImplDef, Subreg: Src1);
5199
5200 if (IsMasked) {
5201 // Widen the mask.
5202 unsigned RegClass = TLI->getRegClassFor(VT: MaskVT)->getID();
5203 SDValue RC = CurDAG->getTargetConstant(Val: RegClass, DL: dl, VT: MVT::i32);
5204 InMask = SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
5205 dl, VT: MaskVT, Op1: InMask, Op2: RC), 0);
5206 }
5207 }
5208
5209 bool IsTestN = CC == ISD::SETEQ;
5210 unsigned Opc = getVPTESTMOpc(TestVT: CmpVT, IsTestN, FoldedLoad, FoldedBCast,
5211 Masked: IsMasked);
5212
5213 MachineSDNode *CNode;
5214 if (FoldedLoad) {
5215 SDVTList VTs = CurDAG->getVTList(VT1: MaskVT, VT2: MVT::Other);
5216
5217 if (IsMasked) {
5218 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
5219 Src1.getOperand(i: 0) };
5220 CNode = CurDAG->getMachineNode(Opcode: Opc, dl, VTs, Ops);
5221 } else {
5222 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
5223 Src1.getOperand(i: 0) };
5224 CNode = CurDAG->getMachineNode(Opcode: Opc, dl, VTs, Ops);
5225 }
5226
5227 // Update the chain.
5228 ReplaceUses(F: Src1.getValue(R: 1), T: SDValue(CNode, 1));
5229 // Record the mem-refs
5230 CurDAG->setNodeMemRefs(N: CNode, NewMemRefs: {cast<MemSDNode>(Val&: Src1)->getMemOperand()});
5231 } else {
5232 if (IsMasked)
5233 CNode = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MaskVT, Op1: InMask, Op2: Src0, Op3: Src1);
5234 else
5235 CNode = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MaskVT, Op1: Src0, Op2: Src1);
5236 }
5237
5238 // If we widened, we need to shrink the mask VT.
5239 if (Widen) {
5240 unsigned RegClass = TLI->getRegClassFor(VT: ResVT)->getID();
5241 SDValue RC = CurDAG->getTargetConstant(Val: RegClass, DL: dl, VT: MVT::i32);
5242 CNode = CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
5243 dl, VT: ResVT, Op1: SDValue(CNode, 0), Op2: RC);
5244 }
5245
5246 ReplaceUses(F: SDValue(Root, 0), T: SDValue(CNode, 0));
5247 CurDAG->RemoveDeadNode(N: Root);
5248 return true;
5249}
5250
5251// Try to match the bitselect pattern (or (and A, B), (andn A, C)). Turn it
5252// into vpternlog.
5253bool X86DAGToDAGISel::tryMatchBitSelect(SDNode *N) {
5254 assert(N->getOpcode() == ISD::OR && "Unexpected opcode!");
5255
5256 MVT NVT = N->getSimpleValueType(ResNo: 0);
5257
5258 // Make sure we support VPTERNLOG.
5259 if (!NVT.isVector() || !Subtarget->hasAVX512())
5260 return false;
5261
5262 // We need VLX for 128/256-bit.
5263 if (!(Subtarget->hasVLX() || NVT.is512BitVector()))
5264 return false;
5265
5266 SDValue N0 = N->getOperand(Num: 0);
5267 SDValue N1 = N->getOperand(Num: 1);
5268
5269 // Canonicalize AND to LHS.
5270 if (N1.getOpcode() == ISD::AND)
5271 std::swap(a&: N0, b&: N1);
5272
5273 if (N0.getOpcode() != ISD::AND ||
5274 N1.getOpcode() != X86ISD::ANDNP ||
5275 !N0.hasOneUse() || !N1.hasOneUse())
5276 return false;
5277
5278 // ANDN is not commutable, use it to pick down A and C.
5279 SDValue A = N1.getOperand(i: 0);
5280 SDValue C = N1.getOperand(i: 1);
5281
5282 // AND is commutable, if one operand matches A, the other operand is B.
5283 // Otherwise this isn't a match.
5284 SDValue B;
5285 if (N0.getOperand(i: 0) == A)
5286 B = N0.getOperand(i: 1);
5287 else if (N0.getOperand(i: 1) == A)
5288 B = N0.getOperand(i: 0);
5289 else
5290 return false;
5291
5292 SDLoc dl(N);
5293 SDValue Imm = CurDAG->getTargetConstant(Val: 0xCA, DL: dl, VT: MVT::i8);
5294 SDValue Ternlog = CurDAG->getNode(Opcode: X86ISD::VPTERNLOG, DL: dl, VT: NVT, N1: A, N2: B, N3: C, N4: Imm);
5295 ReplaceNode(F: N, T: Ternlog.getNode());
5296
5297 return matchVPTERNLOG(Root: Ternlog.getNode(), ParentA: Ternlog.getNode(), ParentB: Ternlog.getNode(),
5298 ParentC: Ternlog.getNode(), A, B, C, Imm: 0xCA);
5299}
5300
5301void X86DAGToDAGISel::Select(SDNode *Node) {
5302 MVT NVT = Node->getSimpleValueType(ResNo: 0);
5303 unsigned Opcode = Node->getOpcode();
5304 SDLoc dl(Node);
5305
5306 if (Node->isMachineOpcode()) {
5307 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n');
5308 Node->setNodeId(-1);
5309 return; // Already selected.
5310 }
5311
5312 switch (Opcode) {
5313 default: break;
5314 case ISD::INTRINSIC_W_CHAIN: {
5315 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
5316 switch (IntNo) {
5317 default: break;
5318 case Intrinsic::x86_encodekey128:
5319 case Intrinsic::x86_encodekey256: {
5320 if (!Subtarget->hasKL())
5321 break;
5322
5323 unsigned Opcode;
5324 switch (IntNo) {
5325 default: llvm_unreachable("Impossible intrinsic");
5326 case Intrinsic::x86_encodekey128:
5327 Opcode = X86::ENCODEKEY128;
5328 break;
5329 case Intrinsic::x86_encodekey256:
5330 Opcode = X86::ENCODEKEY256;
5331 break;
5332 }
5333
5334 SDValue Chain = Node->getOperand(Num: 0);
5335 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::XMM0, N: Node->getOperand(Num: 3),
5336 Glue: SDValue());
5337 if (Opcode == X86::ENCODEKEY256)
5338 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::XMM1, N: Node->getOperand(Num: 4),
5339 Glue: Chain.getValue(R: 1));
5340
5341 MachineSDNode *Res = CurDAG->getMachineNode(
5342 Opcode, dl, VTs: Node->getVTList(),
5343 Ops: {Node->getOperand(Num: 2), Chain, Chain.getValue(R: 1)});
5344 ReplaceNode(F: Node, T: Res);
5345 return;
5346 }
5347 case Intrinsic::x86_tileloaddrs64_internal:
5348 case Intrinsic::x86_tileloaddrst164_internal:
5349 if (!Subtarget->hasAMXMOVRS())
5350 break;
5351 [[fallthrough]];
5352 case Intrinsic::x86_tileloadd64_internal:
5353 case Intrinsic::x86_tileloaddt164_internal: {
5354 if (!Subtarget->hasAMXTILE())
5355 break;
5356 auto *MFI =
5357 CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
5358 MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
5359 unsigned Opc;
5360 switch (IntNo) {
5361 default:
5362 llvm_unreachable("Unexpected intrinsic!");
5363 case Intrinsic::x86_tileloaddrs64_internal:
5364 Opc = X86::PTILELOADDRSV;
5365 break;
5366 case Intrinsic::x86_tileloaddrst164_internal:
5367 Opc = X86::PTILELOADDRST1V;
5368 break;
5369 case Intrinsic::x86_tileloadd64_internal:
5370 Opc = X86::PTILELOADDV;
5371 break;
5372 case Intrinsic::x86_tileloaddt164_internal:
5373 Opc = X86::PTILELOADDT1V;
5374 break;
5375 }
5376 // _tile_loadd_internal(row, col, buf, STRIDE)
5377 SDValue Base = Node->getOperand(Num: 4);
5378 SDValue Scale = getI8Imm(Imm: 1, DL: dl);
5379 SDValue Index = Node->getOperand(Num: 5);
5380 SDValue Disp = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
5381 SDValue Segment = CurDAG->getRegister(Reg: 0, VT: MVT::i16);
5382 SDValue Chain = Node->getOperand(Num: 0);
5383 MachineSDNode *CNode;
5384 SDValue Ops[] = {Node->getOperand(Num: 2),
5385 Node->getOperand(Num: 3),
5386 Base,
5387 Scale,
5388 Index,
5389 Disp,
5390 Segment,
5391 Chain};
5392 CNode = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: {MVT::x86amx, MVT::Other}, Ops);
5393 ReplaceNode(F: Node, T: CNode);
5394 return;
5395 }
5396 }
5397 break;
5398 }
5399 case ISD::INTRINSIC_VOID: {
5400 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
5401 switch (IntNo) {
5402 default: break;
5403 case Intrinsic::x86_sse3_monitor:
5404 case Intrinsic::x86_monitorx:
5405 case Intrinsic::x86_clzero: {
5406 bool Use64BitPtr = Node->getOperand(Num: 2).getValueType() == MVT::i64;
5407
5408 unsigned Opc = 0;
5409 switch (IntNo) {
5410 default: llvm_unreachable("Unexpected intrinsic!");
5411 case Intrinsic::x86_sse3_monitor:
5412 if (!Subtarget->hasSSE3())
5413 break;
5414 Opc = Use64BitPtr ? X86::MONITOR64rrr : X86::MONITOR32rrr;
5415 break;
5416 case Intrinsic::x86_monitorx:
5417 if (!Subtarget->hasMWAITX())
5418 break;
5419 Opc = Use64BitPtr ? X86::MONITORX64rrr : X86::MONITORX32rrr;
5420 break;
5421 case Intrinsic::x86_clzero:
5422 if (!Subtarget->hasCLZERO())
5423 break;
5424 Opc = Use64BitPtr ? X86::CLZERO64r : X86::CLZERO32r;
5425 break;
5426 }
5427
5428 if (Opc) {
5429 unsigned PtrReg = Use64BitPtr ? X86::RAX : X86::EAX;
5430 SDValue Chain = CurDAG->getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: PtrReg,
5431 N: Node->getOperand(Num: 2), Glue: SDValue());
5432 SDValue InGlue = Chain.getValue(R: 1);
5433
5434 if (IntNo == Intrinsic::x86_sse3_monitor ||
5435 IntNo == Intrinsic::x86_monitorx) {
5436 // Copy the other two operands to ECX and EDX.
5437 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::ECX, N: Node->getOperand(Num: 3),
5438 Glue: InGlue);
5439 InGlue = Chain.getValue(R: 1);
5440 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::EDX, N: Node->getOperand(Num: 4),
5441 Glue: InGlue);
5442 InGlue = Chain.getValue(R: 1);
5443 }
5444
5445 MachineSDNode *CNode = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::Other,
5446 Ops: { Chain, InGlue});
5447 ReplaceNode(F: Node, T: CNode);
5448 return;
5449 }
5450
5451 break;
5452 }
5453 case Intrinsic::x86_tilestored64_internal: {
5454 auto *MFI =
5455 CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
5456 MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
5457 unsigned Opc = X86::PTILESTOREDV;
5458 // _tile_stored_internal(row, col, buf, STRIDE, c)
5459 SDValue Base = Node->getOperand(Num: 4);
5460 SDValue Scale = getI8Imm(Imm: 1, DL: dl);
5461 SDValue Index = Node->getOperand(Num: 5);
5462 SDValue Disp = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
5463 SDValue Segment = CurDAG->getRegister(Reg: 0, VT: MVT::i16);
5464 SDValue Chain = Node->getOperand(Num: 0);
5465 MachineSDNode *CNode;
5466 SDValue Ops[] = {Node->getOperand(Num: 2),
5467 Node->getOperand(Num: 3),
5468 Base,
5469 Scale,
5470 Index,
5471 Disp,
5472 Segment,
5473 Node->getOperand(Num: 6),
5474 Chain};
5475 CNode = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::Other, Ops);
5476 ReplaceNode(F: Node, T: CNode);
5477 return;
5478 }
5479 case Intrinsic::x86_tileloaddrs64:
5480 case Intrinsic::x86_tileloaddrst164:
5481 if (!Subtarget->hasAMXMOVRS())
5482 break;
5483 [[fallthrough]];
5484 case Intrinsic::x86_tileloadd64:
5485 case Intrinsic::x86_tileloaddt164:
5486 case Intrinsic::x86_tilestored64: {
5487 if (!Subtarget->hasAMXTILE())
5488 break;
5489 auto *MFI =
5490 CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
5491 MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
5492 unsigned Opc;
5493 switch (IntNo) {
5494 default: llvm_unreachable("Unexpected intrinsic!");
5495 case Intrinsic::x86_tileloadd64: Opc = X86::PTILELOADD; break;
5496 case Intrinsic::x86_tileloaddrs64:
5497 Opc = X86::PTILELOADDRS;
5498 break;
5499 case Intrinsic::x86_tileloaddt164: Opc = X86::PTILELOADDT1; break;
5500 case Intrinsic::x86_tileloaddrst164:
5501 Opc = X86::PTILELOADDRST1;
5502 break;
5503 case Intrinsic::x86_tilestored64: Opc = X86::PTILESTORED; break;
5504 }
5505 // FIXME: Match displacement and scale.
5506 unsigned TIndex = Node->getConstantOperandVal(Num: 2);
5507 SDValue TReg = getI8Imm(Imm: TIndex, DL: dl);
5508 SDValue Base = Node->getOperand(Num: 3);
5509 SDValue Scale = getI8Imm(Imm: 1, DL: dl);
5510 SDValue Index = Node->getOperand(Num: 4);
5511 SDValue Disp = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
5512 SDValue Segment = CurDAG->getRegister(Reg: 0, VT: MVT::i16);
5513 SDValue Chain = Node->getOperand(Num: 0);
5514 MachineSDNode *CNode;
5515 if (Opc == X86::PTILESTORED) {
5516 SDValue Ops[] = { Base, Scale, Index, Disp, Segment, TReg, Chain };
5517 CNode = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::Other, Ops);
5518 } else {
5519 SDValue Ops[] = { TReg, Base, Scale, Index, Disp, Segment, Chain };
5520 CNode = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::Other, Ops);
5521 }
5522 ReplaceNode(F: Node, T: CNode);
5523 return;
5524 }
5525 }
5526 break;
5527 }
5528 case ISD::BRIND:
5529 case X86ISD::NT_BRIND: {
5530 if (Subtarget->isTarget64BitILP32()) {
5531 // Converts a 32-bit register to a 64-bit, zero-extended version of
5532 // it. This is needed because x86-64 can do many things, but jmp %r32
5533 // ain't one of them.
5534 SDValue Target = Node->getOperand(Num: 1);
5535 assert(Target.getValueType() == MVT::i32 && "Unexpected VT!");
5536 SDValue ZextTarget = CurDAG->getZExtOrTrunc(Op: Target, DL: dl, VT: MVT::i64);
5537 SDValue Brind = CurDAG->getNode(Opcode, DL: dl, VT: MVT::Other,
5538 N1: Node->getOperand(Num: 0), N2: ZextTarget);
5539 ReplaceNode(F: Node, T: Brind.getNode());
5540 SelectCode(N: ZextTarget.getNode());
5541 SelectCode(N: Brind.getNode());
5542 return;
5543 }
5544 break;
5545 }
5546 case X86ISD::GlobalBaseReg:
5547 ReplaceNode(F: Node, T: getGlobalBaseReg());
5548 return;
5549
5550 case ISD::BITCAST:
5551 // Just drop all 128/256/512-bit bitcasts.
5552 if (NVT.is512BitVector() || NVT.is256BitVector() || NVT.is128BitVector() ||
5553 NVT == MVT::f128) {
5554 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
5555 CurDAG->RemoveDeadNode(N: Node);
5556 return;
5557 }
5558 break;
5559
5560 case ISD::SRL:
5561 if (matchBitExtract(Node))
5562 return;
5563 [[fallthrough]];
5564 case ISD::SRA:
5565 case ISD::SHL:
5566 if (tryShiftAmountMod(N: Node))
5567 return;
5568 break;
5569
5570 case X86ISD::VPTERNLOG: {
5571 uint8_t Imm = Node->getConstantOperandVal(Num: 3);
5572 if (matchVPTERNLOG(Root: Node, ParentA: Node, ParentB: Node, ParentC: Node, A: Node->getOperand(Num: 0),
5573 B: Node->getOperand(Num: 1), C: Node->getOperand(Num: 2), Imm))
5574 return;
5575 break;
5576 }
5577
5578 case X86ISD::ANDNP:
5579 if (tryVPTERNLOG(N: Node))
5580 return;
5581 break;
5582
5583 case ISD::AND:
5584 if (NVT.isVectorOf(EltVT: MVT::i1)) {
5585 // Try to form a masked VPTESTM. Operands can be in either order.
5586 SDValue N0 = Node->getOperand(Num: 0);
5587 SDValue N1 = Node->getOperand(Num: 1);
5588 if (N0.getOpcode() == ISD::SETCC && N0.hasOneUse() &&
5589 tryVPTESTM(Root: Node, Setcc: N0, InMask: N1))
5590 return;
5591 if (N1.getOpcode() == ISD::SETCC && N1.hasOneUse() &&
5592 tryVPTESTM(Root: Node, Setcc: N1, InMask: N0))
5593 return;
5594 }
5595
5596 if (MachineSDNode *NewNode = matchBEXTRFromAndImm(Node)) {
5597 ReplaceUses(F: SDValue(Node, 0), T: SDValue(NewNode, 0));
5598 CurDAG->RemoveDeadNode(N: Node);
5599 return;
5600 }
5601 if (matchBitExtract(Node))
5602 return;
5603 if (AndImmShrink && shrinkAndImmediate(And: Node))
5604 return;
5605
5606 [[fallthrough]];
5607 case ISD::OR:
5608 case ISD::XOR:
5609 if (tryShrinkShlLogicImm(N: Node))
5610 return;
5611 if (Opcode == ISD::OR && tryMatchBitSelect(N: Node))
5612 return;
5613 if (tryVPTERNLOG(N: Node))
5614 return;
5615
5616 [[fallthrough]];
5617 case ISD::ADD:
5618 if (Opcode == ISD::ADD && matchBitExtract(Node))
5619 return;
5620 [[fallthrough]];
5621 case ISD::SUB: {
5622 // Try to avoid folding immediates with multiple uses for optsize.
5623 // This code tries to select to register form directly to avoid going
5624 // through the isel table which might fold the immediate. We can't change
5625 // the patterns on the add/sub/and/or/xor with immediate paterns in the
5626 // tablegen files to check immediate use count without making the patterns
5627 // unavailable to the fast-isel table.
5628 if (!CurDAG->shouldOptForSize())
5629 break;
5630
5631 // Only handle i8/i16/i32/i64.
5632 if (NVT != MVT::i8 && NVT != MVT::i16 && NVT != MVT::i32 && NVT != MVT::i64)
5633 break;
5634
5635 SDValue N0 = Node->getOperand(Num: 0);
5636 SDValue N1 = Node->getOperand(Num: 1);
5637
5638 auto *Cst = dyn_cast<ConstantSDNode>(Val&: N1);
5639 if (!Cst)
5640 break;
5641
5642 int64_t Val = Cst->getSExtValue();
5643
5644 // Make sure its an immediate that is considered foldable.
5645 // FIXME: Handle unsigned 32 bit immediates for 64-bit AND.
5646 if (!isInt<8>(x: Val) && !isInt<32>(x: Val))
5647 break;
5648
5649 // If this can match to INC/DEC, let it go.
5650 if (Opcode == ISD::ADD && (Val == 1 || Val == -1))
5651 break;
5652
5653 // Check if we should avoid folding this immediate.
5654 if (!shouldAvoidImmediateInstFormsForSize(N: N1.getNode()))
5655 break;
5656
5657 // We should not fold the immediate. So we need a register form instead.
5658 unsigned ROpc, MOpc;
5659 switch (NVT.SimpleTy) {
5660 default: llvm_unreachable("Unexpected VT!");
5661 case MVT::i8:
5662 switch (Opcode) {
5663 default: llvm_unreachable("Unexpected opcode!");
5664 case ISD::ADD:
5665 ROpc = GET_ND_IF_ENABLED(X86::ADD8rr);
5666 MOpc = GET_NDM_IF_ENABLED(X86::ADD8rm);
5667 break;
5668 case ISD::SUB:
5669 ROpc = GET_ND_IF_ENABLED(X86::SUB8rr);
5670 MOpc = GET_NDM_IF_ENABLED(X86::SUB8rm);
5671 break;
5672 case ISD::AND:
5673 ROpc = GET_ND_IF_ENABLED(X86::AND8rr);
5674 MOpc = GET_NDM_IF_ENABLED(X86::AND8rm);
5675 break;
5676 case ISD::OR:
5677 ROpc = GET_ND_IF_ENABLED(X86::OR8rr);
5678 MOpc = GET_NDM_IF_ENABLED(X86::OR8rm);
5679 break;
5680 case ISD::XOR:
5681 ROpc = GET_ND_IF_ENABLED(X86::XOR8rr);
5682 MOpc = GET_NDM_IF_ENABLED(X86::XOR8rm);
5683 break;
5684 }
5685 break;
5686 case MVT::i16:
5687 switch (Opcode) {
5688 default: llvm_unreachable("Unexpected opcode!");
5689 case ISD::ADD:
5690 ROpc = GET_ND_IF_ENABLED(X86::ADD16rr);
5691 MOpc = GET_NDM_IF_ENABLED(X86::ADD16rm);
5692 break;
5693 case ISD::SUB:
5694 ROpc = GET_ND_IF_ENABLED(X86::SUB16rr);
5695 MOpc = GET_NDM_IF_ENABLED(X86::SUB16rm);
5696 break;
5697 case ISD::AND:
5698 ROpc = GET_ND_IF_ENABLED(X86::AND16rr);
5699 MOpc = GET_NDM_IF_ENABLED(X86::AND16rm);
5700 break;
5701 case ISD::OR:
5702 ROpc = GET_ND_IF_ENABLED(X86::OR16rr);
5703 MOpc = GET_NDM_IF_ENABLED(X86::OR16rm);
5704 break;
5705 case ISD::XOR:
5706 ROpc = GET_ND_IF_ENABLED(X86::XOR16rr);
5707 MOpc = GET_NDM_IF_ENABLED(X86::XOR16rm);
5708 break;
5709 }
5710 break;
5711 case MVT::i32:
5712 switch (Opcode) {
5713 default: llvm_unreachable("Unexpected opcode!");
5714 case ISD::ADD:
5715 ROpc = GET_ND_IF_ENABLED(X86::ADD32rr);
5716 MOpc = GET_NDM_IF_ENABLED(X86::ADD32rm);
5717 break;
5718 case ISD::SUB:
5719 ROpc = GET_ND_IF_ENABLED(X86::SUB32rr);
5720 MOpc = GET_NDM_IF_ENABLED(X86::SUB32rm);
5721 break;
5722 case ISD::AND:
5723 ROpc = GET_ND_IF_ENABLED(X86::AND32rr);
5724 MOpc = GET_NDM_IF_ENABLED(X86::AND32rm);
5725 break;
5726 case ISD::OR:
5727 ROpc = GET_ND_IF_ENABLED(X86::OR32rr);
5728 MOpc = GET_NDM_IF_ENABLED(X86::OR32rm);
5729 break;
5730 case ISD::XOR:
5731 ROpc = GET_ND_IF_ENABLED(X86::XOR32rr);
5732 MOpc = GET_NDM_IF_ENABLED(X86::XOR32rm);
5733 break;
5734 }
5735 break;
5736 case MVT::i64:
5737 switch (Opcode) {
5738 default: llvm_unreachable("Unexpected opcode!");
5739 case ISD::ADD:
5740 ROpc = GET_ND_IF_ENABLED(X86::ADD64rr);
5741 MOpc = GET_NDM_IF_ENABLED(X86::ADD64rm);
5742 break;
5743 case ISD::SUB:
5744 ROpc = GET_ND_IF_ENABLED(X86::SUB64rr);
5745 MOpc = GET_NDM_IF_ENABLED(X86::SUB64rm);
5746 break;
5747 case ISD::AND:
5748 ROpc = GET_ND_IF_ENABLED(X86::AND64rr);
5749 MOpc = GET_NDM_IF_ENABLED(X86::AND64rm);
5750 break;
5751 case ISD::OR:
5752 ROpc = GET_ND_IF_ENABLED(X86::OR64rr);
5753 MOpc = GET_NDM_IF_ENABLED(X86::OR64rm);
5754 break;
5755 case ISD::XOR:
5756 ROpc = GET_ND_IF_ENABLED(X86::XOR64rr);
5757 MOpc = GET_NDM_IF_ENABLED(X86::XOR64rm);
5758 break;
5759 }
5760 break;
5761 }
5762
5763 // Ok this is a AND/OR/XOR/ADD/SUB with constant.
5764
5765 // If this is a not a subtract, we can still try to fold a load.
5766 if (Opcode != ISD::SUB) {
5767 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5768 if (tryFoldLoad(P: Node, N: N0, Base&: Tmp0, Scale&: Tmp1, Index&: Tmp2, Disp&: Tmp3, Segment&: Tmp4)) {
5769 SDValue Ops[] = { N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(i: 0) };
5770 SDVTList VTs = CurDAG->getVTList(VT1: NVT, VT2: MVT::i32, VT3: MVT::Other);
5771 MachineSDNode *CNode = CurDAG->getMachineNode(Opcode: MOpc, dl, VTs, Ops);
5772 // Update the chain.
5773 ReplaceUses(F: N0.getValue(R: 1), T: SDValue(CNode, 2));
5774 // Record the mem-refs
5775 CurDAG->setNodeMemRefs(N: CNode, NewMemRefs: {cast<LoadSDNode>(Val&: N0)->getMemOperand()});
5776 ReplaceUses(F: SDValue(Node, 0), T: SDValue(CNode, 0));
5777 CurDAG->RemoveDeadNode(N: Node);
5778 return;
5779 }
5780 }
5781
5782 CurDAG->SelectNodeTo(N: Node, MachineOpc: ROpc, VT1: NVT, VT2: MVT::i32, Op1: N0, Op2: N1);
5783 return;
5784 }
5785
5786 case X86ISD::SMUL:
5787 // i16/i32/i64 are handled with isel patterns.
5788 if (NVT != MVT::i8)
5789 break;
5790 [[fallthrough]];
5791 case X86ISD::UMUL: {
5792 SDValue N0 = Node->getOperand(Num: 0);
5793 SDValue N1 = Node->getOperand(Num: 1);
5794
5795 unsigned LoReg, ROpc, MOpc;
5796 switch (NVT.SimpleTy) {
5797 default: llvm_unreachable("Unsupported VT!");
5798 case MVT::i8:
5799 LoReg = X86::AL;
5800 ROpc = Opcode == X86ISD::SMUL ? X86::IMUL8r : X86::MUL8r;
5801 MOpc = Opcode == X86ISD::SMUL ? X86::IMUL8m : X86::MUL8m;
5802 break;
5803 case MVT::i16:
5804 LoReg = X86::AX;
5805 ROpc = X86::MUL16r;
5806 MOpc = X86::MUL16m;
5807 break;
5808 case MVT::i32:
5809 LoReg = X86::EAX;
5810 ROpc = X86::MUL32r;
5811 MOpc = X86::MUL32m;
5812 break;
5813 case MVT::i64:
5814 LoReg = X86::RAX;
5815 ROpc = X86::MUL64r;
5816 MOpc = X86::MUL64m;
5817 break;
5818 }
5819
5820 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5821 bool FoldedLoad = tryFoldLoad(P: Node, N: N1, Base&: Tmp0, Scale&: Tmp1, Index&: Tmp2, Disp&: Tmp3, Segment&: Tmp4);
5822 // Multiply is commutative.
5823 if (!FoldedLoad) {
5824 FoldedLoad = tryFoldLoad(P: Node, N: N0, Base&: Tmp0, Scale&: Tmp1, Index&: Tmp2, Disp&: Tmp3, Segment&: Tmp4);
5825 if (FoldedLoad)
5826 std::swap(a&: N0, b&: N1);
5827 }
5828
5829 // UMUL/SMUL have an implicit source in LoReg (AL/AX/EAX/RAX). Prefer the
5830 // operand that's already there to avoid an extra register-to-register move.
5831 if (!FoldedLoad)
5832 orderRegForMul(N0, N1, LoReg, MRI: CurDAG->getMachineFunction().getRegInfo());
5833
5834 SDValue InGlue = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl, Reg: LoReg,
5835 N: N0, Glue: SDValue()).getValue(R: 1);
5836
5837 MachineSDNode *CNode;
5838 if (FoldedLoad) {
5839 // i16/i32/i64 use an instruction that produces a low and high result even
5840 // though only the low result is used.
5841 SDVTList VTs;
5842 if (NVT == MVT::i8)
5843 VTs = CurDAG->getVTList(VT1: NVT, VT2: MVT::i32, VT3: MVT::Other);
5844 else
5845 VTs = CurDAG->getVTList(VT1: NVT, VT2: NVT, VT3: MVT::i32, VT4: MVT::Other);
5846
5847 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(i: 0),
5848 InGlue };
5849 CNode = CurDAG->getMachineNode(Opcode: MOpc, dl, VTs, Ops);
5850
5851 // Update the chain.
5852 ReplaceUses(F: N1.getValue(R: 1), T: SDValue(CNode, NVT == MVT::i8 ? 2 : 3));
5853 // Record the mem-refs
5854 CurDAG->setNodeMemRefs(N: CNode, NewMemRefs: {cast<LoadSDNode>(Val&: N1)->getMemOperand()});
5855 } else {
5856 // i16/i32/i64 use an instruction that produces a low and high result even
5857 // though only the low result is used.
5858 SDVTList VTs;
5859 if (NVT == MVT::i8)
5860 VTs = CurDAG->getVTList(VT1: NVT, VT2: MVT::i32);
5861 else
5862 VTs = CurDAG->getVTList(VT1: NVT, VT2: NVT, VT3: MVT::i32);
5863
5864 CNode = CurDAG->getMachineNode(Opcode: ROpc, dl, VTs, Ops: {N1, InGlue});
5865 }
5866
5867 ReplaceUses(F: SDValue(Node, 0), T: SDValue(CNode, 0));
5868 ReplaceUses(F: SDValue(Node, 1), T: SDValue(CNode, NVT == MVT::i8 ? 1 : 2));
5869 CurDAG->RemoveDeadNode(N: Node);
5870 return;
5871 }
5872
5873 case ISD::SMUL_LOHI:
5874 case ISD::UMUL_LOHI: {
5875 SDValue N0 = Node->getOperand(Num: 0);
5876 SDValue N1 = Node->getOperand(Num: 1);
5877
5878 unsigned Opc, MOpc;
5879 unsigned LoReg, HiReg;
5880 bool IsSigned = Opcode == ISD::SMUL_LOHI;
5881 bool UseMULX = !IsSigned && Subtarget->hasBMI2();
5882 bool UseMULXHi = UseMULX && SDValue(Node, 0).use_empty();
5883 switch (NVT.SimpleTy) {
5884 default: llvm_unreachable("Unsupported VT!");
5885 case MVT::i32:
5886 Opc = UseMULXHi ? X86::MULX32Hrr
5887 : UseMULX ? GET_EGPR_IF_ENABLED(X86::MULX32rr)
5888 : IsSigned ? X86::IMUL32r
5889 : X86::MUL32r;
5890 MOpc = UseMULXHi ? X86::MULX32Hrm
5891 : UseMULX ? GET_EGPR_IF_ENABLED(X86::MULX32rm)
5892 : IsSigned ? X86::IMUL32m
5893 : X86::MUL32m;
5894 LoReg = UseMULX ? X86::EDX : X86::EAX;
5895 HiReg = X86::EDX;
5896 break;
5897 case MVT::i64:
5898 Opc = UseMULXHi ? X86::MULX64Hrr
5899 : UseMULX ? GET_EGPR_IF_ENABLED(X86::MULX64rr)
5900 : IsSigned ? X86::IMUL64r
5901 : X86::MUL64r;
5902 MOpc = UseMULXHi ? X86::MULX64Hrm
5903 : UseMULX ? GET_EGPR_IF_ENABLED(X86::MULX64rm)
5904 : IsSigned ? X86::IMUL64m
5905 : X86::MUL64m;
5906 LoReg = UseMULX ? X86::RDX : X86::RAX;
5907 HiReg = X86::RDX;
5908 break;
5909 }
5910
5911 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5912 bool foldedLoad = tryFoldLoad(P: Node, N: N1, Base&: Tmp0, Scale&: Tmp1, Index&: Tmp2, Disp&: Tmp3, Segment&: Tmp4);
5913 // Multiply is commutative.
5914 if (!foldedLoad) {
5915 foldedLoad = tryFoldLoad(P: Node, N: N0, Base&: Tmp0, Scale&: Tmp1, Index&: Tmp2, Disp&: Tmp3, Segment&: Tmp4);
5916 if (foldedLoad)
5917 std::swap(a&: N0, b&: N1);
5918 }
5919
5920 // UMUL/SMUL_LOHI has an implicit source in LoReg (RDX for MULX, RAX for
5921 // MUL/IMUL). Prefer the operand that's already there.
5922 if (!foldedLoad)
5923 orderRegForMul(N0, N1, LoReg, MRI: CurDAG->getMachineFunction().getRegInfo());
5924
5925 SDValue InGlue = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl, Reg: LoReg,
5926 N: N0, Glue: SDValue()).getValue(R: 1);
5927 SDValue ResHi, ResLo;
5928 if (foldedLoad) {
5929 SDValue Chain;
5930 MachineSDNode *CNode = nullptr;
5931 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(i: 0),
5932 InGlue };
5933 if (UseMULXHi) {
5934 SDVTList VTs = CurDAG->getVTList(VT1: NVT, VT2: MVT::Other);
5935 CNode = CurDAG->getMachineNode(Opcode: MOpc, dl, VTs, Ops);
5936 ResHi = SDValue(CNode, 0);
5937 Chain = SDValue(CNode, 1);
5938 } else if (UseMULX) {
5939 SDVTList VTs = CurDAG->getVTList(VT1: NVT, VT2: NVT, VT3: MVT::Other);
5940 CNode = CurDAG->getMachineNode(Opcode: MOpc, dl, VTs, Ops);
5941 ResHi = SDValue(CNode, 0);
5942 ResLo = SDValue(CNode, 1);
5943 Chain = SDValue(CNode, 2);
5944 } else {
5945 SDVTList VTs = CurDAG->getVTList(VT1: MVT::Other, VT2: MVT::Glue);
5946 CNode = CurDAG->getMachineNode(Opcode: MOpc, dl, VTs, Ops);
5947 Chain = SDValue(CNode, 0);
5948 InGlue = SDValue(CNode, 1);
5949 }
5950
5951 // Update the chain.
5952 ReplaceUses(F: N1.getValue(R: 1), T: Chain);
5953 // Record the mem-refs
5954 CurDAG->setNodeMemRefs(N: CNode, NewMemRefs: {cast<LoadSDNode>(Val&: N1)->getMemOperand()});
5955 } else {
5956 SDValue Ops[] = { N1, InGlue };
5957 if (UseMULXHi) {
5958 SDVTList VTs = CurDAG->getVTList(VT: NVT);
5959 SDNode *CNode = CurDAG->getMachineNode(Opcode: Opc, dl, VTs, Ops);
5960 ResHi = SDValue(CNode, 0);
5961 } else if (UseMULX) {
5962 SDVTList VTs = CurDAG->getVTList(VT1: NVT, VT2: NVT);
5963 SDNode *CNode = CurDAG->getMachineNode(Opcode: Opc, dl, VTs, Ops);
5964 ResHi = SDValue(CNode, 0);
5965 ResLo = SDValue(CNode, 1);
5966 } else {
5967 SDVTList VTs = CurDAG->getVTList(VT: MVT::Glue);
5968 SDNode *CNode = CurDAG->getMachineNode(Opcode: Opc, dl, VTs, Ops);
5969 InGlue = SDValue(CNode, 0);
5970 }
5971 }
5972
5973 // Copy the low half of the result, if it is needed.
5974 if (!SDValue(Node, 0).use_empty()) {
5975 if (!ResLo) {
5976 assert(LoReg && "Register for low half is not defined!");
5977 ResLo = CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl, Reg: LoReg,
5978 VT: NVT, Glue: InGlue);
5979 InGlue = ResLo.getValue(R: 2);
5980 }
5981 ReplaceUses(F: SDValue(Node, 0), T: ResLo);
5982 LLVM_DEBUG(dbgs() << "=> "; ResLo.getNode()->dump(CurDAG);
5983 dbgs() << '\n');
5984 }
5985 // Copy the high half of the result, if it is needed.
5986 if (!SDValue(Node, 1).use_empty()) {
5987 if (!ResHi) {
5988 assert(HiReg && "Register for high half is not defined!");
5989 ResHi = CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl, Reg: HiReg,
5990 VT: NVT, Glue: InGlue);
5991 InGlue = ResHi.getValue(R: 2);
5992 }
5993 ReplaceUses(F: SDValue(Node, 1), T: ResHi);
5994 LLVM_DEBUG(dbgs() << "=> "; ResHi.getNode()->dump(CurDAG);
5995 dbgs() << '\n');
5996 }
5997
5998 CurDAG->RemoveDeadNode(N: Node);
5999 return;
6000 }
6001
6002 case ISD::SDIVREM:
6003 case ISD::UDIVREM: {
6004 SDValue N0 = Node->getOperand(Num: 0);
6005 SDValue N1 = Node->getOperand(Num: 1);
6006
6007 unsigned ROpc, MOpc;
6008 bool isSigned = Opcode == ISD::SDIVREM;
6009 if (!isSigned) {
6010 switch (NVT.SimpleTy) {
6011 default: llvm_unreachable("Unsupported VT!");
6012 case MVT::i8: ROpc = X86::DIV8r; MOpc = X86::DIV8m; break;
6013 case MVT::i16: ROpc = X86::DIV16r; MOpc = X86::DIV16m; break;
6014 case MVT::i32: ROpc = X86::DIV32r; MOpc = X86::DIV32m; break;
6015 case MVT::i64: ROpc = X86::DIV64r; MOpc = X86::DIV64m; break;
6016 }
6017 } else {
6018 switch (NVT.SimpleTy) {
6019 default: llvm_unreachable("Unsupported VT!");
6020 case MVT::i8: ROpc = X86::IDIV8r; MOpc = X86::IDIV8m; break;
6021 case MVT::i16: ROpc = X86::IDIV16r; MOpc = X86::IDIV16m; break;
6022 case MVT::i32: ROpc = X86::IDIV32r; MOpc = X86::IDIV32m; break;
6023 case MVT::i64: ROpc = X86::IDIV64r; MOpc = X86::IDIV64m; break;
6024 }
6025 }
6026
6027 unsigned LoReg, HiReg, ClrReg;
6028 unsigned SExtOpcode;
6029 switch (NVT.SimpleTy) {
6030 default: llvm_unreachable("Unsupported VT!");
6031 case MVT::i8:
6032 LoReg = X86::AL; ClrReg = HiReg = X86::AH;
6033 SExtOpcode = 0; // Not used.
6034 break;
6035 case MVT::i16:
6036 LoReg = X86::AX; HiReg = X86::DX;
6037 ClrReg = X86::DX;
6038 SExtOpcode = X86::CWD;
6039 break;
6040 case MVT::i32:
6041 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
6042 SExtOpcode = X86::CDQ;
6043 break;
6044 case MVT::i64:
6045 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
6046 SExtOpcode = X86::CQO;
6047 break;
6048 }
6049
6050 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
6051 bool foldedLoad = tryFoldLoad(P: Node, N: N1, Base&: Tmp0, Scale&: Tmp1, Index&: Tmp2, Disp&: Tmp3, Segment&: Tmp4);
6052 bool signBitIsZero = CurDAG->SignBitIsZero(Op: N0);
6053
6054 SDValue InGlue;
6055 if (NVT == MVT::i8) {
6056 // Special case for div8, just use a move with zero extension to AX to
6057 // clear the upper 8 bits (AH).
6058 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain;
6059 MachineSDNode *Move;
6060 if (tryFoldLoad(P: Node, N: N0, Base&: Tmp0, Scale&: Tmp1, Index&: Tmp2, Disp&: Tmp3, Segment&: Tmp4)) {
6061 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(i: 0) };
6062 unsigned Opc = (isSigned && !signBitIsZero) ? X86::MOVSX16rm8
6063 : X86::MOVZX16rm8;
6064 Move = CurDAG->getMachineNode(Opcode: Opc, dl, VT1: MVT::i16, VT2: MVT::Other, Ops);
6065 Chain = SDValue(Move, 1);
6066 ReplaceUses(F: N0.getValue(R: 1), T: Chain);
6067 // Record the mem-refs
6068 CurDAG->setNodeMemRefs(N: Move, NewMemRefs: {cast<LoadSDNode>(Val&: N0)->getMemOperand()});
6069 } else {
6070 unsigned Opc = (isSigned && !signBitIsZero) ? X86::MOVSX16rr8
6071 : X86::MOVZX16rr8;
6072 Move = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::i16, Op1: N0);
6073 Chain = CurDAG->getEntryNode();
6074 }
6075 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::AX, N: SDValue(Move, 0),
6076 Glue: SDValue());
6077 InGlue = Chain.getValue(R: 1);
6078 } else {
6079 InGlue =
6080 CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl,
6081 Reg: LoReg, N: N0, Glue: SDValue()).getValue(R: 1);
6082 if (isSigned && !signBitIsZero) {
6083 // Sign extend the low part into the high part.
6084 InGlue =
6085 SDValue(CurDAG->getMachineNode(Opcode: SExtOpcode, dl, VT: MVT::Glue, Op1: InGlue),0);
6086 } else {
6087 // Zero out the high part, effectively zero extending the input.
6088 SDVTList VTs = CurDAG->getVTList(VT1: MVT::i32, VT2: MVT::i32);
6089 SDValue ClrNode =
6090 SDValue(CurDAG->getMachineNode(Opcode: X86::MOV32r0, dl, VTs, Ops: {}), 0);
6091 switch (NVT.SimpleTy) {
6092 case MVT::i16:
6093 ClrNode =
6094 SDValue(CurDAG->getMachineNode(
6095 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::i16, Op1: ClrNode,
6096 Op2: CurDAG->getTargetConstant(Val: X86::sub_16bit, DL: dl,
6097 VT: MVT::i32)),
6098 0);
6099 break;
6100 case MVT::i32:
6101 break;
6102 case MVT::i64:
6103 ClrNode = SDValue(
6104 CurDAG->getMachineNode(
6105 Opcode: TargetOpcode::SUBREG_TO_REG, dl, VT: MVT::i64, Op1: ClrNode,
6106 Op2: CurDAG->getTargetConstant(Val: X86::sub_32bit, DL: dl, VT: MVT::i32)),
6107 0);
6108 break;
6109 default:
6110 llvm_unreachable("Unexpected division source");
6111 }
6112
6113 InGlue = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl, Reg: ClrReg,
6114 N: ClrNode, Glue: InGlue).getValue(R: 1);
6115 }
6116 }
6117
6118 if (foldedLoad) {
6119 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(i: 0),
6120 InGlue };
6121 MachineSDNode *CNode =
6122 CurDAG->getMachineNode(Opcode: MOpc, dl, VT1: MVT::Other, VT2: MVT::Glue, Ops);
6123 InGlue = SDValue(CNode, 1);
6124 // Update the chain.
6125 ReplaceUses(F: N1.getValue(R: 1), T: SDValue(CNode, 0));
6126 // Record the mem-refs
6127 CurDAG->setNodeMemRefs(N: CNode, NewMemRefs: {cast<LoadSDNode>(Val&: N1)->getMemOperand()});
6128 } else {
6129 InGlue =
6130 SDValue(CurDAG->getMachineNode(Opcode: ROpc, dl, VT: MVT::Glue, Op1: N1, Op2: InGlue), 0);
6131 }
6132
6133 // Prevent use of AH in a REX instruction by explicitly copying it to
6134 // an ABCD_L register.
6135 //
6136 // The current assumption of the register allocator is that isel
6137 // won't generate explicit references to the GR8_ABCD_H registers. If
6138 // the allocator and/or the backend get enhanced to be more robust in
6139 // that regard, this can be, and should be, removed.
6140 if (HiReg == X86::AH && !SDValue(Node, 1).use_empty()) {
6141 SDValue AHCopy = CurDAG->getRegister(Reg: X86::AH, VT: MVT::i8);
6142 unsigned AHExtOpcode =
6143 isSigned ? X86::MOVSX32rr8_NOREX : X86::MOVZX32rr8_NOREX;
6144
6145 SDNode *RNode = CurDAG->getMachineNode(Opcode: AHExtOpcode, dl, VT1: MVT::i32,
6146 VT2: MVT::Glue, Op1: AHCopy, Op2: InGlue);
6147 SDValue Result(RNode, 0);
6148 InGlue = SDValue(RNode, 1);
6149
6150 Result =
6151 CurDAG->getTargetExtractSubreg(SRIdx: X86::sub_8bit, DL: dl, VT: MVT::i8, Operand: Result);
6152
6153 ReplaceUses(F: SDValue(Node, 1), T: Result);
6154 LLVM_DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG);
6155 dbgs() << '\n');
6156 }
6157 // Copy the division (low) result, if it is needed.
6158 if (!SDValue(Node, 0).use_empty()) {
6159 SDValue Result = CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl,
6160 Reg: LoReg, VT: NVT, Glue: InGlue);
6161 InGlue = Result.getValue(R: 2);
6162 ReplaceUses(F: SDValue(Node, 0), T: Result);
6163 LLVM_DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG);
6164 dbgs() << '\n');
6165 }
6166 // Copy the remainder (high) result, if it is needed.
6167 if (!SDValue(Node, 1).use_empty()) {
6168 SDValue Result = CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl,
6169 Reg: HiReg, VT: NVT, Glue: InGlue);
6170 InGlue = Result.getValue(R: 2);
6171 ReplaceUses(F: SDValue(Node, 1), T: Result);
6172 LLVM_DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG);
6173 dbgs() << '\n');
6174 }
6175 CurDAG->RemoveDeadNode(N: Node);
6176 return;
6177 }
6178
6179 case X86ISD::FCMP:
6180 case X86ISD::STRICT_FCMP:
6181 case X86ISD::STRICT_FCMPS: {
6182 bool IsStrictCmp = Node->getOpcode() == X86ISD::STRICT_FCMP ||
6183 Node->getOpcode() == X86ISD::STRICT_FCMPS;
6184 SDValue N0 = Node->getOperand(Num: IsStrictCmp ? 1 : 0);
6185 SDValue N1 = Node->getOperand(Num: IsStrictCmp ? 2 : 1);
6186
6187 // Save the original VT of the compare.
6188 MVT CmpVT = N0.getSimpleValueType();
6189
6190 // Floating point needs special handling if we don't have FCOMI.
6191 if (Subtarget->canUseCMOV())
6192 break;
6193
6194 bool IsSignaling = Node->getOpcode() == X86ISD::STRICT_FCMPS;
6195
6196 unsigned Opc;
6197 switch (CmpVT.SimpleTy) {
6198 default: llvm_unreachable("Unexpected type!");
6199 case MVT::f32:
6200 Opc = IsSignaling ? X86::COM_Fpr32 : X86::UCOM_Fpr32;
6201 break;
6202 case MVT::f64:
6203 Opc = IsSignaling ? X86::COM_Fpr64 : X86::UCOM_Fpr64;
6204 break;
6205 case MVT::f80:
6206 Opc = IsSignaling ? X86::COM_Fpr80 : X86::UCOM_Fpr80;
6207 break;
6208 }
6209
6210 SDValue Chain =
6211 IsStrictCmp ? Node->getOperand(Num: 0) : CurDAG->getEntryNode();
6212 SDValue Glue;
6213 if (IsStrictCmp) {
6214 SDVTList VTs = CurDAG->getVTList(VT1: MVT::Other, VT2: MVT::Glue);
6215 Chain = SDValue(CurDAG->getMachineNode(Opcode: Opc, dl, VTs, Ops: {N0, N1, Chain}), 0);
6216 Glue = Chain.getValue(R: 1);
6217 } else {
6218 Glue = SDValue(CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::Glue, Op1: N0, Op2: N1), 0);
6219 }
6220
6221 // Move FPSW to AX.
6222 SDValue FNSTSW =
6223 SDValue(CurDAG->getMachineNode(Opcode: X86::FNSTSW16r, dl, VT: MVT::i16, Op1: Glue), 0);
6224
6225 // Extract upper 8-bits of AX.
6226 SDValue Extract =
6227 CurDAG->getTargetExtractSubreg(SRIdx: X86::sub_8bit_hi, DL: dl, VT: MVT::i8, Operand: FNSTSW);
6228
6229 // Move AH into flags.
6230 // Some 64-bit targets lack SAHF support, but they do support FCOMI.
6231 assert(Subtarget->canUseLAHFSAHF() &&
6232 "Target doesn't support SAHF or FCOMI?");
6233 SDValue AH = CurDAG->getCopyToReg(Chain, dl, Reg: X86::AH, N: Extract, Glue: SDValue());
6234 Chain = AH;
6235 SDValue SAHF = SDValue(
6236 CurDAG->getMachineNode(Opcode: X86::SAHF, dl, VT: MVT::i32, Op1: AH.getValue(R: 1)), 0);
6237
6238 if (IsStrictCmp)
6239 ReplaceUses(F: SDValue(Node, 1), T: Chain);
6240
6241 ReplaceUses(F: SDValue(Node, 0), T: SAHF);
6242 CurDAG->RemoveDeadNode(N: Node);
6243 return;
6244 }
6245
6246 case X86ISD::CMP: {
6247 SDValue N0 = Node->getOperand(Num: 0);
6248 SDValue N1 = Node->getOperand(Num: 1);
6249
6250 // Optimizations for TEST compares.
6251 if (!isNullConstant(V: N1))
6252 break;
6253
6254 // Save the original VT of the compare.
6255 MVT CmpVT = N0.getSimpleValueType();
6256
6257 // If we are comparing (and (shr X, C, Mask) with 0, emit a BEXTR followed
6258 // by a test instruction. The test should be removed later by
6259 // analyzeCompare if we are using only the zero flag.
6260 // TODO: Should we check the users and use the BEXTR flags directly?
6261 if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
6262 if (MachineSDNode *NewNode = matchBEXTRFromAndImm(Node: N0.getNode())) {
6263 unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr
6264 : X86::TEST32rr;
6265 SDValue BEXTR = SDValue(NewNode, 0);
6266 NewNode = CurDAG->getMachineNode(Opcode: TestOpc, dl, VT: MVT::i32, Op1: BEXTR, Op2: BEXTR);
6267 ReplaceUses(F: SDValue(Node, 0), T: SDValue(NewNode, 0));
6268 CurDAG->RemoveDeadNode(N: Node);
6269 return;
6270 }
6271 }
6272
6273 // We can peek through truncates, but we need to be careful below.
6274 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse())
6275 N0 = N0.getOperand(i: 0);
6276
6277 // Look for (X86cmp (and $op, $imm), 0) and see if we can convert it to
6278 // use a smaller encoding.
6279 // Look past the truncate if CMP is the only use of it.
6280 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
6281 N0.getValueType() != MVT::i8) {
6282 auto *MaskC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
6283 if (!MaskC)
6284 break;
6285
6286 // We may have looked through a truncate so mask off any bits that
6287 // shouldn't be part of the compare.
6288 uint64_t Mask = MaskC->getZExtValue();
6289 Mask &= maskTrailingOnes<uint64_t>(N: CmpVT.getScalarSizeInBits());
6290
6291 // Check if we can replace AND+IMM{32,64} with a shift. This is possible
6292 // for masks like 0xFF000000 or 0x00FFFFFF and if we care only about the
6293 // zero flag.
6294 if (CmpVT == MVT::i64 && !isInt<8>(x: Mask) && isShiftedMask_64(Value: Mask) &&
6295 onlyUsesZeroFlag(Flags: SDValue(Node, 0))) {
6296 unsigned ShiftOpcode = ISD::DELETED_NODE;
6297 unsigned ShiftAmt;
6298 unsigned SubRegIdx;
6299 MVT SubRegVT;
6300 unsigned TestOpcode;
6301 unsigned LeadingZeros = llvm::countl_zero(Val: Mask);
6302 unsigned TrailingZeros = llvm::countr_zero(Val: Mask);
6303
6304 // With leading/trailing zeros, the transform is profitable if we can
6305 // eliminate a movabsq or shrink a 32-bit immediate to 8-bit without
6306 // incurring any extra register moves.
6307 bool SavesBytes = !isInt<32>(x: Mask) || N0.getOperand(i: 0).hasOneUse();
6308 if (LeadingZeros == 0 && SavesBytes) {
6309 // If the mask covers the most significant bit, then we can replace
6310 // TEST+AND with a SHR and check eflags.
6311 // This emits a redundant TEST which is subsequently eliminated.
6312 ShiftOpcode = GET_ND_IF_ENABLED(X86::SHR64ri);
6313 ShiftAmt = TrailingZeros;
6314 SubRegIdx = 0;
6315 TestOpcode = X86::TEST64rr;
6316 } else if (TrailingZeros == 0 && SavesBytes) {
6317 // If the mask covers the least significant bit, then we can replace
6318 // TEST+AND with a SHL and check eflags.
6319 // This emits a redundant TEST which is subsequently eliminated.
6320 ShiftOpcode = GET_ND_IF_ENABLED(X86::SHL64ri);
6321 ShiftAmt = LeadingZeros;
6322 SubRegIdx = 0;
6323 TestOpcode = X86::TEST64rr;
6324 } else if (MaskC->hasOneUse() && !isInt<32>(x: Mask)) {
6325 // If the shifted mask extends into the high half and is 8/16/32 bits
6326 // wide, then replace it with a SHR and a TEST8rr/TEST16rr/TEST32rr.
6327 unsigned PopCount = 64 - LeadingZeros - TrailingZeros;
6328 if (PopCount == 8) {
6329 ShiftOpcode = GET_ND_IF_ENABLED(X86::SHR64ri);
6330 ShiftAmt = TrailingZeros;
6331 SubRegIdx = X86::sub_8bit;
6332 SubRegVT = MVT::i8;
6333 TestOpcode = X86::TEST8rr;
6334 } else if (PopCount == 16) {
6335 ShiftOpcode = GET_ND_IF_ENABLED(X86::SHR64ri);
6336 ShiftAmt = TrailingZeros;
6337 SubRegIdx = X86::sub_16bit;
6338 SubRegVT = MVT::i16;
6339 TestOpcode = X86::TEST16rr;
6340 } else if (PopCount == 32) {
6341 ShiftOpcode = GET_ND_IF_ENABLED(X86::SHR64ri);
6342 ShiftAmt = TrailingZeros;
6343 SubRegIdx = X86::sub_32bit;
6344 SubRegVT = MVT::i32;
6345 TestOpcode = X86::TEST32rr;
6346 }
6347 }
6348 if (ShiftOpcode != ISD::DELETED_NODE) {
6349 SDValue ShiftC = CurDAG->getTargetConstant(Val: ShiftAmt, DL: dl, VT: MVT::i64);
6350 SDValue Shift = SDValue(
6351 CurDAG->getMachineNode(Opcode: ShiftOpcode, dl, VT1: MVT::i64, VT2: MVT::i32,
6352 Op1: N0.getOperand(i: 0), Op2: ShiftC),
6353 0);
6354 if (SubRegIdx != 0) {
6355 Shift =
6356 CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx, DL: dl, VT: SubRegVT, Operand: Shift);
6357 }
6358 MachineSDNode *Test =
6359 CurDAG->getMachineNode(Opcode: TestOpcode, dl, VT: MVT::i32, Op1: Shift, Op2: Shift);
6360 ReplaceNode(F: Node, T: Test);
6361 return;
6362 }
6363 }
6364
6365 MVT VT;
6366 int SubRegOp;
6367 unsigned ROpc, MOpc;
6368
6369 // For each of these checks we need to be careful if the sign flag is
6370 // being used. It is only safe to use the sign flag in two conditions,
6371 // either the sign bit in the shrunken mask is zero or the final test
6372 // size is equal to the original compare size.
6373
6374 if (isUInt<8>(x: Mask) &&
6375 (!(Mask & 0x80) || CmpVT == MVT::i8 ||
6376 hasNoSignFlagUses(Flags: SDValue(Node, 0)))) {
6377 // For example, convert "testl %eax, $8" to "testb %al, $8"
6378 VT = MVT::i8;
6379 SubRegOp = X86::sub_8bit;
6380 ROpc = X86::TEST8ri;
6381 MOpc = X86::TEST8mi;
6382 } else if (OptForMinSize && isUInt<16>(x: Mask) &&
6383 (!(Mask & 0x8000) || CmpVT == MVT::i16 ||
6384 hasNoSignFlagUses(Flags: SDValue(Node, 0)))) {
6385 // For example, "testl %eax, $32776" to "testw %ax, $32776".
6386 // NOTE: We only want to form TESTW instructions if optimizing for
6387 // min size. Otherwise we only save one byte and possibly get a length
6388 // changing prefix penalty in the decoders.
6389 VT = MVT::i16;
6390 SubRegOp = X86::sub_16bit;
6391 ROpc = X86::TEST16ri;
6392 MOpc = X86::TEST16mi;
6393 } else if (isUInt<32>(x: Mask) && N0.getValueType() != MVT::i16 &&
6394 ((!(Mask & 0x80000000) &&
6395 // Without minsize 16-bit Cmps can get here so we need to
6396 // be sure we calculate the correct sign flag if needed.
6397 (CmpVT != MVT::i16 || !(Mask & 0x8000))) ||
6398 CmpVT == MVT::i32 ||
6399 hasNoSignFlagUses(Flags: SDValue(Node, 0)))) {
6400 // For example, "testq %rax, $268468232" to "testl %eax, $268468232".
6401 // NOTE: We only want to run that transform if N0 is 32 or 64 bits.
6402 // Otherwize, we find ourselves in a position where we have to do
6403 // promotion. If previous passes did not promote the and, we assume
6404 // they had a good reason not to and do not promote here.
6405 VT = MVT::i32;
6406 SubRegOp = X86::sub_32bit;
6407 ROpc = X86::TEST32ri;
6408 MOpc = X86::TEST32mi;
6409 } else {
6410 // No eligible transformation was found.
6411 break;
6412 }
6413
6414 SDValue Imm = CurDAG->getTargetConstant(Val: Mask, DL: dl, VT);
6415 SDValue Reg = N0.getOperand(i: 0);
6416
6417 // Emit a testl or testw.
6418 MachineSDNode *NewNode;
6419 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
6420 if (tryFoldLoad(Root: Node, P: N0.getNode(), N: Reg, Base&: Tmp0, Scale&: Tmp1, Index&: Tmp2, Disp&: Tmp3, Segment&: Tmp4)) {
6421 if (auto *LoadN = dyn_cast<LoadSDNode>(Val: N0.getOperand(i: 0).getNode())) {
6422 if (!LoadN->isSimple()) {
6423 unsigned NumVolBits = LoadN->getValueType(ResNo: 0).getSizeInBits();
6424 if ((MOpc == X86::TEST8mi && NumVolBits != 8) ||
6425 (MOpc == X86::TEST16mi && NumVolBits != 16) ||
6426 (MOpc == X86::TEST32mi && NumVolBits != 32))
6427 break;
6428 }
6429 }
6430 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
6431 Reg.getOperand(i: 0) };
6432 NewNode = CurDAG->getMachineNode(Opcode: MOpc, dl, VT1: MVT::i32, VT2: MVT::Other, Ops);
6433 // Update the chain.
6434 ReplaceUses(F: Reg.getValue(R: 1), T: SDValue(NewNode, 1));
6435 // Record the mem-refs
6436 CurDAG->setNodeMemRefs(N: NewNode,
6437 NewMemRefs: {cast<LoadSDNode>(Val&: Reg)->getMemOperand()});
6438 } else {
6439 // Extract the subregister if necessary.
6440 if (N0.getValueType() != VT)
6441 Reg = CurDAG->getTargetExtractSubreg(SRIdx: SubRegOp, DL: dl, VT, Operand: Reg);
6442
6443 NewNode = CurDAG->getMachineNode(Opcode: ROpc, dl, VT: MVT::i32, Op1: Reg, Op2: Imm);
6444 }
6445 // Replace CMP with TEST.
6446 ReplaceNode(F: Node, T: NewNode);
6447 return;
6448 }
6449 break;
6450 }
6451 case X86ISD::PCMPISTR: {
6452 if (!Subtarget->hasSSE42())
6453 break;
6454
6455 bool NeedIndex = !SDValue(Node, 0).use_empty();
6456 bool NeedMask = !SDValue(Node, 1).use_empty();
6457 // We can't fold a load if we are going to make two instructions.
6458 bool MayFoldLoad = !NeedIndex || !NeedMask;
6459
6460 MachineSDNode *CNode;
6461 if (NeedMask) {
6462 unsigned ROpc =
6463 Subtarget->hasAVX() ? X86::VPCMPISTRMrri : X86::PCMPISTRMrri;
6464 unsigned MOpc =
6465 Subtarget->hasAVX() ? X86::VPCMPISTRMrmi : X86::PCMPISTRMrmi;
6466 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, VT: MVT::v16i8, Node);
6467 ReplaceUses(F: SDValue(Node, 1), T: SDValue(CNode, 0));
6468 }
6469 if (NeedIndex || !NeedMask) {
6470 unsigned ROpc =
6471 Subtarget->hasAVX() ? X86::VPCMPISTRIrri : X86::PCMPISTRIrri;
6472 unsigned MOpc =
6473 Subtarget->hasAVX() ? X86::VPCMPISTRIrmi : X86::PCMPISTRIrmi;
6474 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, VT: MVT::i32, Node);
6475 ReplaceUses(F: SDValue(Node, 0), T: SDValue(CNode, 0));
6476 }
6477
6478 // Connect the flag usage to the last instruction created.
6479 ReplaceUses(F: SDValue(Node, 2), T: SDValue(CNode, 1));
6480 CurDAG->RemoveDeadNode(N: Node);
6481 return;
6482 }
6483 case X86ISD::PCMPESTR: {
6484 if (!Subtarget->hasSSE42())
6485 break;
6486
6487 // Copy the two implicit register inputs.
6488 SDValue InGlue = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl, Reg: X86::EAX,
6489 N: Node->getOperand(Num: 1),
6490 Glue: SDValue()).getValue(R: 1);
6491 InGlue = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl, Reg: X86::EDX,
6492 N: Node->getOperand(Num: 3), Glue: InGlue).getValue(R: 1);
6493
6494 bool NeedIndex = !SDValue(Node, 0).use_empty();
6495 bool NeedMask = !SDValue(Node, 1).use_empty();
6496 // We can't fold a load if we are going to make two instructions.
6497 bool MayFoldLoad = !NeedIndex || !NeedMask;
6498
6499 MachineSDNode *CNode;
6500 if (NeedMask) {
6501 unsigned ROpc =
6502 Subtarget->hasAVX() ? X86::VPCMPESTRMrri : X86::PCMPESTRMrri;
6503 unsigned MOpc =
6504 Subtarget->hasAVX() ? X86::VPCMPESTRMrmi : X86::PCMPESTRMrmi;
6505 CNode =
6506 emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, VT: MVT::v16i8, Node, InGlue);
6507 ReplaceUses(F: SDValue(Node, 1), T: SDValue(CNode, 0));
6508 }
6509 if (NeedIndex || !NeedMask) {
6510 unsigned ROpc =
6511 Subtarget->hasAVX() ? X86::VPCMPESTRIrri : X86::PCMPESTRIrri;
6512 unsigned MOpc =
6513 Subtarget->hasAVX() ? X86::VPCMPESTRIrmi : X86::PCMPESTRIrmi;
6514 CNode = emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, VT: MVT::i32, Node, InGlue);
6515 ReplaceUses(F: SDValue(Node, 0), T: SDValue(CNode, 0));
6516 }
6517 // Connect the flag usage to the last instruction created.
6518 ReplaceUses(F: SDValue(Node, 2), T: SDValue(CNode, 1));
6519 CurDAG->RemoveDeadNode(N: Node);
6520 return;
6521 }
6522
6523 case ISD::SETCC: {
6524 if (NVT.isVector() && tryVPTESTM(Root: Node, Setcc: SDValue(Node, 0), InMask: SDValue()))
6525 return;
6526
6527 break;
6528 }
6529
6530 case ISD::STORE:
6531 if (foldLoadStoreIntoMemOperand(Node))
6532 return;
6533 break;
6534
6535 case X86ISD::SETCC_CARRY: {
6536 MVT VT = Node->getSimpleValueType(ResNo: 0);
6537 SDValue Result;
6538 if (Subtarget->hasSBBDepBreaking()) {
6539 // We have to do this manually because tblgen will put the eflags copy in
6540 // the wrong place if we use an extract_subreg in the pattern.
6541 // Copy flags to the EFLAGS register and glue it to next node.
6542 SDValue EFLAGS =
6543 CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl, Reg: X86::EFLAGS,
6544 N: Node->getOperand(Num: 1), Glue: SDValue());
6545
6546 // Create a 64-bit instruction if the result is 64-bits otherwise use the
6547 // 32-bit version.
6548 unsigned Opc = VT == MVT::i64 ? X86::SETB_C64r : X86::SETB_C32r;
6549 MVT SetVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
6550 Result = SDValue(
6551 CurDAG->getMachineNode(Opcode: Opc, dl, VT: SetVT, Op1: EFLAGS, Op2: EFLAGS.getValue(R: 1)),
6552 0);
6553 } else {
6554 // The target does not recognize sbb with the same reg operand as a
6555 // no-source idiom, so we explicitly zero the input values.
6556 Result = getSBBZero(N: Node);
6557 }
6558
6559 // For less than 32-bits we need to extract from the 32-bit node.
6560 if (VT == MVT::i8 || VT == MVT::i16) {
6561 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6562 Result = CurDAG->getTargetExtractSubreg(SRIdx: SubIndex, DL: dl, VT, Operand: Result);
6563 }
6564
6565 ReplaceUses(F: SDValue(Node, 0), T: Result);
6566 CurDAG->RemoveDeadNode(N: Node);
6567 return;
6568 }
6569 case X86ISD::SBB: {
6570 if (isNullConstant(V: Node->getOperand(Num: 0)) &&
6571 isNullConstant(V: Node->getOperand(Num: 1))) {
6572 SDValue Result = getSBBZero(N: Node);
6573
6574 // Replace the flag use.
6575 ReplaceUses(F: SDValue(Node, 1), T: Result.getValue(R: 1));
6576
6577 // Replace the result use.
6578 if (!SDValue(Node, 0).use_empty()) {
6579 // For less than 32-bits we need to extract from the 32-bit node.
6580 MVT VT = Node->getSimpleValueType(ResNo: 0);
6581 if (VT == MVT::i8 || VT == MVT::i16) {
6582 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6583 Result = CurDAG->getTargetExtractSubreg(SRIdx: SubIndex, DL: dl, VT, Operand: Result);
6584 }
6585 ReplaceUses(F: SDValue(Node, 0), T: Result);
6586 }
6587
6588 CurDAG->RemoveDeadNode(N: Node);
6589 return;
6590 }
6591 break;
6592 }
6593 case X86ISD::MGATHER: {
6594 auto *Mgt = cast<X86MaskedGatherSDNode>(Val: Node);
6595 SDValue IndexOp = Mgt->getIndex();
6596 SDValue Mask = Mgt->getMask();
6597 MVT IndexVT = IndexOp.getSimpleValueType();
6598 MVT ValueVT = Node->getSimpleValueType(ResNo: 0);
6599 MVT MaskVT = Mask.getSimpleValueType();
6600
6601 // This is just to prevent crashes if the nodes are malformed somehow. We're
6602 // otherwise only doing loose type checking in here based on type what
6603 // a type constraint would say just like table based isel.
6604 if (!ValueVT.isVector() || !MaskVT.isVector())
6605 break;
6606
6607 unsigned NumElts = ValueVT.getVectorNumElements();
6608 MVT ValueSVT = ValueVT.getVectorElementType();
6609
6610 bool IsFP = ValueSVT.isFloatingPoint();
6611 unsigned EltSize = ValueSVT.getSizeInBits();
6612
6613 unsigned Opc = 0;
6614 bool AVX512Gather = MaskVT.getVectorElementType() == MVT::i1;
6615 if (AVX512Gather) {
6616 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6617 Opc = IsFP ? X86::VGATHERDPSZ128rm : X86::VPGATHERDDZ128rm;
6618 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6619 Opc = IsFP ? X86::VGATHERDPSZ256rm : X86::VPGATHERDDZ256rm;
6620 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6621 Opc = IsFP ? X86::VGATHERDPSZrm : X86::VPGATHERDDZrm;
6622 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6623 Opc = IsFP ? X86::VGATHERDPDZ128rm : X86::VPGATHERDQZ128rm;
6624 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6625 Opc = IsFP ? X86::VGATHERDPDZ256rm : X86::VPGATHERDQZ256rm;
6626 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6627 Opc = IsFP ? X86::VGATHERDPDZrm : X86::VPGATHERDQZrm;
6628 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6629 Opc = IsFP ? X86::VGATHERQPSZ128rm : X86::VPGATHERQDZ128rm;
6630 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6631 Opc = IsFP ? X86::VGATHERQPSZ256rm : X86::VPGATHERQDZ256rm;
6632 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6633 Opc = IsFP ? X86::VGATHERQPSZrm : X86::VPGATHERQDZrm;
6634 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6635 Opc = IsFP ? X86::VGATHERQPDZ128rm : X86::VPGATHERQQZ128rm;
6636 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6637 Opc = IsFP ? X86::VGATHERQPDZ256rm : X86::VPGATHERQQZ256rm;
6638 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6639 Opc = IsFP ? X86::VGATHERQPDZrm : X86::VPGATHERQQZrm;
6640 } else {
6641 assert(EVT(MaskVT) == EVT(ValueVT).changeVectorElementTypeToInteger() &&
6642 "Unexpected mask VT!");
6643 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6644 Opc = IsFP ? X86::VGATHERDPSrm : X86::VPGATHERDDrm;
6645 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6646 Opc = IsFP ? X86::VGATHERDPSYrm : X86::VPGATHERDDYrm;
6647 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6648 Opc = IsFP ? X86::VGATHERDPDrm : X86::VPGATHERDQrm;
6649 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6650 Opc = IsFP ? X86::VGATHERDPDYrm : X86::VPGATHERDQYrm;
6651 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6652 Opc = IsFP ? X86::VGATHERQPSrm : X86::VPGATHERQDrm;
6653 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6654 Opc = IsFP ? X86::VGATHERQPSYrm : X86::VPGATHERQDYrm;
6655 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6656 Opc = IsFP ? X86::VGATHERQPDrm : X86::VPGATHERQQrm;
6657 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6658 Opc = IsFP ? X86::VGATHERQPDYrm : X86::VPGATHERQQYrm;
6659 }
6660
6661 if (!Opc)
6662 break;
6663
6664 SDValue Base, Scale, Index, Disp, Segment;
6665 if (!selectVectorAddr(Parent: Mgt, BasePtr: Mgt->getBasePtr(), IndexOp, ScaleOp: Mgt->getScale(),
6666 Base, Scale, Index, Disp, Segment))
6667 break;
6668
6669 SDValue PassThru = Mgt->getPassThru();
6670 SDValue Chain = Mgt->getChain();
6671 // Gather instructions have a mask output not in the ISD node.
6672 SDVTList VTs = CurDAG->getVTList(VT1: ValueVT, VT2: MaskVT, VT3: MVT::Other);
6673
6674 MachineSDNode *NewNode;
6675 if (AVX512Gather) {
6676 SDValue Ops[] = {PassThru, Mask, Base, Scale,
6677 Index, Disp, Segment, Chain};
6678 NewNode = CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(dl), VTs, Ops);
6679 } else {
6680 SDValue Ops[] = {PassThru, Base, Scale, Index,
6681 Disp, Segment, Mask, Chain};
6682 NewNode = CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(dl), VTs, Ops);
6683 }
6684 CurDAG->setNodeMemRefs(N: NewNode, NewMemRefs: {Mgt->getMemOperand()});
6685 ReplaceUses(F: SDValue(Node, 0), T: SDValue(NewNode, 0));
6686 ReplaceUses(F: SDValue(Node, 1), T: SDValue(NewNode, 2));
6687 CurDAG->RemoveDeadNode(N: Node);
6688 return;
6689 }
6690 case X86ISD::MSCATTER: {
6691 auto *Sc = cast<X86MaskedScatterSDNode>(Val: Node);
6692 SDValue Value = Sc->getValue();
6693 SDValue IndexOp = Sc->getIndex();
6694 MVT IndexVT = IndexOp.getSimpleValueType();
6695 MVT ValueVT = Value.getSimpleValueType();
6696
6697 // This is just to prevent crashes if the nodes are malformed somehow. We're
6698 // otherwise only doing loose type checking in here based on type what
6699 // a type constraint would say just like table based isel.
6700 if (!ValueVT.isVector())
6701 break;
6702
6703 unsigned NumElts = ValueVT.getVectorNumElements();
6704 MVT ValueSVT = ValueVT.getVectorElementType();
6705
6706 bool IsFP = ValueSVT.isFloatingPoint();
6707 unsigned EltSize = ValueSVT.getSizeInBits();
6708
6709 unsigned Opc;
6710 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6711 Opc = IsFP ? X86::VSCATTERDPSZ128mr : X86::VPSCATTERDDZ128mr;
6712 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6713 Opc = IsFP ? X86::VSCATTERDPSZ256mr : X86::VPSCATTERDDZ256mr;
6714 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6715 Opc = IsFP ? X86::VSCATTERDPSZmr : X86::VPSCATTERDDZmr;
6716 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6717 Opc = IsFP ? X86::VSCATTERDPDZ128mr : X86::VPSCATTERDQZ128mr;
6718 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6719 Opc = IsFP ? X86::VSCATTERDPDZ256mr : X86::VPSCATTERDQZ256mr;
6720 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6721 Opc = IsFP ? X86::VSCATTERDPDZmr : X86::VPSCATTERDQZmr;
6722 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6723 Opc = IsFP ? X86::VSCATTERQPSZ128mr : X86::VPSCATTERQDZ128mr;
6724 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6725 Opc = IsFP ? X86::VSCATTERQPSZ256mr : X86::VPSCATTERQDZ256mr;
6726 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6727 Opc = IsFP ? X86::VSCATTERQPSZmr : X86::VPSCATTERQDZmr;
6728 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6729 Opc = IsFP ? X86::VSCATTERQPDZ128mr : X86::VPSCATTERQQZ128mr;
6730 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6731 Opc = IsFP ? X86::VSCATTERQPDZ256mr : X86::VPSCATTERQQZ256mr;
6732 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6733 Opc = IsFP ? X86::VSCATTERQPDZmr : X86::VPSCATTERQQZmr;
6734 else
6735 break;
6736
6737 SDValue Base, Scale, Index, Disp, Segment;
6738 if (!selectVectorAddr(Parent: Sc, BasePtr: Sc->getBasePtr(), IndexOp, ScaleOp: Sc->getScale(),
6739 Base, Scale, Index, Disp, Segment))
6740 break;
6741
6742 SDValue Mask = Sc->getMask();
6743 SDValue Chain = Sc->getChain();
6744 // Scatter instructions have a mask output not in the ISD node.
6745 SDVTList VTs = CurDAG->getVTList(VT1: Mask.getValueType(), VT2: MVT::Other);
6746 SDValue Ops[] = {Base, Scale, Index, Disp, Segment, Mask, Value, Chain};
6747
6748 MachineSDNode *NewNode = CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(dl), VTs, Ops);
6749 CurDAG->setNodeMemRefs(N: NewNode, NewMemRefs: {Sc->getMemOperand()});
6750 ReplaceUses(F: SDValue(Node, 0), T: SDValue(NewNode, 1));
6751 CurDAG->RemoveDeadNode(N: Node);
6752 return;
6753 }
6754 case ISD::PREALLOCATED_SETUP: {
6755 auto *MFI = CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
6756 auto CallId = MFI->getPreallocatedIdForCallSite(
6757 CS: cast<SrcValueSDNode>(Val: Node->getOperand(Num: 1))->getValue());
6758 SDValue Chain = Node->getOperand(Num: 0);
6759 SDValue CallIdValue = CurDAG->getTargetConstant(Val: CallId, DL: dl, VT: MVT::i32);
6760 MachineSDNode *New = CurDAG->getMachineNode(
6761 Opcode: TargetOpcode::PREALLOCATED_SETUP, dl, VT: MVT::Other, Op1: CallIdValue, Op2: Chain);
6762 ReplaceUses(F: SDValue(Node, 0), T: SDValue(New, 0)); // Chain
6763 CurDAG->RemoveDeadNode(N: Node);
6764 return;
6765 }
6766 case ISD::PREALLOCATED_ARG: {
6767 auto *MFI = CurDAG->getMachineFunction().getInfo<X86MachineFunctionInfo>();
6768 auto CallId = MFI->getPreallocatedIdForCallSite(
6769 CS: cast<SrcValueSDNode>(Val: Node->getOperand(Num: 1))->getValue());
6770 SDValue Chain = Node->getOperand(Num: 0);
6771 SDValue CallIdValue = CurDAG->getTargetConstant(Val: CallId, DL: dl, VT: MVT::i32);
6772 SDValue ArgIndex = Node->getOperand(Num: 2);
6773 SDValue Ops[3];
6774 Ops[0] = CallIdValue;
6775 Ops[1] = ArgIndex;
6776 Ops[2] = Chain;
6777 MachineSDNode *New = CurDAG->getMachineNode(
6778 Opcode: TargetOpcode::PREALLOCATED_ARG, dl,
6779 VTs: CurDAG->getVTList(VT1: TLI->getPointerTy(DL: CurDAG->getDataLayout()),
6780 VT2: MVT::Other),
6781 Ops);
6782 ReplaceUses(F: SDValue(Node, 0), T: SDValue(New, 0)); // Arg pointer
6783 ReplaceUses(F: SDValue(Node, 1), T: SDValue(New, 1)); // Chain
6784 CurDAG->RemoveDeadNode(N: Node);
6785 return;
6786 }
6787 case X86ISD::AESENCWIDE128KL:
6788 case X86ISD::AESDECWIDE128KL:
6789 case X86ISD::AESENCWIDE256KL:
6790 case X86ISD::AESDECWIDE256KL: {
6791 if (!Subtarget->hasWIDEKL())
6792 break;
6793
6794 unsigned Opcode;
6795 switch (Node->getOpcode()) {
6796 default:
6797 llvm_unreachable("Unexpected opcode!");
6798 case X86ISD::AESENCWIDE128KL:
6799 Opcode = X86::AESENCWIDE128KL;
6800 break;
6801 case X86ISD::AESDECWIDE128KL:
6802 Opcode = X86::AESDECWIDE128KL;
6803 break;
6804 case X86ISD::AESENCWIDE256KL:
6805 Opcode = X86::AESENCWIDE256KL;
6806 break;
6807 case X86ISD::AESDECWIDE256KL:
6808 Opcode = X86::AESDECWIDE256KL;
6809 break;
6810 }
6811
6812 SDValue Chain = Node->getOperand(Num: 0);
6813 SDValue Addr = Node->getOperand(Num: 1);
6814
6815 SDValue Base, Scale, Index, Disp, Segment;
6816 if (!selectAddr(Parent: Node, N: Addr, Base, Scale, Index, Disp, Segment))
6817 break;
6818
6819 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::XMM0, N: Node->getOperand(Num: 2),
6820 Glue: SDValue());
6821 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::XMM1, N: Node->getOperand(Num: 3),
6822 Glue: Chain.getValue(R: 1));
6823 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::XMM2, N: Node->getOperand(Num: 4),
6824 Glue: Chain.getValue(R: 1));
6825 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::XMM3, N: Node->getOperand(Num: 5),
6826 Glue: Chain.getValue(R: 1));
6827 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::XMM4, N: Node->getOperand(Num: 6),
6828 Glue: Chain.getValue(R: 1));
6829 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::XMM5, N: Node->getOperand(Num: 7),
6830 Glue: Chain.getValue(R: 1));
6831 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::XMM6, N: Node->getOperand(Num: 8),
6832 Glue: Chain.getValue(R: 1));
6833 Chain = CurDAG->getCopyToReg(Chain, dl, Reg: X86::XMM7, N: Node->getOperand(Num: 9),
6834 Glue: Chain.getValue(R: 1));
6835
6836 MachineSDNode *Res = CurDAG->getMachineNode(
6837 Opcode, dl, VTs: Node->getVTList(),
6838 Ops: {Base, Scale, Index, Disp, Segment, Chain, Chain.getValue(R: 1)});
6839 CurDAG->setNodeMemRefs(N: Res, NewMemRefs: cast<MemSDNode>(Val: Node)->getMemOperand());
6840 ReplaceNode(F: Node, T: Res);
6841 return;
6842 }
6843 case X86ISD::POP_FROM_X87_REG: {
6844 SDValue Chain = Node->getOperand(Num: 0);
6845 Register Reg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1))->getReg();
6846 SDValue Glue;
6847 if (Node->getNumValues() == 3)
6848 Glue = Node->getOperand(Num: 2);
6849 SDValue Copy =
6850 CurDAG->getCopyFromReg(Chain, dl, Reg, VT: Node->getValueType(ResNo: 0), Glue);
6851 ReplaceNode(F: Node, T: Copy.getNode());
6852 return;
6853 }
6854 }
6855
6856 SelectCode(N: Node);
6857}
6858
6859bool X86DAGToDAGISel::SelectInlineAsmMemoryOperand(
6860 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
6861 std::vector<SDValue> &OutOps) {
6862 SDValue Op0, Op1, Op2, Op3, Op4;
6863 switch (ConstraintID) {
6864 default:
6865 llvm_unreachable("Unexpected asm memory constraint");
6866 case InlineAsm::ConstraintCode::o: // offsetable ??
6867 case InlineAsm::ConstraintCode::v: // not offsetable ??
6868 case InlineAsm::ConstraintCode::m: // memory
6869 case InlineAsm::ConstraintCode::X:
6870 case InlineAsm::ConstraintCode::p: // address
6871 if (!selectAddr(Parent: nullptr, N: Op, Base&: Op0, Scale&: Op1, Index&: Op2, Disp&: Op3, Segment&: Op4))
6872 return true;
6873 break;
6874 }
6875
6876 OutOps.push_back(x: Op0);
6877 OutOps.push_back(x: Op1);
6878 OutOps.push_back(x: Op2);
6879 OutOps.push_back(x: Op3);
6880 OutOps.push_back(x: Op4);
6881 return false;
6882}
6883
6884X86ISelDAGToDAGPass::X86ISelDAGToDAGPass(X86TargetMachine &TM)
6885 : SelectionDAGISelPass(
6886 std::make_unique<X86DAGToDAGISel>(args&: TM, args: TM.getOptLevel())) {}
6887
6888/// This pass converts a legalized DAG into a X86-specific DAG,
6889/// ready for instruction scheduling.
6890FunctionPass *llvm::createX86ISelDag(X86TargetMachine &TM,
6891 CodeGenOptLevel OptLevel) {
6892 return new X86DAGToDAGISelLegacy(TM, OptLevel);
6893}
6894