1//===-- X86SelectionDAGInfo.cpp - X86 SelectionDAG Info -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the X86SelectionDAGInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86SelectionDAGInfo.h"
14#include "X86InstrInfo.h"
15#include "X86RegisterInfo.h"
16#include "X86Subtarget.h"
17#include "llvm/CodeGen/MachineFrameInfo.h"
18#include "llvm/CodeGen/SelectionDAG.h"
19#include "llvm/CodeGen/TargetLowering.h"
20
21#define GET_SDNODE_DESC
22#include "X86GenSDNodeInfo.inc"
23
24using namespace llvm;
25
26#define DEBUG_TYPE "x86-selectiondag-info"
27
28X86SelectionDAGInfo::X86SelectionDAGInfo()
29 : SelectionDAGGenTargetInfo(X86GenSDNodeInfo) {}
30
31const char *X86SelectionDAGInfo::getTargetNodeName(unsigned Opcode) const {
32#define NODE_NAME_CASE(NODE) \
33 case X86ISD::NODE: \
34 return "X86ISD::" #NODE;
35
36 // These nodes don't have corresponding entries in *.td files yet.
37 switch (static_cast<X86ISD::NodeType>(Opcode)) {
38 NODE_NAME_CASE(POP_FROM_X87_REG)
39 NODE_NAME_CASE(GlobalBaseReg)
40 NODE_NAME_CASE(LCMPXCHG16_SAVE_RBX_DAG)
41 NODE_NAME_CASE(PCMPESTR)
42 NODE_NAME_CASE(PCMPISTR)
43 NODE_NAME_CASE(MGATHER)
44 NODE_NAME_CASE(MSCATTER)
45 NODE_NAME_CASE(AESENCWIDE128KL)
46 NODE_NAME_CASE(AESDECWIDE128KL)
47 NODE_NAME_CASE(AESENCWIDE256KL)
48 NODE_NAME_CASE(AESDECWIDE256KL)
49 }
50#undef NODE_NAME_CASE
51
52 return SelectionDAGGenTargetInfo::getTargetNodeName(Opcode);
53}
54
55bool X86SelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const {
56 // These nodes don't have corresponding entries in *.td files yet.
57 if (Opcode >= X86ISD::FIRST_MEMORY_OPCODE &&
58 Opcode <= X86ISD::LAST_MEMORY_OPCODE)
59 return true;
60
61 return SelectionDAGGenTargetInfo::isTargetMemoryOpcode(Opcode);
62}
63
64void X86SelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG,
65 const SDNode *N) const {
66 SelectionDAGGenTargetInfo::verifyTargetNode(DAG, N);
67
68 switch (N->getOpcode()) {
69 default:
70 break;
71 case X86ISD::CALL:
72 case X86ISD::TC_RETURN:
73 case X86ISD::TC_RETURN_GLOBALADDR: {
74 // The call target is an integer whose width depends on both the
75 // subtarget and on how the callee is addressed:
76 // * A direct call to a GlobalAddress/ExternalSymbol is i32 on the
77 // x32 ABI (as well as plain 32-bit mode) and i64 under LP64.
78 // * Anything else (register, folded load, or RIP-relative CFGuard call)
79 // uses the register width the subtarget executes in, i.e. i64 whenever
80 // the subtarget runs in 64-bit mode (including x32) and i32 otherwise.
81 const X86Subtarget &Subtarget =
82 DAG.getMachineFunction().getSubtarget<X86Subtarget>();
83 SDValue Target = N->getOperand(Num: 1);
84 bool IsDirect =
85 isa<GlobalAddressSDNode>(Val: Target) || isa<ExternalSymbolSDNode>(Val: Target);
86 bool WantI64 =
87 IsDirect ? Subtarget.isTarget64BitLP64() : Subtarget.is64Bit();
88 EVT ExpectedVT = WantI64 ? MVT::i64 : MVT::i32;
89 EVT VT = Target.getValueType();
90 if (VT != ExpectedVT)
91 report_fatal_error(reason: "invalid node: " + Twine(N->getOperationName(G: &DAG)) +
92 " operand #1 must have type " +
93 ExpectedVT.getEVTString() + ", but has type " +
94 VT.getEVTString());
95 break;
96 }
97 case X86ISD::SETCC_CARRY: {
98 // SETCC_CARRY materializes CF as 0/-1, so the condition must be COND_B.
99 auto *CC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0));
100 if (!CC || CC->getZExtValue() != X86::COND_B)
101 report_fatal_error(reason: "invalid node: " + Twine(N->getOperationName(G: &DAG)) +
102 " operand #0 must be X86::COND_B");
103 break;
104 }
105 }
106}
107
108/// Returns the best type to use with repmovs/repstos depending on alignment.
109static MVT getOptimalRepType(const X86Subtarget &Subtarget, Align Alignment) {
110 uint64_t Align = Alignment.value();
111 assert((Align != 0) && "Align is normalized");
112 assert(isPowerOf2_64(Align) && "Align is a power of 2");
113 switch (Align) {
114 case 1:
115 return MVT::i8;
116 case 2:
117 return MVT::i16;
118 case 4:
119 return MVT::i32;
120 default:
121 return Subtarget.is64Bit() ? MVT::i64 : MVT::i32;
122 }
123}
124
125bool X86SelectionDAGInfo::isBaseRegConflictPossible(
126 SelectionDAG &DAG, ArrayRef<MCPhysReg> ClobberSet) const {
127 // We cannot use TRI->hasBasePointer() until *after* we select all basic
128 // blocks. Legalization may introduce new stack temporaries with large
129 // alignment requirements. Fall back to generic code if there are any
130 // dynamic stack adjustments (hopefully rare) and the base pointer would
131 // conflict if we had to use it.
132 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
133 if (!MFI.hasVarSizedObjects() && !MFI.hasOpaqueSPAdjustment())
134 return false;
135
136 const X86RegisterInfo *TRI = static_cast<const X86RegisterInfo *>(
137 DAG.getSubtarget().getRegisterInfo());
138 return llvm::is_contained(Range&: ClobberSet, Element: TRI->getBaseRegister());
139}
140
141/// Emit a single REP STOSB instruction for a particular constant size.
142static SDValue emitRepstos(const X86Subtarget &Subtarget, SelectionDAG &DAG,
143 const SDLoc &dl, SDValue Chain, SDValue Dst,
144 SDValue Val, SDValue Size, MVT AVT) {
145 const bool Use64BitRegs = Subtarget.isTarget64BitLP64();
146 unsigned AX = X86::AL;
147 switch (AVT.getSizeInBits()) {
148 case 8:
149 AX = X86::AL;
150 break;
151 case 16:
152 AX = X86::AX;
153 break;
154 case 32:
155 AX = X86::EAX;
156 break;
157 default:
158 AX = X86::RAX;
159 break;
160 }
161
162 const unsigned CX = Use64BitRegs ? X86::RCX : X86::ECX;
163 const unsigned DI = Use64BitRegs ? X86::RDI : X86::EDI;
164
165 SDValue InGlue;
166 Chain = DAG.getCopyToReg(Chain, dl, Reg: AX, N: Val, Glue: InGlue);
167 InGlue = Chain.getValue(R: 1);
168 Chain = DAG.getCopyToReg(Chain, dl, Reg: CX, N: Size, Glue: InGlue);
169 InGlue = Chain.getValue(R: 1);
170 Chain = DAG.getCopyToReg(Chain, dl, Reg: DI, N: Dst, Glue: InGlue);
171 InGlue = Chain.getValue(R: 1);
172
173 SDVTList Tys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
174 SDValue Ops[] = {Chain, DAG.getValueType(AVT), InGlue};
175 return DAG.getNode(Opcode: X86ISD::REP_STOS, DL: dl, VTList: Tys, Ops);
176}
177
178/// Emit a single REP STOSB instruction for a particular constant size.
179static SDValue emitRepstosB(const X86Subtarget &Subtarget, SelectionDAG &DAG,
180 const SDLoc &dl, SDValue Chain, SDValue Dst,
181 SDValue Val, uint64_t Size) {
182 return emitRepstos(Subtarget, DAG, dl, Chain, Dst, Val,
183 Size: DAG.getIntPtrConstant(Val: Size, DL: dl), AVT: MVT::i8);
184}
185
186/// Returns a REP STOS instruction, possibly with a few load/stores to implement
187/// a constant size memory set. In some cases where we know REP MOVS is
188/// inefficient we return an empty SDValue so the calling code can either
189/// generate a store sequence or call the runtime memset function.
190static SDValue emitConstantSizeRepstos(SelectionDAG &DAG,
191 const X86Subtarget &Subtarget,
192 const SDLoc &dl, SDValue Chain,
193 SDValue Dst, SDValue Val, uint64_t Size,
194 EVT SizeVT, Align Alignment,
195 bool isVolatile, bool AlwaysInline,
196 MachinePointerInfo DstPtrInfo) {
197 /// In case we optimize for size, we use repstosb even if it's less efficient
198 /// so we can save the loads/stores of the leftover.
199 if (DAG.getMachineFunction().getFunction().hasMinSize()) {
200 if (auto *ValC = dyn_cast<ConstantSDNode>(Val)) {
201 // Special case 0 because otherwise we get large literals,
202 // which causes larger encoding.
203 if ((Size & 31) == 0 && (ValC->getZExtValue() & 255) == 0) {
204 MVT BlockType = MVT::i32;
205 const uint64_t BlockBits = BlockType.getSizeInBits();
206 const uint64_t BlockBytes = BlockBits / 8;
207 const uint64_t BlockCount = Size / BlockBytes;
208
209 Val = DAG.getConstant(Val: 0, DL: dl, VT: BlockType);
210 // repstosd is same size as repstosb
211 return emitRepstos(Subtarget, DAG, dl, Chain, Dst, Val,
212 Size: DAG.getIntPtrConstant(Val: BlockCount, DL: dl), AVT: BlockType);
213 }
214 }
215 return emitRepstosB(Subtarget, DAG, dl, Chain, Dst, Val, Size);
216 }
217
218 if (Size > Subtarget.getMaxInlineSizeThreshold())
219 return SDValue();
220
221 // If not DWORD aligned or size is more than the threshold, call the library.
222 // The libc version is likely to be faster for these cases. It can use the
223 // address value and run time information about the CPU.
224 if (Alignment < Align(4))
225 return SDValue();
226
227 MVT BlockType = MVT::i8;
228 uint64_t BlockCount = Size;
229 uint64_t BytesLeft = 0;
230
231 SDValue OriginalVal = Val;
232 if (auto *ValC = dyn_cast<ConstantSDNode>(Val)) {
233 BlockType = getOptimalRepType(Subtarget, Alignment);
234 uint64_t Value = ValC->getZExtValue() & 255;
235 const uint64_t BlockBits = BlockType.getSizeInBits();
236
237 if (BlockBits >= 16)
238 Value = (Value << 8) | Value;
239
240 if (BlockBits >= 32)
241 Value = (Value << 16) | Value;
242
243 if (BlockBits >= 64)
244 Value = (Value << 32) | Value;
245
246 const uint64_t BlockBytes = BlockBits / 8;
247 BlockCount = Size / BlockBytes;
248 BytesLeft = Size % BlockBytes;
249 Val = DAG.getConstant(Val: Value, DL: dl, VT: BlockType);
250 }
251
252 SDValue RepStos =
253 emitRepstos(Subtarget, DAG, dl, Chain, Dst, Val,
254 Size: DAG.getIntPtrConstant(Val: BlockCount, DL: dl), AVT: BlockType);
255 /// RepStos can process the whole length.
256 if (BytesLeft == 0)
257 return RepStos;
258
259 // Handle the last 1 - 7 bytes.
260 SmallVector<SDValue, 4> Results;
261 Results.push_back(Elt: RepStos);
262 unsigned Offset = Size - BytesLeft;
263 EVT AddrVT = Dst.getValueType();
264
265 Results.push_back(
266 Elt: DAG.getMemset(Chain, dl,
267 Dst: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AddrVT, N1: Dst,
268 N2: DAG.getConstant(Val: Offset, DL: dl, VT: AddrVT)),
269 Src: OriginalVal, Size: DAG.getConstant(Val: BytesLeft, DL: dl, VT: SizeVT),
270 Alignment, isVol: isVolatile, AlwaysInline,
271 /* CI */ nullptr, DstPtrInfo: DstPtrInfo.getWithOffset(O: Offset)));
272
273 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Results);
274}
275
276SDValue X86SelectionDAGInfo::EmitTargetCodeForMemset(
277 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Val,
278 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
279 bool &IsTailCall, MachinePointerInfo DstPtrInfo) const {
280 IsTailCall = false;
281 const X86Subtarget &Subtarget =
282 DAG.getMachineFunction().getSubtarget<X86Subtarget>();
283
284 // If to a segment-relative address space, use the default lowering.
285 if (DstPtrInfo.getAddrSpace() >= 256)
286 return SDValue();
287
288 // REP STOS uses EDI on x86-32. Fall back if the user reserved EDI, so the
289 // generic expander can avoid emitting REP STOS.
290 if (!Subtarget.is64Bit() && Subtarget.isRegisterReservedByUser(i: X86::EDI))
291 return SDValue();
292
293 // If the base register might conflict with our physical registers, bail out.
294 const MCPhysReg ClobberSet[] = {X86::RCX, X86::RAX, X86::RDI,
295 X86::ECX, X86::EAX, X86::EDI};
296 if (isBaseRegConflictPossible(DAG, ClobberSet))
297 return SDValue();
298
299 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
300 if (!ConstantSize)
301 return SDValue();
302
303 return emitConstantSizeRepstos(
304 DAG, Subtarget, dl, Chain, Dst, Val, Size: ConstantSize->getZExtValue(),
305 SizeVT: Size.getValueType(), Alignment, isVolatile, AlwaysInline, DstPtrInfo);
306}
307
308/// Emit a single REP MOVS{B,W,D,Q} instruction.
309static SDValue emitRepmovs(const X86Subtarget &Subtarget, SelectionDAG &DAG,
310 const SDLoc &dl, SDValue Chain, SDValue Dst,
311 SDValue Src, SDValue Size, MVT AVT) {
312 const bool Use64BitRegs = Subtarget.isTarget64BitLP64();
313 const unsigned CX = Use64BitRegs ? X86::RCX : X86::ECX;
314 const unsigned DI = Use64BitRegs ? X86::RDI : X86::EDI;
315 const unsigned SI = Use64BitRegs ? X86::RSI : X86::ESI;
316
317 SDValue InGlue;
318 Chain = DAG.getCopyToReg(Chain, dl, Reg: CX, N: Size, Glue: InGlue);
319 InGlue = Chain.getValue(R: 1);
320 Chain = DAG.getCopyToReg(Chain, dl, Reg: DI, N: Dst, Glue: InGlue);
321 InGlue = Chain.getValue(R: 1);
322 Chain = DAG.getCopyToReg(Chain, dl, Reg: SI, N: Src, Glue: InGlue);
323 InGlue = Chain.getValue(R: 1);
324
325 SDVTList Tys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
326 SDValue Ops[] = {Chain, DAG.getValueType(AVT), InGlue};
327 return DAG.getNode(Opcode: X86ISD::REP_MOVS, DL: dl, VTList: Tys, Ops);
328}
329
330/// Emit a single REP MOVSB instruction for a particular constant size.
331static SDValue emitRepmovsB(const X86Subtarget &Subtarget, SelectionDAG &DAG,
332 const SDLoc &dl, SDValue Chain, SDValue Dst,
333 SDValue Src, uint64_t Size) {
334 return emitRepmovs(Subtarget, DAG, dl, Chain, Dst, Src,
335 Size: DAG.getIntPtrConstant(Val: Size, DL: dl), AVT: MVT::i8);
336}
337
338/// Returns a REP MOVS instruction, possibly with a few load/stores to implement
339/// a constant size memory copy. In some cases where we know REP MOVS is
340/// inefficient we return an empty SDValue so the calling code can either
341/// generate a load/store sequence or call the runtime memcpy function.
342static SDValue emitConstantSizeRepmov(
343 SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &dl,
344 SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, EVT SizeVT,
345 Align Alignment, bool isVolatile, bool AlwaysInline,
346 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) {
347 /// In case we optimize for size, we use repmovsb even if it's less efficient
348 /// so we can save the loads/stores of the leftover.
349 if (DAG.getMachineFunction().getFunction().hasMinSize())
350 return emitRepmovsB(Subtarget, DAG, dl, Chain, Dst, Src, Size);
351
352 /// TODO: Revisit next line: big copy with ERMSB on march >= haswell are very
353 /// efficient.
354 if (!AlwaysInline && Size > Subtarget.getMaxInlineSizeThreshold())
355 return SDValue();
356
357 /// If we have enhanced repmovs we use it.
358 if (Subtarget.hasERMSB())
359 return emitRepmovsB(Subtarget, DAG, dl, Chain, Dst, Src, Size);
360
361 assert(!Subtarget.hasERMSB() && "No efficient RepMovs");
362 /// We assume runtime memcpy will do a better job for unaligned copies when
363 /// ERMS is not present.
364 if (!AlwaysInline && (Alignment < Align(4)))
365 return SDValue();
366
367 const MVT BlockType = getOptimalRepType(Subtarget, Alignment);
368 const uint64_t BlockBytes = BlockType.getSizeInBits() / 8;
369 const uint64_t BlockCount = Size / BlockBytes;
370 const uint64_t BytesLeft = Size % BlockBytes;
371 SDValue RepMovs =
372 emitRepmovs(Subtarget, DAG, dl, Chain, Dst, Src,
373 Size: DAG.getIntPtrConstant(Val: BlockCount, DL: dl), AVT: BlockType);
374
375 /// RepMov can process the whole length.
376 if (BytesLeft == 0)
377 return RepMovs;
378
379 assert(BytesLeft && "We have leftover at this point");
380
381 // Handle the last 1 - 7 bytes.
382 SmallVector<SDValue, 4> Results;
383 Results.push_back(Elt: RepMovs);
384 unsigned Offset = Size - BytesLeft;
385 EVT DstVT = Dst.getValueType();
386 EVT SrcVT = Src.getValueType();
387 Results.push_back(Elt: DAG.getMemcpy(
388 Chain, dl,
389 Dst: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: DstVT, N1: Dst, N2: DAG.getConstant(Val: Offset, DL: dl, VT: DstVT)),
390 Src: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: SrcVT, N1: Src, N2: DAG.getConstant(Val: Offset, DL: dl, VT: SrcVT)),
391 Size: DAG.getConstant(Val: BytesLeft, DL: dl, VT: SizeVT), DstAlign: Alignment, SrcAlign: Alignment, isVol: isVolatile,
392 /*AlwaysInline*/ true, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
393 DstPtrInfo: DstPtrInfo.getWithOffset(O: Offset), SrcPtrInfo: SrcPtrInfo.getWithOffset(O: Offset)));
394 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Results);
395}
396
397SDValue X86SelectionDAGInfo::EmitTargetCodeForMemcpy(
398 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
399 SDValue Size, Align DstAlign, Align SrcAlign, bool isVolatile,
400 bool AlwaysInline, bool &IsTailCall, MachinePointerInfo DstPtrInfo,
401 MachinePointerInfo SrcPtrInfo) const {
402 IsTailCall = false;
403 const X86Subtarget &Subtarget =
404 DAG.getMachineFunction().getSubtarget<X86Subtarget>();
405
406 // If to a segment-relative address space, use the default lowering.
407 if (DstPtrInfo.getAddrSpace() >= 256 || SrcPtrInfo.getAddrSpace() >= 256)
408 return SDValue();
409
410 // REP MOVS uses EDI/ESI on x86-32. fall back only when EDI is
411 // reserved so the generic expander can avoid emitting REP MOVS.
412 if (!Subtarget.is64Bit() && Subtarget.isRegisterReservedByUser(i: X86::EDI))
413 return SDValue();
414
415 // If the base registers conflict with our physical registers, use the default
416 // lowering.
417 const MCPhysReg ClobberSet[] = {X86::RCX, X86::RSI, X86::RDI,
418 X86::ECX, X86::ESI, X86::EDI};
419 if (isBaseRegConflictPossible(DAG, ClobberSet))
420 return SDValue();
421
422 // If enabled and available, use fast short rep mov.
423 if (Subtarget.getCLOpts().use_fsrm_for_memcpy && Subtarget.hasFSRM())
424 return emitRepmovs(Subtarget, DAG, dl, Chain, Dst, Src, Size, AVT: MVT::i8);
425
426 // Handle constant sizes
427 if (ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size)) {
428 Align Alignment = std::min(a: DstAlign, b: SrcAlign);
429 return emitConstantSizeRepmov(DAG, Subtarget, dl, Chain, Dst, Src,
430 Size: ConstantSize->getZExtValue(),
431 SizeVT: Size.getValueType(), Alignment, isVolatile,
432 AlwaysInline, DstPtrInfo, SrcPtrInfo);
433 }
434
435 return SDValue();
436}
437