1//===-- ARMSelectionDAGInfo.cpp - ARM SelectionDAG Info -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the ARMSelectionDAGInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ARMSelectionDAGInfo.h"
14#include "ARMTargetTransformInfo.h"
15#include "llvm/CodeGen/SelectionDAG.h"
16#include "llvm/Support/CommandLine.h"
17
18#define GET_SDNODE_DESC
19#include "ARMGenSDNodeInfo.inc"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "arm-selectiondag-info"
24
25static cl::opt<TPLoop::MemTransfer> EnableMemtransferTPLoop(
26 "arm-memtransfer-tploop", cl::Hidden,
27 cl::desc("Control conversion of memcpy to "
28 "Tail predicated loops (WLSTP)"),
29 cl::init(Val: TPLoop::ForceDisabled),
30 cl::values(clEnumValN(TPLoop::ForceDisabled, "force-disabled",
31 "Don't convert memcpy to TP loop."),
32 clEnumValN(TPLoop::ForceEnabled, "force-enabled",
33 "Always convert memcpy to TP loop."),
34 clEnumValN(TPLoop::Allow, "allow",
35 "Allow (may be subject to certain conditions) "
36 "conversion of memcpy to TP loop.")));
37
38ARMSelectionDAGInfo::ARMSelectionDAGInfo()
39 : SelectionDAGGenTargetInfo(ARMGenSDNodeInfo) {}
40
41const char *ARMSelectionDAGInfo::getTargetNodeName(unsigned Opcode) const {
42#define MAKE_CASE(V) \
43 case V: \
44 return #V;
45
46 // These nodes don't have corresponding entries in *.td files yet.
47 switch (static_cast<ARMISD::NodeType>(Opcode)) {
48 MAKE_CASE(ARMISD::DYN_ALLOC)
49 MAKE_CASE(ARMISD::MVESEXT)
50 MAKE_CASE(ARMISD::MVEZEXT)
51 MAKE_CASE(ARMISD::MVETRUNC)
52 MAKE_CASE(ARMISD::BUILD_VECTOR)
53 MAKE_CASE(ARMISD::VLD1DUP)
54 MAKE_CASE(ARMISD::VLD2DUP)
55 MAKE_CASE(ARMISD::VLD3DUP)
56 MAKE_CASE(ARMISD::VLD4DUP)
57 MAKE_CASE(ARMISD::VLD1_UPD)
58 MAKE_CASE(ARMISD::VLD2_UPD)
59 MAKE_CASE(ARMISD::VLD3_UPD)
60 MAKE_CASE(ARMISD::VLD4_UPD)
61 MAKE_CASE(ARMISD::VLD1x2_UPD)
62 MAKE_CASE(ARMISD::VLD1x3_UPD)
63 MAKE_CASE(ARMISD::VLD1x4_UPD)
64 MAKE_CASE(ARMISD::VLD2LN_UPD)
65 MAKE_CASE(ARMISD::VLD3LN_UPD)
66 MAKE_CASE(ARMISD::VLD4LN_UPD)
67 MAKE_CASE(ARMISD::VLD1DUP_UPD)
68 MAKE_CASE(ARMISD::VLD2DUP_UPD)
69 MAKE_CASE(ARMISD::VLD3DUP_UPD)
70 MAKE_CASE(ARMISD::VLD4DUP_UPD)
71 MAKE_CASE(ARMISD::VST1_UPD)
72 MAKE_CASE(ARMISD::VST3_UPD)
73 MAKE_CASE(ARMISD::VST1x2_UPD)
74 MAKE_CASE(ARMISD::VST1x3_UPD)
75 MAKE_CASE(ARMISD::VST1x4_UPD)
76 MAKE_CASE(ARMISD::VST2LN_UPD)
77 MAKE_CASE(ARMISD::VST3LN_UPD)
78 MAKE_CASE(ARMISD::VST4LN_UPD)
79 }
80#undef MAKE_CASE
81
82 return SelectionDAGGenTargetInfo::getTargetNodeName(Opcode);
83}
84
85bool ARMSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const {
86 // These nodes don't have corresponding entries in *.td files yet.
87 if (Opcode >= ARMISD::FIRST_MEMORY_OPCODE &&
88 Opcode <= ARMISD::LAST_MEMORY_OPCODE)
89 return true;
90
91 return SelectionDAGGenTargetInfo::isTargetMemoryOpcode(Opcode);
92}
93
94void ARMSelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG,
95 const SDNode *N) const {
96 switch (N->getOpcode()) {
97 default:
98 break;
99 case ARMISD::WIN__DBZCHK:
100 // invalid number of results; expected 2, got 1
101 case ARMISD::WIN__CHKSTK:
102 // invalid number of results; expected 1, got 2
103 case ARMISD::COPY_STRUCT_BYVAL:
104 // invalid number of operands; expected 6, got 5
105 case ARMISD::MEMCPY:
106 // invalid number of operands; expected 5, got 4
107 case ARMISD::VMOVRRD:
108 // operand #0 must have type f64, but has type v1i64/v4f16/v8i8
109 case ARMISD::VMOVIMM:
110 // operand #0 must have type i32, but has type i16
111 return;
112 }
113
114 SelectionDAGGenTargetInfo::verifyTargetNode(DAG, N);
115}
116
117// Emit, if possible, a specialized version of the given Libcall. Typically this
118// means selecting the appropriately aligned version, but we also convert memset
119// of 0 into memclr.
120SDValue ARMSelectionDAGInfo::EmitSpecializedLibcall(
121 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
122 SDValue Size, unsigned Align, RTLIB::Libcall LC, bool &IsTailCall) const {
123 bool CanTailCall = IsTailCall;
124 IsTailCall = false;
125
126 const ARMSubtarget &Subtarget =
127 DAG.getMachineFunction().getSubtarget<ARMSubtarget>();
128 const ARMTargetLowering *TLI = Subtarget.getTargetLowering();
129
130 // Only use the AEABI functions if they are available.
131 //
132 // Translate RTLIB::Libcall to AEABILibcall. We only do this in order to be
133 // able to translate memset to memclr and use the value to index the function
134 // name array.
135 enum {
136 AEABI_MEMCPY = 0,
137 AEABI_MEMMOVE,
138 AEABI_MEMSET,
139 AEABI_MEMCLR
140 } AEABILibcall;
141 switch (LC) {
142 case RTLIB::MEMCPY:
143 if (DAG.getLibcalls().getLibcallImpl(Call: RTLIB::AEABI_MEMCPY) ==
144 RTLIB::Unsupported)
145 return SDValue();
146
147 AEABILibcall = AEABI_MEMCPY;
148 break;
149 case RTLIB::MEMMOVE:
150 if (DAG.getLibcalls().getLibcallImpl(Call: RTLIB::AEABI_MEMMOVE) ==
151 RTLIB::Unsupported)
152 return SDValue();
153
154 AEABILibcall = AEABI_MEMMOVE;
155 break;
156 case RTLIB::MEMSET:
157 if (DAG.getLibcalls().getLibcallImpl(Call: RTLIB::AEABI_MEMSET) ==
158 RTLIB::Unsupported)
159 return SDValue();
160
161 AEABILibcall = AEABI_MEMSET;
162 if (isNullConstant(V: Src))
163 AEABILibcall = AEABI_MEMCLR;
164 break;
165 default:
166 return SDValue();
167 }
168
169 // Choose the most-aligned libcall variant that we can
170 enum {
171 ALIGN1 = 0,
172 ALIGN4,
173 ALIGN8
174 } AlignVariant;
175 if ((Align & 7) == 0)
176 AlignVariant = ALIGN8;
177 else if ((Align & 3) == 0)
178 AlignVariant = ALIGN4;
179 else
180 AlignVariant = ALIGN1;
181
182 TargetLowering::ArgListTy Args;
183 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(C&: *DAG.getContext());
184 Args.emplace_back(args&: Dst, args&: IntPtrTy);
185 if (AEABILibcall == AEABI_MEMCLR) {
186 Args.emplace_back(args&: Size, args&: IntPtrTy);
187 } else if (AEABILibcall == AEABI_MEMSET) {
188 // Adjust parameters for memset, EABI uses format (ptr, size, value),
189 // GNU library uses (ptr, value, size)
190 // See RTABI section 4.3.4
191 Args.emplace_back(args&: Size, args&: IntPtrTy);
192
193 // Extend or truncate the argument to be an i32 value for the call.
194 if (Src.getValueType().bitsGT(VT: MVT::i32))
195 Src = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Src);
196 else if (Src.getValueType().bitsLT(VT: MVT::i32))
197 Src = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i32, Operand: Src);
198
199 TargetLowering::ArgListEntry Entry(Src,
200 Type::getInt32Ty(C&: *DAG.getContext()));
201 Entry.IsSExt = false;
202 Args.push_back(x: Entry);
203 } else {
204 Args.emplace_back(args&: Src, args&: IntPtrTy);
205 Args.emplace_back(args&: Size, args&: IntPtrTy);
206 }
207
208 static const RTLIB::Libcall FunctionImpls[4][3] = {
209 {RTLIB::AEABI_MEMCPY, RTLIB::AEABI_MEMCPY4, RTLIB::AEABI_MEMCPY8},
210 {RTLIB::AEABI_MEMMOVE, RTLIB::AEABI_MEMMOVE4, RTLIB::AEABI_MEMMOVE8},
211 {RTLIB::AEABI_MEMSET, RTLIB::AEABI_MEMSET4, RTLIB::AEABI_MEMSET8},
212 {RTLIB::AEABI_MEMCLR, RTLIB::AEABI_MEMCLR4, RTLIB::AEABI_MEMCLR8}};
213
214 RTLIB::Libcall NewLC = FunctionImpls[AEABILibcall][AlignVariant];
215 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: NewLC);
216 if (LCImpl == RTLIB::Unsupported)
217 return SDValue();
218
219 TargetLowering::CallLoweringInfo CLI(DAG);
220 CLI.setDebugLoc(dl)
221 .setChain(Chain)
222 .setLibCallee(
223 CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
224 ResultType: Type::getVoidTy(C&: *DAG.getContext()),
225 Target: DAG.getExternalSymbol(LCImpl, VT: TLI->getPointerTy(DL: DAG.getDataLayout())),
226 ArgsList: std::move(Args))
227 .setDiscardResult()
228 .setTailCall(CanTailCall);
229 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
230 IsTailCall = CLI.IsTailCall;
231 return CallResult.second;
232}
233
234static bool shouldGenerateInlineTPLoop(const ARMSubtarget &Subtarget,
235 const SelectionDAG &DAG,
236 ConstantSDNode *ConstantSize,
237 Align Alignment, bool IsMemcpy) {
238 auto &F = DAG.getMachineFunction().getFunction();
239 if (!EnableMemtransferTPLoop)
240 return false;
241 if (EnableMemtransferTPLoop == TPLoop::ForceEnabled)
242 return true;
243 // Do not generate inline TP loop if optimizations is disabled,
244 // or if optimization for size (-Os or -Oz) is on.
245 if (F.hasOptNone() || F.hasOptSize())
246 return false;
247 // If cli option is unset, for memset always generate inline TP.
248 // For memcpy, check some conditions
249 if (!IsMemcpy)
250 return true;
251 if (!ConstantSize && Alignment >= Align(4))
252 return true;
253 if (ConstantSize &&
254 ConstantSize->getZExtValue() > Subtarget.getMaxInlineSizeThreshold() &&
255 ConstantSize->getZExtValue() <
256 Subtarget.getMaxMemcpyTPInlineSizeThreshold())
257 return true;
258 return false;
259}
260
261SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(
262 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
263 SDValue Size, Align DstAlign, Align SrcAlign, bool isVolatile,
264 bool AlwaysInline, bool &IsTailCall, MachinePointerInfo DstPtrInfo,
265 MachinePointerInfo SrcPtrInfo) const {
266 Align Alignment = std::min(a: DstAlign, b: SrcAlign);
267 const ARMSubtarget &Subtarget =
268 DAG.getMachineFunction().getSubtarget<ARMSubtarget>();
269 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
270
271 if (Subtarget.hasMVEIntegerOps() &&
272 shouldGenerateInlineTPLoop(Subtarget, DAG, ConstantSize, Alignment, IsMemcpy: true))
273 return DAG.getNode(Opcode: ARMISD::MEMCPYLOOP, DL: dl, VT: MVT::Other, N1: Chain, N2: Dst, N3: Src,
274 N4: DAG.getZExtOrTrunc(Op: Size, DL: dl, VT: MVT::i32));
275
276 // Do repeated 4-byte loads and stores. To be improved.
277 // This requires 4-byte alignment.
278 if (Alignment < Align(4)) {
279 if (AlwaysInline) {
280 IsTailCall = false;
281 return SDValue();
282 }
283
284 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
285 Align: Alignment.value(), LC: RTLIB::MEMCPY, IsTailCall);
286 }
287
288 // This requires the copy size to be a constant, preferably
289 // within a subtarget-specific limit.
290 if (!ConstantSize)
291 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
292 Align: Alignment.value(), LC: RTLIB::MEMCPY, IsTailCall);
293 uint64_t SizeVal = ConstantSize->getZExtValue();
294 if (!AlwaysInline && SizeVal > Subtarget.getMaxInlineSizeThreshold())
295 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
296 Align: Alignment.value(), LC: RTLIB::MEMCPY, IsTailCall);
297
298 unsigned BytesLeft = SizeVal & 3;
299 unsigned NumMemOps = SizeVal >> 2;
300 unsigned EmittedNumMemOps = 0;
301 EVT VT = MVT::i32;
302 unsigned VTSize = 4;
303 unsigned i = 0;
304 // Emit a maximum of 4 loads in Thumb1 since we have fewer registers
305 const unsigned MaxLoadsInLDM = Subtarget.isThumb1Only() ? 4 : 6;
306 SDValue TFOps[6];
307 SDValue Loads[6];
308 uint64_t SrcOff = 0, DstOff = 0;
309
310 // FIXME: We should invent a VMEMCPY pseudo-instruction that lowers to
311 // VLDM/VSTM and make this code emit it when appropriate. This would reduce
312 // pressure on the general purpose registers. However this seems harder to map
313 // onto the register allocator's view of the world.
314
315 // The number of MEMCPY pseudo-instructions to emit. We use up to
316 // MaxLoadsInLDM registers per mcopy, which will get lowered into ldm/stm
317 // later on. This is a lower bound on the number of MEMCPY operations we must
318 // emit.
319 unsigned NumMEMCPYs = (NumMemOps + MaxLoadsInLDM - 1) / MaxLoadsInLDM;
320
321 // Code size optimisation: do not inline memcpy if expansion results in
322 // more instructions than the library call.
323 if (NumMEMCPYs > 1 && Subtarget.hasMinSize()) {
324 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
325 Align: Alignment.value(), LC: RTLIB::MEMCPY, IsTailCall);
326 }
327
328 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32, VT3: MVT::Other, VT4: MVT::Glue);
329
330 for (unsigned I = 0; I != NumMEMCPYs; ++I) {
331 // Evenly distribute registers among MEMCPY operations to reduce register
332 // pressure.
333 unsigned NextEmittedNumMemOps = NumMemOps * (I + 1) / NumMEMCPYs;
334 unsigned NumRegs = NextEmittedNumMemOps - EmittedNumMemOps;
335
336 Dst = DAG.getNode(Opcode: ARMISD::MEMCPY, DL: dl, VTList: VTs, N1: Chain, N2: Dst, N3: Src,
337 N4: DAG.getConstant(Val: NumRegs, DL: dl, VT: MVT::i32));
338 Src = Dst.getValue(R: 1);
339 Chain = Dst.getValue(R: 2);
340
341 DstPtrInfo = DstPtrInfo.getWithOffset(O: NumRegs * VTSize);
342 SrcPtrInfo = SrcPtrInfo.getWithOffset(O: NumRegs * VTSize);
343
344 EmittedNumMemOps = NextEmittedNumMemOps;
345 }
346
347 if (BytesLeft == 0)
348 return Chain;
349
350 // Issue loads / stores for the trailing (1 - 3) bytes.
351 auto getRemainingValueType = [](unsigned BytesLeft) {
352 return (BytesLeft >= 2) ? MVT::i16 : MVT::i8;
353 };
354 auto getRemainingSize = [](unsigned BytesLeft) {
355 return (BytesLeft >= 2) ? 2 : 1;
356 };
357
358 unsigned BytesLeftSave = BytesLeft;
359 i = 0;
360 while (BytesLeft) {
361 VT = getRemainingValueType(BytesLeft);
362 VTSize = getRemainingSize(BytesLeft);
363 Loads[i] = DAG.getLoad(VT, dl, Chain,
364 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: Src,
365 N2: DAG.getConstant(Val: SrcOff, DL: dl, VT: MVT::i32)),
366 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff));
367 TFOps[i] = Loads[i].getValue(R: 1);
368 ++i;
369 SrcOff += VTSize;
370 BytesLeft -= VTSize;
371 }
372 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: ArrayRef(TFOps, i));
373
374 i = 0;
375 BytesLeft = BytesLeftSave;
376 while (BytesLeft) {
377 VT = getRemainingValueType(BytesLeft);
378 VTSize = getRemainingSize(BytesLeft);
379 TFOps[i] = DAG.getStore(Chain, dl, Val: Loads[i],
380 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: Dst,
381 N2: DAG.getConstant(Val: DstOff, DL: dl, VT: MVT::i32)),
382 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff));
383 ++i;
384 DstOff += VTSize;
385 BytesLeft -= VTSize;
386 }
387 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: ArrayRef(TFOps, i));
388}
389
390SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemmove(
391 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
392 SDValue Size, Align DstAlign, Align SrcAlign, bool isVolatile,
393 bool &IsTailCall, MachinePointerInfo DstPtrInfo,
394 MachinePointerInfo SrcPtrInfo) const {
395 Align Alignment = std::min(a: DstAlign, b: SrcAlign);
396 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
397 Align: Alignment.value(), LC: RTLIB::MEMMOVE, IsTailCall);
398}
399
400SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemset(
401 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
402 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
403 bool &IsTailCall, MachinePointerInfo DstPtrInfo) const {
404
405 const ARMSubtarget &Subtarget =
406 DAG.getMachineFunction().getSubtarget<ARMSubtarget>();
407
408 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
409
410 // Generate TP loop for llvm.memset
411 if (Subtarget.hasMVEIntegerOps() &&
412 shouldGenerateInlineTPLoop(Subtarget, DAG, ConstantSize, Alignment,
413 IsMemcpy: false)) {
414 Src = DAG.getSplatBuildVector(VT: MVT::v16i8, DL: dl,
415 Op: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i8, Operand: Src));
416 return DAG.getNode(Opcode: ARMISD::MEMSETLOOP, DL: dl, VT: MVT::Other, N1: Chain, N2: Dst, N3: Src,
417 N4: DAG.getZExtOrTrunc(Op: Size, DL: dl, VT: MVT::i32));
418 }
419
420 if (!AlwaysInline)
421 return EmitSpecializedLibcall(DAG, dl, Chain, Dst, Src, Size,
422 Align: Alignment.value(), LC: RTLIB::MEMSET, IsTailCall);
423
424 IsTailCall = false;
425 return SDValue();
426}
427