1//===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64SelectionDAGInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64SelectionDAGInfo.h"
14#include "AArch64MachineFunctionInfo.h"
15
16#define GET_SDNODE_DESC
17#include "AArch64GenSDNodeInfo.inc"
18#undef GET_SDNODE_DESC
19
20using namespace llvm;
21
22#define DEBUG_TYPE "aarch64-selectiondag-info"
23
24AArch64SelectionDAGInfo::AArch64SelectionDAGInfo()
25 : SelectionDAGGenTargetInfo(AArch64GenSDNodeInfo) {}
26
27void AArch64SelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG,
28 const SDNode *N) const {
29 switch (N->getOpcode()) {
30 case AArch64ISD::WrapperLarge:
31 // operand #0 must have type i32, but has type i64
32 return;
33 }
34
35 SelectionDAGGenTargetInfo::verifyTargetNode(DAG, N);
36
37#ifndef NDEBUG
38 // Some additional checks not yet implemented by verifyTargetNode.
39 switch (N->getOpcode()) {
40 case AArch64ISD::CTTZ_ELTS:
41 case AArch64ISD::CTTZ_ELTS_ZERO_POISON:
42 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
43 "Expected the general-predicate and mask to have matching types");
44 break;
45 case AArch64ISD::SUNPKLO:
46 case AArch64ISD::SUNPKHI:
47 case AArch64ISD::UUNPKLO:
48 case AArch64ISD::UUNPKHI: {
49 EVT VT = N->getValueType(0);
50 EVT OpVT = N->getOperand(0).getValueType();
51 assert(OpVT.isVector() && VT.isVector() && OpVT.isInteger() &&
52 VT.isInteger() && "Expected integer vectors!");
53 assert(OpVT.getSizeInBits() == VT.getSizeInBits() &&
54 "Expected vectors of equal size!");
55 assert(OpVT.getVectorElementCount() == VT.getVectorElementCount() * 2 &&
56 "Expected result vector with half the lanes of its input!");
57 break;
58 }
59 case AArch64ISD::TRN1:
60 case AArch64ISD::TRN2:
61 case AArch64ISD::UZP1:
62 case AArch64ISD::UZP2:
63 case AArch64ISD::ZIP1:
64 case AArch64ISD::ZIP2: {
65 EVT VT = N->getValueType(0);
66 EVT Op0VT = N->getOperand(0).getValueType();
67 EVT Op1VT = N->getOperand(1).getValueType();
68 assert(VT.isVector() && Op0VT.isVector() && Op1VT.isVector() &&
69 "Expected vectors!");
70 assert(VT == Op0VT && VT == Op1VT && "Expected matching vectors!");
71 break;
72 }
73 case AArch64ISD::RSHRNB_I: {
74 EVT VT = N->getValueType(0);
75 EVT Op0VT = N->getOperand(0).getValueType();
76 assert(VT.isVector() && VT.isInteger() &&
77 "Expected integer vector result type!");
78 assert(Op0VT.isVector() && Op0VT.isInteger() &&
79 "Expected first operand to be an integer vector!");
80 assert(VT.getSizeInBits() == Op0VT.getSizeInBits() &&
81 "Expected vectors of equal size!");
82 assert(VT.getVectorElementCount() == Op0VT.getVectorElementCount() * 2 &&
83 "Expected input vector with half the lanes of its result!");
84 assert(isa<ConstantSDNode>(N->getOperand(1)) &&
85 "Expected second operand to be a constant!");
86 break;
87 }
88 }
89#endif
90}
91
92SDValue AArch64SelectionDAGInfo::EmitMOPS(unsigned Opcode, SelectionDAG &DAG,
93 const SDLoc &DL, SDValue Chain,
94 SDValue Dst, SDValue SrcOrValue,
95 SDValue Size, Align DstAlign,
96 Align SrcAlign, bool isVolatile,
97 MachinePointerInfo DstPtrInfo,
98 MachinePointerInfo SrcPtrInfo) const {
99
100 // Get the constant size of the copy/set.
101 LocationSize MemSize = LocationSize::afterPointer();
102 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Size))
103 MemSize = LocationSize::precise(Value: C->getZExtValue());
104
105 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
106 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
107
108 MachineFunction &MF = DAG.getMachineFunction();
109
110 auto Vol =
111 isVolatile ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
112 auto DstFlags = MachineMemOperand::MOStore | Vol;
113 auto *DstOp =
114 MF.getMachineMemOperand(PtrInfo: DstPtrInfo, F: DstFlags, Size: MemSize, BaseAlignment: DstAlign);
115
116 if (IsSet) {
117 // Extend value to i64, if required.
118 if (SrcOrValue.getValueType() != MVT::i64)
119 SrcOrValue = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: SrcOrValue);
120 SDValue Ops[] = {Dst, Size, SrcOrValue, Chain};
121 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other};
122 MachineSDNode *Node = DAG.getMachineNode(Opcode, dl: DL, ResultTys, Ops);
123 DAG.setNodeMemRefs(N: Node, NewMemRefs: {DstOp});
124 return SDValue(Node, 2);
125 } else {
126 SDValue Ops[] = {Dst, SrcOrValue, Size, Chain};
127 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other};
128 MachineSDNode *Node = DAG.getMachineNode(Opcode, dl: DL, ResultTys, Ops);
129
130 auto SrcFlags = MachineMemOperand::MOLoad | Vol;
131 auto *SrcOp =
132 MF.getMachineMemOperand(PtrInfo: SrcPtrInfo, F: SrcFlags, Size: MemSize, BaseAlignment: SrcAlign);
133 DAG.setNodeMemRefs(N: Node, NewMemRefs: {DstOp, SrcOp});
134 return SDValue(Node, 3);
135 }
136}
137
138SDValue AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(
139 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Op0, SDValue Op1,
140 SDValue Size, RTLIB::Libcall LC, bool &IsTailCall) const {
141 bool CanTailCall = IsTailCall;
142 IsTailCall = false;
143
144 const AArch64Subtarget &STI =
145 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
146 const AArch64TargetLowering *TLI = STI.getTargetLowering();
147 TargetLowering::ArgListTy Args;
148 Args.emplace_back(args&: Op0, args: PointerType::getUnqual(C&: *DAG.getContext()));
149
150 bool UsesResult = false;
151 RTLIB::Libcall NewLC;
152 switch (LC) {
153 case RTLIB::MEMCPY: {
154 NewLC = RTLIB::SC_MEMCPY;
155 Args.emplace_back(args&: Op1, args: PointerType::getUnqual(C&: *DAG.getContext()));
156 break;
157 }
158 case RTLIB::MEMMOVE: {
159 NewLC = RTLIB::SC_MEMMOVE;
160 Args.emplace_back(args&: Op1, args: PointerType::getUnqual(C&: *DAG.getContext()));
161 break;
162 }
163 case RTLIB::MEMSET: {
164 NewLC = RTLIB::SC_MEMSET;
165 Args.emplace_back(args: DAG.getZExtOrTrunc(Op: Op1, DL, VT: MVT::i32),
166 args: Type::getInt32Ty(C&: *DAG.getContext()));
167 break;
168 }
169 case RTLIB::MEMCHR: {
170 UsesResult = true;
171 NewLC = RTLIB::SC_MEMCHR;
172 Args.emplace_back(args: DAG.getZExtOrTrunc(Op: Op1, DL, VT: MVT::i32),
173 args: Type::getInt32Ty(C&: *DAG.getContext()));
174 break;
175 }
176 default:
177 return SDValue();
178 }
179
180 RTLIB::LibcallImpl NewLCImpl = DAG.getLibcalls().getLibcallImpl(Call: NewLC);
181 if (NewLCImpl == RTLIB::Unsupported)
182 return SDValue();
183
184 EVT PointerVT = TLI->getPointerTy(DL: DAG.getDataLayout());
185 SDValue Symbol = DAG.getExternalSymbol(LCImpl: NewLCImpl, VT: PointerVT);
186 Args.emplace_back(args&: Size, args: DAG.getDataLayout().getIntPtrType(C&: *DAG.getContext()));
187
188 TargetLowering::CallLoweringInfo CLI(DAG);
189 PointerType *RetTy = PointerType::getUnqual(C&: *DAG.getContext());
190 CLI.setDebugLoc(DL)
191 .setChain(Chain)
192 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: NewLCImpl),
193 ResultType: RetTy, Target: Symbol, ArgsList: std::move(Args))
194 .setTailCall(CanTailCall);
195
196 auto [Result, ChainOut] = TLI->LowerCallTo(CLI);
197 IsTailCall = CLI.IsTailCall;
198 return UsesResult ? DAG.getMergeValues(Ops: {Result, ChainOut}, dl: DL) : ChainOut;
199}
200
201SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy(
202 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
203 SDValue Size, Align DstAlign, Align SrcAlign, bool isVolatile,
204 bool AlwaysInline, bool &IsTailCall, MachinePointerInfo DstPtrInfo,
205 MachinePointerInfo SrcPtrInfo) const {
206 const AArch64Subtarget &STI =
207 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
208
209 if (STI.getCLOpts().use_mops && STI.hasMOPS())
210 return EmitMOPS(Opcode: AArch64::MOPSMemoryCopyPseudo, DAG, DL, Chain, Dst, SrcOrValue: Src,
211 Size, DstAlign, SrcAlign, isVolatile, DstPtrInfo,
212 SrcPtrInfo);
213
214 auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
215 SMEAttrs Attrs = AFI->getSMEFnAttrs();
216 if (STI.getCLOpts().lower_to_sme_routines &&
217 !Attrs.hasNonStreamingInterfaceAndBody())
218 return EmitStreamingCompatibleMemLibCall(DAG, DL, Chain, Op0: Dst, Op1: Src, Size,
219 LC: RTLIB::MEMCPY, IsTailCall);
220
221 IsTailCall = false;
222 return SDValue();
223}
224
225SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
226 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
227 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
228 bool &IsTailCall, MachinePointerInfo DstPtrInfo) const {
229 const AArch64Subtarget &STI =
230 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
231
232 if (STI.getCLOpts().use_mops && STI.hasMOPS())
233 return EmitMOPS(Opcode: AArch64::MOPSMemorySetPseudo, DAG, DL: dl, Chain, Dst, SrcOrValue: Src,
234 Size, DstAlign: Alignment, SrcAlign: Alignment, isVolatile, DstPtrInfo,
235 SrcPtrInfo: MachinePointerInfo{});
236
237 auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
238 SMEAttrs Attrs = AFI->getSMEFnAttrs();
239 if (STI.getCLOpts().lower_to_sme_routines &&
240 !Attrs.hasNonStreamingInterfaceAndBody())
241 return EmitStreamingCompatibleMemLibCall(DAG, DL: dl, Chain, Op0: Dst, Op1: Src, Size,
242 LC: RTLIB::MEMSET, IsTailCall);
243
244 IsTailCall = false;
245 return SDValue();
246}
247
248SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemmove(
249 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
250 SDValue Size, Align DstAlign, Align SrcAlign, bool isVolatile,
251 bool &IsTailCall, MachinePointerInfo DstPtrInfo,
252 MachinePointerInfo SrcPtrInfo) const {
253 const AArch64Subtarget &STI =
254 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
255
256 if (STI.getCLOpts().use_mops && STI.hasMOPS())
257 return EmitMOPS(Opcode: AArch64::MOPSMemoryMovePseudo, DAG, DL: dl, Chain, Dst, SrcOrValue: Src,
258 Size, DstAlign, SrcAlign, isVolatile, DstPtrInfo,
259 SrcPtrInfo);
260
261 auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
262 SMEAttrs Attrs = AFI->getSMEFnAttrs();
263 if (STI.getCLOpts().lower_to_sme_routines &&
264 !Attrs.hasNonStreamingInterfaceAndBody())
265 return EmitStreamingCompatibleMemLibCall(DAG, DL: dl, Chain, Op0: Dst, Op1: Src, Size,
266 LC: RTLIB::MEMMOVE, IsTailCall);
267
268 IsTailCall = false;
269 return SDValue();
270}
271
272std::pair<SDValue, SDValue> AArch64SelectionDAGInfo::EmitTargetCodeForMemchr(
273 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src,
274 SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const {
275 auto *AFI = DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
276 SMEAttrs Attrs = AFI->getSMEFnAttrs();
277 if (DAG.getSubtarget<AArch64Subtarget>().getCLOpts().lower_to_sme_routines &&
278 !Attrs.hasNonStreamingInterfaceAndBody()) {
279 bool IsTailCall = false;
280 SDValue Result = EmitStreamingCompatibleMemLibCall(
281 DAG, DL: dl, Chain, Op0: Src, Op1: Char, Size: Length, LC: RTLIB::MEMCHR, IsTailCall);
282 return std::make_pair(x: Result.getValue(R: 0), y: Result.getValue(R: 1));
283 }
284 return std::make_pair(x: SDValue(), y: SDValue());
285}
286
287static const int kSetTagLoopThreshold = 176;
288
289static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
290 SDValue Chain, SDValue Ptr, uint64_t ObjSize,
291 const MachineMemOperand *BaseMemOperand,
292 bool ZeroData) {
293 MachineFunction &MF = DAG.getMachineFunction();
294 unsigned ObjSizeScaled = ObjSize / 16;
295
296 SDValue TagSrc = Ptr;
297 if (Ptr.getOpcode() == ISD::FrameIndex) {
298 int FI = cast<FrameIndexSDNode>(Val&: Ptr)->getIndex();
299 Ptr = DAG.getTargetFrameIndex(FI, VT: MVT::i64);
300 // A frame index operand may end up as [SP + offset] => it is fine to use SP
301 // register as the tag source.
302 TagSrc = DAG.getRegister(Reg: AArch64::SP, VT: MVT::i64);
303 }
304
305 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
306 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
307
308 SmallVector<SDValue, 8> OutChains;
309 unsigned OffsetScaled = 0;
310 while (OffsetScaled < ObjSizeScaled) {
311 if (ObjSizeScaled - OffsetScaled >= 2) {
312 SDValue AddrNode = DAG.getMemBasePlusOffset(
313 Base: Ptr, Offset: TypeSize::getFixed(ExactSize: OffsetScaled * 16), DL: dl);
314 SDValue St = DAG.getMemIntrinsicNode(
315 Opcode: OpCode2, dl, VTList: DAG.getVTList(VT: MVT::Other),
316 Ops: {Chain, TagSrc, AddrNode},
317 MemVT: MVT::v4i64,
318 MMO: MF.getMachineMemOperand(MMO: BaseMemOperand, Offset: OffsetScaled * 16, Size: 16 * 2));
319 OffsetScaled += 2;
320 OutChains.push_back(Elt: St);
321 continue;
322 }
323
324 if (ObjSizeScaled - OffsetScaled > 0) {
325 SDValue AddrNode = DAG.getMemBasePlusOffset(
326 Base: Ptr, Offset: TypeSize::getFixed(ExactSize: OffsetScaled * 16), DL: dl);
327 SDValue St = DAG.getMemIntrinsicNode(
328 Opcode: OpCode1, dl, VTList: DAG.getVTList(VT: MVT::Other),
329 Ops: {Chain, TagSrc, AddrNode},
330 MemVT: MVT::v2i64,
331 MMO: MF.getMachineMemOperand(MMO: BaseMemOperand, Offset: OffsetScaled * 16, Size: 16));
332 OffsetScaled += 1;
333 OutChains.push_back(Elt: St);
334 }
335 }
336
337 SDValue Res = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
338 return Res;
339}
340
341SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
342 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
343 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
344 uint64_t ObjSize = Size->getAsZExtVal();
345 assert(ObjSize % 16 == 0);
346
347 MachineFunction &MF = DAG.getMachineFunction();
348 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
349 PtrInfo: DstPtrInfo, F: MachineMemOperand::MOStore, Size: ObjSize, BaseAlignment: Align(16));
350
351 bool UseSetTagRangeLoop =
352 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
353 if (!UseSetTagRangeLoop)
354 return EmitUnrolledSetTag(DAG, dl, Chain, Ptr: Addr, ObjSize, BaseMemOperand,
355 ZeroData);
356
357 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
358
359 unsigned Opcode;
360 if (Addr.getOpcode() == ISD::FrameIndex) {
361 int FI = cast<FrameIndexSDNode>(Val&: Addr)->getIndex();
362 Addr = DAG.getTargetFrameIndex(FI, VT: MVT::i64);
363 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
364 } else {
365 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
366 }
367 SDValue Ops[] = {DAG.getTargetConstant(Val: ObjSize, DL: dl, VT: MVT::i64), Addr, Chain};
368 SDNode *St = DAG.getMachineNode(Opcode, dl, ResultTys: ResTys, Ops);
369
370 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {BaseMemOperand});
371 return SDValue(St, 2);
372}
373