1//===-- SystemZSelectionDAGInfo.cpp - SystemZ SelectionDAG Info -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SystemZSelectionDAGInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZSelectionDAGInfo.h"
14#include "SystemZTargetMachine.h"
15#include "llvm/CodeGen/SelectionDAG.h"
16
17#define GET_SDNODE_DESC
18#include "SystemZGenSDNodeInfo.inc"
19
20using namespace llvm;
21
22#define DEBUG_TYPE "systemz-selectiondag-info"
23
24SystemZSelectionDAGInfo::SystemZSelectionDAGInfo()
25 : SelectionDAGGenTargetInfo(SystemZGenSDNodeInfo) {}
26
27const char *SystemZSelectionDAGInfo::getTargetNodeName(unsigned Opcode) const {
28 switch (static_cast<SystemZISD::NodeType>(Opcode)) {
29 case SystemZISD::GET_CCMASK:
30 return "SystemZISD::GET_CCMASK";
31 }
32
33 return SelectionDAGGenTargetInfo::getTargetNodeName(Opcode);
34}
35
36static unsigned getMemMemLenAdj(unsigned Op) {
37 return Op == SystemZISD::MEMSET_MVC ? 2 : 1;
38}
39
40static SDValue createMemMemNode(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
41 SDValue Chain, SDValue Dst, SDValue Src,
42 SDValue LenAdj, SDValue Byte) {
43 SDVTList VTs = Op == SystemZISD::CLC ? DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other)
44 : DAG.getVTList(VT: MVT::Other);
45 SmallVector<SDValue, 6> Ops;
46 if (Op == SystemZISD::MEMSET_MVC)
47 Ops = { Chain, Dst, LenAdj, Byte };
48 else
49 Ops = { Chain, Dst, Src, LenAdj };
50 return DAG.getNode(Opcode: Op, DL, VTList: VTs, Ops);
51}
52
53// Emit a mem-mem operation after subtracting one (or two for memset) from
54// size, which will be added back during pseudo expansion. As the Reg case
55// emitted here may be converted by DAGCombiner into having an Imm length,
56// they are both emitted the same way.
57static SDValue emitMemMemImm(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
58 SDValue Chain, SDValue Dst, SDValue Src,
59 uint64_t Size, SDValue Byte = SDValue()) {
60 unsigned Adj = getMemMemLenAdj(Op);
61 assert(Size >= Adj && "Adjusted length overflow.");
62 SDValue LenAdj = DAG.getConstant(Val: Size - Adj, DL, VT: Dst.getValueType());
63 return createMemMemNode(DAG, DL, Op, Chain, Dst, Src, LenAdj, Byte);
64}
65
66static SDValue emitMemMemReg(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
67 SDValue Chain, SDValue Dst, SDValue Src,
68 SDValue Size, SDValue Byte = SDValue()) {
69 int64_t Adj = getMemMemLenAdj(Op);
70 SDValue LenAdj = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64,
71 N1: DAG.getZExtOrTrunc(Op: Size, DL, VT: MVT::i64),
72 N2: DAG.getSignedConstant(Val: 0 - Adj, DL, VT: MVT::i64));
73 return createMemMemNode(DAG, DL, Op, Chain, Dst, Src, LenAdj, Byte);
74}
75
76SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(
77 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
78 SDValue Size, Align DstAlign, Align SrcAlign, bool IsVolatile,
79 bool AlwaysInline, bool &IsTailCall, MachinePointerInfo DstPtrInfo,
80 MachinePointerInfo SrcPtrInfo) const {
81 IsTailCall = false;
82 if (IsVolatile)
83 return SDValue();
84
85 if (auto *CSize = dyn_cast<ConstantSDNode>(Val&: Size))
86 return emitMemMemImm(DAG, DL, Op: SystemZISD::MVC, Chain, Dst, Src,
87 Size: CSize->getZExtValue());
88
89 return emitMemMemReg(DAG, DL, Op: SystemZISD::MVC, Chain, Dst, Src, Size);
90}
91
92SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemmove(
93 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
94 SDValue Size, Align DstAlign, Align SrcAlign, bool IsVolatile,
95 bool &IsTailCall, MachinePointerInfo DstPtrInfo,
96 MachinePointerInfo SrcPtrInfo) const {
97 IsTailCall = false;
98 if (IsVolatile)
99 return SDValue();
100
101 const SystemZSubtarget &Subtarget =
102 DAG.getMachineFunction().getSubtarget<SystemZSubtarget>();
103
104 if (auto *CSize = dyn_cast<ConstantSDNode>(Val&: Size))
105 if (Subtarget.hasMiscellaneousExtensions3() && CSize->getZExtValue() > 0 &&
106 CSize->getZExtValue() <= 256) {
107 SDValue Size64 = DAG.getZExtOrTrunc(Op: Size, DL, VT: MVT::i64);
108 return DAG.getNode(Opcode: SystemZISD::MEMMOVE, DL, VT: MVT::Other,
109 Ops: {Chain, Dst, Src, Size64});
110 }
111
112 return SDValue();
113}
114
115// Handle a memset of 1, 2, 4 or 8 bytes with the operands given by
116// Chain, Dst, ByteVal and Size. These cases are expected to use
117// MVI, MVHHI, MVHI and MVGHI respectively.
118static SDValue memsetStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
119 SDValue Dst, uint64_t ByteVal, uint64_t Size,
120 Align Alignment, MachinePointerInfo DstPtrInfo) {
121 uint64_t StoreVal = ByteVal;
122 for (unsigned I = 1; I < Size; ++I)
123 StoreVal |= ByteVal << (I * 8);
124 return DAG.getStore(
125 Chain, dl: DL, Val: DAG.getConstant(Val: StoreVal, DL, VT: MVT::getIntegerVT(BitWidth: Size * 8)),
126 Ptr: Dst, PtrInfo: DstPtrInfo, Alignment);
127}
128
129SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset(
130 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst,
131 SDValue Byte, SDValue Size, Align Alignment, bool IsVolatile,
132 bool AlwaysInline, bool &IsTailCall, MachinePointerInfo DstPtrInfo) const {
133 IsTailCall = false;
134 EVT PtrVT = Dst.getValueType();
135
136 if (IsVolatile)
137 return SDValue();
138
139 auto *CByte = dyn_cast<ConstantSDNode>(Val&: Byte);
140 if (auto *CSize = dyn_cast<ConstantSDNode>(Val&: Size)) {
141 uint64_t Bytes = CSize->getZExtValue();
142 if (Bytes == 0)
143 return SDValue();
144 if (CByte) {
145 // Handle cases that can be done using at most two of
146 // MVI, MVHI, MVHHI and MVGHI. The latter two can only be
147 // used if ByteVal is all zeros or all ones; in other cases,
148 // we can move at most 2 halfwords.
149 uint64_t ByteVal = CByte->getZExtValue();
150 if (ByteVal == 0 || ByteVal == 255
151 ? Bytes <= 16 && llvm::popcount(Value: Bytes) <= 2
152 : Bytes <= 4) {
153 unsigned Size1 = Bytes == 16 ? 8 : llvm::bit_floor(Value: Bytes);
154 unsigned Size2 = Bytes - Size1;
155 SDValue Chain1 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size: Size1,
156 Alignment, DstPtrInfo);
157 if (Size2 == 0)
158 return Chain1;
159 Dst = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Dst,
160 N2: DAG.getConstant(Val: Size1, DL, VT: PtrVT));
161 DstPtrInfo = DstPtrInfo.getWithOffset(O: Size1);
162 SDValue Chain2 =
163 memsetStore(DAG, DL, Chain, Dst, ByteVal, Size: Size2,
164 Alignment: std::min(a: Alignment, b: Align(Size1)), DstPtrInfo);
165 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Chain1, N2: Chain2);
166 }
167 } else {
168 // Handle one and two bytes using STC.
169 if (Bytes <= 2) {
170 SDValue Chain1 =
171 DAG.getStore(Chain, dl: DL, Val: Byte, Ptr: Dst, PtrInfo: DstPtrInfo, Alignment);
172 if (Bytes == 1)
173 return Chain1;
174 SDValue Dst2 = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Dst,
175 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
176 SDValue Chain2 = DAG.getStore(Chain, dl: DL, Val: Byte, Ptr: Dst2,
177 PtrInfo: DstPtrInfo.getWithOffset(O: 1), Alignment: Align(1));
178 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Chain1, N2: Chain2);
179 }
180 }
181 assert(Bytes >= 2 && "Should have dealt with 0- and 1-byte cases already");
182
183 // Handle the special case of a memset of 0, which can use XC.
184 if (CByte && CByte->getZExtValue() == 0)
185 return emitMemMemImm(DAG, DL, Op: SystemZISD::XC, Chain, Dst, Src: Dst, Size: Bytes);
186
187 return emitMemMemImm(DAG, DL, Op: SystemZISD::MEMSET_MVC, Chain, Dst, Src: SDValue(),
188 Size: Bytes, Byte: DAG.getAnyExtOrTrunc(Op: Byte, DL, VT: MVT::i32));
189 }
190
191 // Variable length
192 if (CByte && CByte->getZExtValue() == 0)
193 // Handle the special case of a variable length memset of 0 with XC.
194 return emitMemMemReg(DAG, DL, Op: SystemZISD::XC, Chain, Dst, Src: Dst, Size);
195
196 return emitMemMemReg(DAG, DL, Op: SystemZISD::MEMSET_MVC, Chain, Dst, Src: SDValue(),
197 Size, Byte: DAG.getAnyExtOrTrunc(Op: Byte, DL, VT: MVT::i32));
198}
199
200// Convert the current CC value into an integer that is 0 if CC == 0,
201// greater than zero if CC == 1 and less than zero if CC >= 2.
202// The sequence starts with IPM, which puts CC into bits 29 and 28
203// of an integer and clears bits 30 and 31.
204static SDValue addIPMSequence(const SDLoc &DL, SDValue CCReg,
205 SelectionDAG &DAG) {
206 SDValue IPM = DAG.getNode(Opcode: SystemZISD::IPM, DL, VT: MVT::i32, Operand: CCReg);
207 SDValue SHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: IPM,
208 N2: DAG.getConstant(Val: 30 - SystemZ::IPM_CC, DL, VT: MVT::i32));
209 SDValue SRA = DAG.getNode(Opcode: ISD::SRA, DL, VT: MVT::i32, N1: SHL,
210 N2: DAG.getConstant(Val: 30, DL, VT: MVT::i32));
211 return SRA;
212}
213
214std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp(
215 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
216 SDValue Src2, SDValue Size, const CallInst *CI) const {
217 SDValue CCReg;
218 // Swap operands to invert CC == 1 vs. CC == 2 cases.
219 if (auto *CSize = dyn_cast<ConstantSDNode>(Val&: Size)) {
220 uint64_t Bytes = CSize->getZExtValue();
221 assert(Bytes > 0 && "Caller should have handled 0-size case");
222 CCReg = emitMemMemImm(DAG, DL, Op: SystemZISD::CLC, Chain, Dst: Src2, Src: Src1, Size: Bytes);
223 } else
224 CCReg = emitMemMemReg(DAG, DL, Op: SystemZISD::CLC, Chain, Dst: Src2, Src: Src1, Size);
225 Chain = CCReg.getValue(R: 1);
226 return std::make_pair(x: addIPMSequence(DL, CCReg, DAG), y&: Chain);
227}
228
229std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemchr(
230 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
231 SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const {
232 // Use SRST to find the character. End is its address on success.
233 EVT PtrVT = Src.getValueType();
234 SDVTList VTs = DAG.getVTList(VT1: PtrVT, VT2: MVT::i32, VT3: MVT::Other);
235 Length = DAG.getZExtOrTrunc(Op: Length, DL, VT: PtrVT);
236 Char = DAG.getZExtOrTrunc(Op: Char, DL, VT: MVT::i32);
237 Char = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Char,
238 N2: DAG.getConstant(Val: 255, DL, VT: MVT::i32));
239 SDValue Limit = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Src, N2: Length);
240 SDValue End = DAG.getNode(Opcode: SystemZISD::SEARCH_STRING, DL, VTList: VTs, N1: Chain,
241 N2: Limit, N3: Src, N4: Char);
242 SDValue CCReg = End.getValue(R: 1);
243 Chain = End.getValue(R: 2);
244
245 // Now select between End and null, depending on whether the character
246 // was found.
247 SDValue Ops[] = {
248 End, DAG.getConstant(Val: 0, DL, VT: PtrVT),
249 DAG.getTargetConstant(Val: SystemZ::CCMASK_SRST, DL, VT: MVT::i32),
250 DAG.getTargetConstant(Val: SystemZ::CCMASK_SRST_FOUND, DL, VT: MVT::i32), CCReg};
251 End = DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: PtrVT, Ops);
252 return std::make_pair(x&: End, y&: Chain);
253}
254
255std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcpy(
256 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest,
257 SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo,
258 bool isStpcpy, const CallInst *CI) const {
259 SDVTList VTs = DAG.getVTList(VT1: Dest.getValueType(), VT2: MVT::Other);
260 SDValue EndDest = DAG.getNode(Opcode: SystemZISD::STPCPY, DL, VTList: VTs, N1: Chain, N2: Dest, N3: Src,
261 N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
262 return std::make_pair(x&: isStpcpy ? EndDest : Dest, y: EndDest.getValue(R: 1));
263}
264
265std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcmp(
266 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
267 SDValue Src2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo,
268 const CallInst *CI) const {
269 SDVTList VTs = DAG.getVTList(VT1: Src1.getValueType(), VT2: MVT::i32, VT3: MVT::Other);
270 // Swap operands to invert CC == 1 vs. CC == 2 cases.
271 SDValue Unused = DAG.getNode(Opcode: SystemZISD::STRCMP, DL, VTList: VTs, N1: Chain, N2: Src2, N3: Src1,
272 N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
273 SDValue CCReg = Unused.getValue(R: 1);
274 Chain = Unused.getValue(R: 2);
275 return std::make_pair(x: addIPMSequence(DL, CCReg, DAG), y&: Chain);
276}
277
278// Search from Src for a null character, stopping once Src reaches Limit.
279// Return a pair of values, the first being the number of nonnull characters
280// and the second being the out chain.
281//
282// This can be used for strlen by setting Limit to 0.
283static std::pair<SDValue, SDValue> getBoundedStrlen(SelectionDAG &DAG,
284 const SDLoc &DL,
285 SDValue Chain, SDValue Src,
286 SDValue Limit) {
287 EVT PtrVT = Src.getValueType();
288 SDVTList VTs = DAG.getVTList(VT1: PtrVT, VT2: MVT::i32, VT3: MVT::Other);
289 SDValue End = DAG.getNode(Opcode: SystemZISD::SEARCH_STRING, DL, VTList: VTs, N1: Chain,
290 N2: Limit, N3: Src, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
291 Chain = End.getValue(R: 2);
292 SDValue Len = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: End, N2: Src);
293 return std::make_pair(x&: Len, y&: Chain);
294}
295
296std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrlen(
297 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
298 const CallInst *CI) const {
299 EVT PtrVT = Src.getValueType();
300 return getBoundedStrlen(DAG, DL, Chain, Src, Limit: DAG.getConstant(Val: 0, DL, VT: PtrVT));
301}
302
303std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrnlen(
304 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
305 SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const {
306 EVT PtrVT = Src.getValueType();
307 MaxLength = DAG.getZExtOrTrunc(Op: MaxLength, DL, VT: PtrVT);
308 SDValue Limit = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Src, N2: MaxLength);
309 return getBoundedStrlen(DAG, DL, Chain, Src, Limit);
310}
311