1//===-- SystemZSelectionDAGInfo.cpp - SystemZ SelectionDAG Info -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SystemZSelectionDAGInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZTargetMachine.h"
14#include "llvm/CodeGen/SelectionDAG.h"
15
16using namespace llvm;
17
18#define DEBUG_TYPE "systemz-selectiondag-info"
19
20bool SystemZSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const {
21 return Opcode >= SystemZISD::FIRST_MEMORY_OPCODE &&
22 Opcode <= SystemZISD::LAST_MEMORY_OPCODE;
23}
24
25bool SystemZSelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const {
26 return Opcode >= SystemZISD::FIRST_STRICTFP_OPCODE &&
27 Opcode <= SystemZISD::LAST_STRICTFP_OPCODE;
28}
29
30static unsigned getMemMemLenAdj(unsigned Op) {
31 return Op == SystemZISD::MEMSET_MVC ? 2 : 1;
32}
33
34static SDValue createMemMemNode(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
35 SDValue Chain, SDValue Dst, SDValue Src,
36 SDValue LenAdj, SDValue Byte) {
37 SDVTList VTs = Op == SystemZISD::CLC ? DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other)
38 : DAG.getVTList(VT: MVT::Other);
39 SmallVector<SDValue, 6> Ops;
40 if (Op == SystemZISD::MEMSET_MVC)
41 Ops = { Chain, Dst, LenAdj, Byte };
42 else
43 Ops = { Chain, Dst, Src, LenAdj };
44 return DAG.getNode(Opcode: Op, DL, VTList: VTs, Ops);
45}
46
47// Emit a mem-mem operation after subtracting one (or two for memset) from
48// size, which will be added back during pseudo expansion. As the Reg case
49// emitted here may be converted by DAGCombiner into having an Imm length,
50// they are both emitted the same way.
51static SDValue emitMemMemImm(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
52 SDValue Chain, SDValue Dst, SDValue Src,
53 uint64_t Size, SDValue Byte = SDValue()) {
54 unsigned Adj = getMemMemLenAdj(Op);
55 assert(Size >= Adj && "Adjusted length overflow.");
56 SDValue LenAdj = DAG.getConstant(Val: Size - Adj, DL, VT: Dst.getValueType());
57 return createMemMemNode(DAG, DL, Op, Chain, Dst, Src, LenAdj, Byte);
58}
59
60static SDValue emitMemMemReg(SelectionDAG &DAG, const SDLoc &DL, unsigned Op,
61 SDValue Chain, SDValue Dst, SDValue Src,
62 SDValue Size, SDValue Byte = SDValue()) {
63 int64_t Adj = getMemMemLenAdj(Op);
64 SDValue LenAdj = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64,
65 N1: DAG.getZExtOrTrunc(Op: Size, DL, VT: MVT::i64),
66 N2: DAG.getSignedConstant(Val: 0 - Adj, DL, VT: MVT::i64));
67 return createMemMemNode(DAG, DL, Op, Chain, Dst, Src, LenAdj, Byte);
68}
69
70SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(
71 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
72 SDValue Size, Align Alignment, bool IsVolatile, bool AlwaysInline,
73 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
74 if (IsVolatile)
75 return SDValue();
76
77 if (auto *CSize = dyn_cast<ConstantSDNode>(Val&: Size))
78 return emitMemMemImm(DAG, DL, Op: SystemZISD::MVC, Chain, Dst, Src,
79 Size: CSize->getZExtValue());
80
81 return emitMemMemReg(DAG, DL, Op: SystemZISD::MVC, Chain, Dst, Src, Size);
82}
83
84// Handle a memset of 1, 2, 4 or 8 bytes with the operands given by
85// Chain, Dst, ByteVal and Size. These cases are expected to use
86// MVI, MVHHI, MVHI and MVGHI respectively.
87static SDValue memsetStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
88 SDValue Dst, uint64_t ByteVal, uint64_t Size,
89 Align Alignment, MachinePointerInfo DstPtrInfo) {
90 uint64_t StoreVal = ByteVal;
91 for (unsigned I = 1; I < Size; ++I)
92 StoreVal |= ByteVal << (I * 8);
93 return DAG.getStore(
94 Chain, dl: DL, Val: DAG.getConstant(Val: StoreVal, DL, VT: MVT::getIntegerVT(BitWidth: Size * 8)),
95 Ptr: Dst, PtrInfo: DstPtrInfo, Alignment);
96}
97
98SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset(
99 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst,
100 SDValue Byte, SDValue Size, Align Alignment, bool IsVolatile,
101 bool AlwaysInline, MachinePointerInfo DstPtrInfo) const {
102 EVT PtrVT = Dst.getValueType();
103
104 if (IsVolatile)
105 return SDValue();
106
107 auto *CByte = dyn_cast<ConstantSDNode>(Val&: Byte);
108 if (auto *CSize = dyn_cast<ConstantSDNode>(Val&: Size)) {
109 uint64_t Bytes = CSize->getZExtValue();
110 if (Bytes == 0)
111 return SDValue();
112 if (CByte) {
113 // Handle cases that can be done using at most two of
114 // MVI, MVHI, MVHHI and MVGHI. The latter two can only be
115 // used if ByteVal is all zeros or all ones; in other cases,
116 // we can move at most 2 halfwords.
117 uint64_t ByteVal = CByte->getZExtValue();
118 if (ByteVal == 0 || ByteVal == 255
119 ? Bytes <= 16 && llvm::popcount(Value: Bytes) <= 2
120 : Bytes <= 4) {
121 unsigned Size1 = Bytes == 16 ? 8 : llvm::bit_floor(Value: Bytes);
122 unsigned Size2 = Bytes - Size1;
123 SDValue Chain1 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size: Size1,
124 Alignment, DstPtrInfo);
125 if (Size2 == 0)
126 return Chain1;
127 Dst = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Dst,
128 N2: DAG.getConstant(Val: Size1, DL, VT: PtrVT));
129 DstPtrInfo = DstPtrInfo.getWithOffset(O: Size1);
130 SDValue Chain2 =
131 memsetStore(DAG, DL, Chain, Dst, ByteVal, Size: Size2,
132 Alignment: std::min(a: Alignment, b: Align(Size1)), DstPtrInfo);
133 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Chain1, N2: Chain2);
134 }
135 } else {
136 // Handle one and two bytes using STC.
137 if (Bytes <= 2) {
138 SDValue Chain1 =
139 DAG.getStore(Chain, dl: DL, Val: Byte, Ptr: Dst, PtrInfo: DstPtrInfo, Alignment);
140 if (Bytes == 1)
141 return Chain1;
142 SDValue Dst2 = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Dst,
143 N2: DAG.getConstant(Val: 1, DL, VT: PtrVT));
144 SDValue Chain2 = DAG.getStore(Chain, dl: DL, Val: Byte, Ptr: Dst2,
145 PtrInfo: DstPtrInfo.getWithOffset(O: 1), Alignment: Align(1));
146 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Chain1, N2: Chain2);
147 }
148 }
149 assert(Bytes >= 2 && "Should have dealt with 0- and 1-byte cases already");
150
151 // Handle the special case of a memset of 0, which can use XC.
152 if (CByte && CByte->getZExtValue() == 0)
153 return emitMemMemImm(DAG, DL, Op: SystemZISD::XC, Chain, Dst, Src: Dst, Size: Bytes);
154
155 return emitMemMemImm(DAG, DL, Op: SystemZISD::MEMSET_MVC, Chain, Dst, Src: SDValue(),
156 Size: Bytes, Byte: DAG.getAnyExtOrTrunc(Op: Byte, DL, VT: MVT::i32));
157 }
158
159 // Variable length
160 if (CByte && CByte->getZExtValue() == 0)
161 // Handle the special case of a variable length memset of 0 with XC.
162 return emitMemMemReg(DAG, DL, Op: SystemZISD::XC, Chain, Dst, Src: Dst, Size);
163
164 return emitMemMemReg(DAG, DL, Op: SystemZISD::MEMSET_MVC, Chain, Dst, Src: SDValue(),
165 Size, Byte: DAG.getAnyExtOrTrunc(Op: Byte, DL, VT: MVT::i32));
166}
167
168// Convert the current CC value into an integer that is 0 if CC == 0,
169// greater than zero if CC == 1 and less than zero if CC >= 2.
170// The sequence starts with IPM, which puts CC into bits 29 and 28
171// of an integer and clears bits 30 and 31.
172static SDValue addIPMSequence(const SDLoc &DL, SDValue CCReg,
173 SelectionDAG &DAG) {
174 SDValue IPM = DAG.getNode(Opcode: SystemZISD::IPM, DL, VT: MVT::i32, Operand: CCReg);
175 SDValue SHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: IPM,
176 N2: DAG.getConstant(Val: 30 - SystemZ::IPM_CC, DL, VT: MVT::i32));
177 SDValue SRA = DAG.getNode(Opcode: ISD::SRA, DL, VT: MVT::i32, N1: SHL,
178 N2: DAG.getConstant(Val: 30, DL, VT: MVT::i32));
179 return SRA;
180}
181
182std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp(
183 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
184 SDValue Src2, SDValue Size, MachinePointerInfo Op1PtrInfo,
185 MachinePointerInfo Op2PtrInfo) const {
186 SDValue CCReg;
187 // Swap operands to invert CC == 1 vs. CC == 2 cases.
188 if (auto *CSize = dyn_cast<ConstantSDNode>(Val&: Size)) {
189 uint64_t Bytes = CSize->getZExtValue();
190 assert(Bytes > 0 && "Caller should have handled 0-size case");
191 CCReg = emitMemMemImm(DAG, DL, Op: SystemZISD::CLC, Chain, Dst: Src2, Src: Src1, Size: Bytes);
192 } else
193 CCReg = emitMemMemReg(DAG, DL, Op: SystemZISD::CLC, Chain, Dst: Src2, Src: Src1, Size);
194 Chain = CCReg.getValue(R: 1);
195 return std::make_pair(x: addIPMSequence(DL, CCReg, DAG), y&: Chain);
196}
197
198std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemchr(
199 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
200 SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const {
201 // Use SRST to find the character. End is its address on success.
202 EVT PtrVT = Src.getValueType();
203 SDVTList VTs = DAG.getVTList(VT1: PtrVT, VT2: MVT::i32, VT3: MVT::Other);
204 Length = DAG.getZExtOrTrunc(Op: Length, DL, VT: PtrVT);
205 Char = DAG.getZExtOrTrunc(Op: Char, DL, VT: MVT::i32);
206 Char = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Char,
207 N2: DAG.getConstant(Val: 255, DL, VT: MVT::i32));
208 SDValue Limit = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Src, N2: Length);
209 SDValue End = DAG.getNode(Opcode: SystemZISD::SEARCH_STRING, DL, VTList: VTs, N1: Chain,
210 N2: Limit, N3: Src, N4: Char);
211 SDValue CCReg = End.getValue(R: 1);
212 Chain = End.getValue(R: 2);
213
214 // Now select between End and null, depending on whether the character
215 // was found.
216 SDValue Ops[] = {
217 End, DAG.getConstant(Val: 0, DL, VT: PtrVT),
218 DAG.getTargetConstant(Val: SystemZ::CCMASK_SRST, DL, VT: MVT::i32),
219 DAG.getTargetConstant(Val: SystemZ::CCMASK_SRST_FOUND, DL, VT: MVT::i32), CCReg};
220 End = DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: PtrVT, Ops);
221 return std::make_pair(x&: End, y&: Chain);
222}
223
224std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcpy(
225 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest,
226 SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo,
227 bool isStpcpy) const {
228 SDVTList VTs = DAG.getVTList(VT1: Dest.getValueType(), VT2: MVT::Other);
229 SDValue EndDest = DAG.getNode(Opcode: SystemZISD::STPCPY, DL, VTList: VTs, N1: Chain, N2: Dest, N3: Src,
230 N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
231 return std::make_pair(x&: isStpcpy ? EndDest : Dest, y: EndDest.getValue(R: 1));
232}
233
234std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcmp(
235 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
236 SDValue Src2, MachinePointerInfo Op1PtrInfo,
237 MachinePointerInfo Op2PtrInfo) const {
238 SDVTList VTs = DAG.getVTList(VT1: Src1.getValueType(), VT2: MVT::i32, VT3: MVT::Other);
239 // Swap operands to invert CC == 1 vs. CC == 2 cases.
240 SDValue Unused = DAG.getNode(Opcode: SystemZISD::STRCMP, DL, VTList: VTs, N1: Chain, N2: Src2, N3: Src1,
241 N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
242 SDValue CCReg = Unused.getValue(R: 1);
243 Chain = Unused.getValue(R: 2);
244 return std::make_pair(x: addIPMSequence(DL, CCReg, DAG), y&: Chain);
245}
246
247// Search from Src for a null character, stopping once Src reaches Limit.
248// Return a pair of values, the first being the number of nonnull characters
249// and the second being the out chain.
250//
251// This can be used for strlen by setting Limit to 0.
252static std::pair<SDValue, SDValue> getBoundedStrlen(SelectionDAG &DAG,
253 const SDLoc &DL,
254 SDValue Chain, SDValue Src,
255 SDValue Limit) {
256 EVT PtrVT = Src.getValueType();
257 SDVTList VTs = DAG.getVTList(VT1: PtrVT, VT2: MVT::i32, VT3: MVT::Other);
258 SDValue End = DAG.getNode(Opcode: SystemZISD::SEARCH_STRING, DL, VTList: VTs, N1: Chain,
259 N2: Limit, N3: Src, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
260 Chain = End.getValue(R: 2);
261 SDValue Len = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: End, N2: Src);
262 return std::make_pair(x&: Len, y&: Chain);
263}
264
265std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrlen(
266 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
267 MachinePointerInfo SrcPtrInfo) const {
268 EVT PtrVT = Src.getValueType();
269 return getBoundedStrlen(DAG, DL, Chain, Src, Limit: DAG.getConstant(Val: 0, DL, VT: PtrVT));
270}
271
272std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrnlen(
273 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src,
274 SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const {
275 EVT PtrVT = Src.getValueType();
276 MaxLength = DAG.getZExtOrTrunc(Op: MaxLength, DL, VT: PtrVT);
277 SDValue Limit = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Src, N2: MaxLength);
278 return getBoundedStrlen(DAG, DL, Chain, Src, Limit);
279}
280