1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SystemZTargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZISelLowering.h"
14#include "SystemZCallingConv.h"
15#include "SystemZConstantPoolValue.h"
16#include "SystemZMachineFunctionInfo.h"
17#include "SystemZTargetMachine.h"
18#include "llvm/ADT/SmallSet.h"
19#include "llvm/CodeGen/CallingConvLower.h"
20#include "llvm/CodeGen/ISDOpcodes.h"
21#include "llvm/CodeGen/MachineInstrBuilder.h"
22#include "llvm/CodeGen/MachineRegisterInfo.h"
23#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
24#include "llvm/IR/GlobalAlias.h"
25#include "llvm/IR/IntrinsicInst.h"
26#include "llvm/IR/Intrinsics.h"
27#include "llvm/IR/IntrinsicsS390.h"
28#include "llvm/IR/PatternMatch.h"
29#include "llvm/Support/CommandLine.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Support/KnownBits.h"
32#include <cctype>
33#include <optional>
34
35using namespace llvm;
36
37#define DEBUG_TYPE "systemz-lower"
38
39// Temporarily let this be disabled by default until all known problems
40// related to argument extensions are fixed.
41static cl::opt<bool> EnableIntArgExtCheck(
42 "argext-abi-check", cl::init(Val: false),
43 cl::desc("Verify that narrow int args are properly extended per the "
44 "SystemZ ABI."));
45
46namespace {
47// Represents information about a comparison.
48struct Comparison {
49 Comparison(SDValue Op0In, SDValue Op1In, SDValue ChainIn)
50 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
51 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
52
53 // The operands to the comparison.
54 SDValue Op0, Op1;
55
56 // Chain if this is a strict floating-point comparison.
57 SDValue Chain;
58
59 // The opcode that should be used to compare Op0 and Op1.
60 unsigned Opcode;
61
62 // A SystemZICMP value. Only used for integer comparisons.
63 unsigned ICmpType;
64
65 // The mask of CC values that Opcode can produce.
66 unsigned CCValid;
67
68 // The mask of CC values for which the original condition is true.
69 unsigned CCMask;
70};
71} // end anonymous namespace
72
73// Classify VT as either 32 or 64 bit.
74static bool is32Bit(EVT VT) {
75 switch (VT.getSimpleVT().SimpleTy) {
76 case MVT::i32:
77 return true;
78 case MVT::i64:
79 return false;
80 default:
81 llvm_unreachable("Unsupported type");
82 }
83}
84
85// Return a version of MachineOperand that can be safely used before the
86// final use.
87static MachineOperand earlyUseOperand(MachineOperand Op) {
88 if (Op.isReg())
89 Op.setIsKill(false);
90 return Op;
91}
92
93SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
94 const SystemZSubtarget &STI)
95 : TargetLowering(TM, STI), Subtarget(STI) {
96 MVT PtrVT = MVT::getIntegerVT(BitWidth: TM.getPointerSizeInBits(AS: 0));
97
98 auto *Regs = STI.getSpecialRegisters();
99
100 // Set up the register classes.
101 if (Subtarget.hasHighWord())
102 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GRX32BitRegClass);
103 else
104 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GR32BitRegClass);
105 addRegisterClass(VT: MVT::i64, RC: &SystemZ::GR64BitRegClass);
106 if (!useSoftFloat()) {
107 if (Subtarget.hasVector()) {
108 addRegisterClass(VT: MVT::f16, RC: &SystemZ::VR16BitRegClass);
109 addRegisterClass(VT: MVT::f32, RC: &SystemZ::VR32BitRegClass);
110 addRegisterClass(VT: MVT::f64, RC: &SystemZ::VR64BitRegClass);
111 } else {
112 addRegisterClass(VT: MVT::f16, RC: &SystemZ::FP16BitRegClass);
113 addRegisterClass(VT: MVT::f32, RC: &SystemZ::FP32BitRegClass);
114 addRegisterClass(VT: MVT::f64, RC: &SystemZ::FP64BitRegClass);
115 }
116 if (Subtarget.hasVectorEnhancements1())
117 addRegisterClass(VT: MVT::f128, RC: &SystemZ::VR128BitRegClass);
118 else
119 addRegisterClass(VT: MVT::f128, RC: &SystemZ::FP128BitRegClass);
120
121 if (Subtarget.hasVector()) {
122 addRegisterClass(VT: MVT::v16i8, RC: &SystemZ::VR128BitRegClass);
123 addRegisterClass(VT: MVT::v8i16, RC: &SystemZ::VR128BitRegClass);
124 addRegisterClass(VT: MVT::v4i32, RC: &SystemZ::VR128BitRegClass);
125 addRegisterClass(VT: MVT::v2i64, RC: &SystemZ::VR128BitRegClass);
126 addRegisterClass(VT: MVT::v8f16, RC: &SystemZ::VR128BitRegClass);
127 addRegisterClass(VT: MVT::v4f32, RC: &SystemZ::VR128BitRegClass);
128 addRegisterClass(VT: MVT::v2f64, RC: &SystemZ::VR128BitRegClass);
129 }
130
131 if (Subtarget.hasVector())
132 addRegisterClass(VT: MVT::i128, RC: &SystemZ::VR128BitRegClass);
133 }
134
135 // Compute derived properties from the register classes
136 computeRegisterProperties(TRI: Subtarget.getRegisterInfo());
137
138 // Set up special registers.
139 setStackPointerRegisterToSaveRestore(Regs->getStackPointerRegister());
140
141 // TODO: It may be better to default to latency-oriented scheduling, however
142 // LLVM's current latency-oriented scheduler can't handle physreg definitions
143 // such as SystemZ has with CC, so set this to the register-pressure
144 // scheduler, because it can.
145 setSchedulingPreference(Sched::RegPressure);
146
147 setBooleanContents(ZeroOrOneBooleanContent);
148 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
149
150 setMaxAtomicSizeInBitsSupported(128);
151
152 // Instructions are strings of 2-byte aligned 2-byte values.
153 setMinFunctionAlignment(Align(2));
154 // For performance reasons we prefer 16-byte alignment.
155 setPrefFunctionAlignment(Align(16));
156
157 // Handle operations that are handled in a similar way for all types.
158 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
159 I <= MVT::LAST_FP_VALUETYPE;
160 ++I) {
161 MVT VT = MVT::SimpleValueType(I);
162 if (isTypeLegal(VT)) {
163 // Lower SET_CC into an IPM-based sequence.
164 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
165 setOperationAction(Op: ISD::STRICT_FSETCC, VT, Action: Custom);
166 setOperationAction(Op: ISD::STRICT_FSETCCS, VT, Action: Custom);
167
168 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
169 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
170
171 // Lower SELECT_CC and BR_CC into separate comparisons and branches.
172 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Custom);
173 setOperationAction(Op: ISD::BR_CC, VT, Action: Custom);
174 }
175 }
176
177 // Expand jump table branches as address arithmetic followed by an
178 // indirect jump.
179 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
180
181 // Expand BRCOND into a BR_CC (see above).
182 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
183
184 // Handle integer types except i128.
185 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
186 I <= MVT::LAST_INTEGER_VALUETYPE;
187 ++I) {
188 MVT VT = MVT::SimpleValueType(I);
189 if (isTypeLegal(VT) && VT != MVT::i128) {
190 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
191
192 // Expand individual DIV and REMs into DIVREMs.
193 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
194 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
195 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
196 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
197 setOperationAction(Op: ISD::SDIVREM, VT, Action: Custom);
198 setOperationAction(Op: ISD::UDIVREM, VT, Action: Custom);
199
200 // Support addition/subtraction with overflow.
201 setOperationAction(Op: ISD::SADDO, VT, Action: Custom);
202 setOperationAction(Op: ISD::SSUBO, VT, Action: Custom);
203
204 // Support addition/subtraction with carry.
205 setOperationAction(Op: ISD::UADDO, VT, Action: Custom);
206 setOperationAction(Op: ISD::USUBO, VT, Action: Custom);
207
208 // Support carry in as value rather than glue.
209 setOperationAction(Op: ISD::UADDO_CARRY, VT, Action: Custom);
210 setOperationAction(Op: ISD::USUBO_CARRY, VT, Action: Custom);
211
212 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
213 // available, or if the operand is constant.
214 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT, Action: Custom);
215
216 // Use POPCNT on z196 and above.
217 if (Subtarget.hasPopulationCount())
218 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
219 else
220 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
221
222 // No special instructions for these.
223 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
224 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
225
226 // Use *MUL_LOHI where possible instead of MULH*.
227 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
228 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
229 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Custom);
230 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Custom);
231
232 // The fp<=>i32/i64 conversions are all Legal except for f16 and for
233 // unsigned on z10 (only z196 and above have native support for
234 // unsigned conversions).
235 for (auto Op : {ISD::FP_TO_SINT, ISD::STRICT_FP_TO_SINT,
236 ISD::SINT_TO_FP, ISD::STRICT_SINT_TO_FP})
237 setOperationAction(Op, VT, Action: Custom);
238 for (auto Op : {ISD::FP_TO_UINT, ISD::STRICT_FP_TO_UINT})
239 setOperationAction(Op, VT, Action: Custom);
240 for (auto Op : {ISD::UINT_TO_FP, ISD::STRICT_UINT_TO_FP}) {
241 // Handle unsigned 32-bit input types as signed 64-bit types on z10.
242 auto OpAction =
243 (!Subtarget.hasFPExtension() && VT == MVT::i32) ? Promote : Custom;
244 setOperationAction(Op, VT, Action: OpAction);
245 }
246 }
247 }
248
249 // Handle i128 if legal.
250 if (isTypeLegal(VT: MVT::i128)) {
251 // No special instructions for these.
252 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i128, Action: Expand);
253 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i128, Action: Expand);
254 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i128, Action: Expand);
255 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i128, Action: Expand);
256 setOperationAction(Op: ISD::ROTR, VT: MVT::i128, Action: Expand);
257 setOperationAction(Op: ISD::ROTL, VT: MVT::i128, Action: Expand);
258
259 // We may be able to use VSLDB/VSLD/VSRD for these.
260 setOperationAction(Op: ISD::FSHL, VT: MVT::i128, Action: Custom);
261 setOperationAction(Op: ISD::FSHR, VT: MVT::i128, Action: Custom);
262
263 // No special instructions for these before z17.
264 if (!Subtarget.hasVectorEnhancements3()) {
265 setOperationAction(Op: ISD::MUL, VT: MVT::i128, Action: Expand);
266 setOperationAction(Op: ISD::MULHS, VT: MVT::i128, Action: Expand);
267 setOperationAction(Op: ISD::MULHU, VT: MVT::i128, Action: Expand);
268 setOperationAction(Op: ISD::SDIV, VT: MVT::i128, Action: Expand);
269 setOperationAction(Op: ISD::UDIV, VT: MVT::i128, Action: Expand);
270 setOperationAction(Op: ISD::SREM, VT: MVT::i128, Action: Expand);
271 setOperationAction(Op: ISD::UREM, VT: MVT::i128, Action: Expand);
272 setOperationAction(Op: ISD::CTLZ, VT: MVT::i128, Action: Expand);
273 setOperationAction(Op: ISD::CTTZ, VT: MVT::i128, Action: Expand);
274 } else {
275 // Even if we do have a legal 128-bit multiply, we do not
276 // want 64-bit multiply-high operations to use it.
277 setOperationAction(Op: ISD::MULHS, VT: MVT::i64, Action: Custom);
278 setOperationAction(Op: ISD::MULHU, VT: MVT::i64, Action: Custom);
279 }
280
281 // Support addition/subtraction with carry.
282 setOperationAction(Op: ISD::UADDO, VT: MVT::i128, Action: Custom);
283 setOperationAction(Op: ISD::USUBO, VT: MVT::i128, Action: Custom);
284 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i128, Action: Custom);
285 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i128, Action: Custom);
286
287 // Use VPOPCT and add up partial results.
288 setOperationAction(Op: ISD::CTPOP, VT: MVT::i128, Action: Custom);
289
290 // Additional instructions available with z17.
291 if (Subtarget.hasVectorEnhancements3()) {
292 setOperationAction(Op: ISD::ABS, VT: MVT::i128, Action: Legal);
293
294 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
295 VT: MVT::i128, Action: Legal);
296 }
297 }
298
299 // These need custom handling in order to handle the f16 conversions.
300 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i128, Action: Custom);
301 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i128, Action: Custom);
302 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i128, Action: Custom);
303 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i128, Action: Custom);
304 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i128, Action: Custom);
305 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i128, Action: Custom);
306 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i128, Action: Custom);
307 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i128, Action: Custom);
308
309 // Type legalization will convert 8- and 16-bit atomic operations into
310 // forms that operate on i32s (but still keeping the original memory VT).
311 // Lower them into full i32 operations.
312 setOperationAction(Op: ISD::ATOMIC_SWAP, VT: MVT::i32, Action: Custom);
313 setOperationAction(Op: ISD::ATOMIC_LOAD_ADD, VT: MVT::i32, Action: Custom);
314 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT: MVT::i32, Action: Custom);
315 setOperationAction(Op: ISD::ATOMIC_LOAD_AND, VT: MVT::i32, Action: Custom);
316 setOperationAction(Op: ISD::ATOMIC_LOAD_OR, VT: MVT::i32, Action: Custom);
317 setOperationAction(Op: ISD::ATOMIC_LOAD_XOR, VT: MVT::i32, Action: Custom);
318 setOperationAction(Op: ISD::ATOMIC_LOAD_NAND, VT: MVT::i32, Action: Custom);
319 setOperationAction(Op: ISD::ATOMIC_LOAD_MIN, VT: MVT::i32, Action: Custom);
320 setOperationAction(Op: ISD::ATOMIC_LOAD_MAX, VT: MVT::i32, Action: Custom);
321 setOperationAction(Op: ISD::ATOMIC_LOAD_UMIN, VT: MVT::i32, Action: Custom);
322 setOperationAction(Op: ISD::ATOMIC_LOAD_UMAX, VT: MVT::i32, Action: Custom);
323
324 // Whether or not i128 is not a legal type, we need to custom lower
325 // the atomic operations in order to exploit SystemZ instructions.
326 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i128, Action: Custom);
327 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i128, Action: Custom);
328 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f128, Action: Custom);
329 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f128, Action: Custom);
330
331 // Mark sign/zero extending atomic loads as legal, which will make
332 // DAGCombiner fold extensions into atomic loads if possible.
333 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64,
334 MemVTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Legal);
335 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i32,
336 MemVTs: {MVT::i8, MVT::i16}, Action: Legal);
337 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i16,
338 MemVT: MVT::i8, Action: Legal);
339
340 // We can use the CC result of compare-and-swap to implement
341 // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
342 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i32, Action: Custom);
343 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i64, Action: Custom);
344 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i128, Action: Custom);
345
346 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
347
348 // Traps are legal, as we will convert them to "j .+2".
349 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
350
351 // We have native support for a 64-bit CTLZ, via FLOGR.
352 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Promote);
353 setOperationAction(Op: ISD::CTLZ_ZERO_UNDEF, VT: MVT::i32, Action: Promote);
354 setOperationAction(Op: ISD::CTLZ, VT: MVT::i64, Action: Legal);
355
356 // On z17 we have native support for a 64-bit CTTZ.
357 if (Subtarget.hasMiscellaneousExtensions4()) {
358 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Promote);
359 setOperationAction(Op: ISD::CTTZ_ZERO_UNDEF, VT: MVT::i32, Action: Promote);
360 setOperationAction(Op: ISD::CTTZ, VT: MVT::i64, Action: Legal);
361 }
362
363 // On z15 we have native support for a 64-bit CTPOP.
364 if (Subtarget.hasMiscellaneousExtensions3()) {
365 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Promote);
366 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Legal);
367 }
368
369 // Give LowerOperation the chance to replace 64-bit ORs with subregs.
370 setOperationAction(Op: ISD::OR, VT: MVT::i64, Action: Custom);
371
372 // Expand 128 bit shifts without using a libcall.
373 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64, Action: Expand);
374 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64, Action: Expand);
375 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64, Action: Expand);
376
377 // Also expand 256 bit shifts if i128 is a legal type.
378 if (isTypeLegal(VT: MVT::i128)) {
379 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i128, Action: Expand);
380 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i128, Action: Expand);
381 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i128, Action: Expand);
382 }
383
384 // Handle bitcast from fp128 to i128.
385 if (!isTypeLegal(VT: MVT::i128))
386 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
387
388 // We have native instructions for i8, i16 and i32 extensions, but not i1.
389 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
390 for (MVT VT : MVT::integer_valuetypes()) {
391 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
392 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
393 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
394 }
395
396 // Handle the various types of symbolic address.
397 setOperationAction(Op: ISD::ConstantPool, VT: PtrVT, Action: Custom);
398 setOperationAction(Op: ISD::GlobalAddress, VT: PtrVT, Action: Custom);
399 setOperationAction(Op: ISD::GlobalTLSAddress, VT: PtrVT, Action: Custom);
400 setOperationAction(Op: ISD::BlockAddress, VT: PtrVT, Action: Custom);
401 setOperationAction(Op: ISD::JumpTable, VT: PtrVT, Action: Custom);
402
403 // We need to handle dynamic allocations specially because of the
404 // 160-byte area at the bottom of the stack.
405 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: PtrVT, Action: Custom);
406 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: PtrVT, Action: Custom);
407
408 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
409 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Custom);
410
411 // Handle prefetches with PFD or PFDRL.
412 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
413
414 // Handle readcyclecounter with STCKF.
415 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Custom);
416
417 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
418 // Assume by default that all vector operations need to be expanded.
419 for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
420 if (getOperationAction(Op: Opcode, VT) == Legal)
421 setOperationAction(Op: Opcode, VT, Action: Expand);
422
423 // Likewise all truncating stores and extending loads.
424 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
425 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
426 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
427 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
428 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
429 }
430
431 if (isTypeLegal(VT)) {
432 // These operations are legal for anything that can be stored in a
433 // vector register, even if there is no native support for the format
434 // as such. In particular, we can do these for v4f32 even though there
435 // are no specific instructions for that format.
436 setOperationAction(Op: ISD::LOAD, VT, Action: Legal);
437 setOperationAction(Op: ISD::STORE, VT, Action: Legal);
438 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
439 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
440 setOperationAction(Op: ISD::UNDEF, VT, Action: Legal);
441
442 // Likewise, except that we need to replace the nodes with something
443 // more specific.
444 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
445 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
446 }
447 }
448
449 // Handle integer vector types.
450 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
451 if (isTypeLegal(VT)) {
452 // These operations have direct equivalents.
453 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Legal);
454 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Legal);
455 setOperationAction(Op: ISD::ADD, VT, Action: Legal);
456 setOperationAction(Op: ISD::SUB, VT, Action: Legal);
457 if (VT != MVT::v2i64 || Subtarget.hasVectorEnhancements3()) {
458 setOperationAction(Op: ISD::MUL, VT, Action: Legal);
459 setOperationAction(Op: ISD::MULHS, VT, Action: Legal);
460 setOperationAction(Op: ISD::MULHU, VT, Action: Legal);
461 }
462 if (Subtarget.hasVectorEnhancements3() &&
463 VT != MVT::v16i8 && VT != MVT::v8i16) {
464 setOperationAction(Op: ISD::SDIV, VT, Action: Legal);
465 setOperationAction(Op: ISD::UDIV, VT, Action: Legal);
466 setOperationAction(Op: ISD::SREM, VT, Action: Legal);
467 setOperationAction(Op: ISD::UREM, VT, Action: Legal);
468 }
469 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
470 setOperationAction(Op: ISD::AND, VT, Action: Legal);
471 setOperationAction(Op: ISD::OR, VT, Action: Legal);
472 setOperationAction(Op: ISD::XOR, VT, Action: Legal);
473 if (Subtarget.hasVectorEnhancements1())
474 setOperationAction(Op: ISD::CTPOP, VT, Action: Legal);
475 else
476 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
477 setOperationAction(Op: ISD::CTTZ, VT, Action: Legal);
478 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
479
480 // Convert a GPR scalar to a vector by inserting it into element 0.
481 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
482
483 // Use a series of unpacks for extensions.
484 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Custom);
485 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT, Action: Custom);
486
487 // Detect shifts/rotates by a scalar amount and convert them into
488 // V*_BY_SCALAR.
489 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
490 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
491 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
492 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
493
494 // Add ISD::VECREDUCE_ADD as custom in order to implement
495 // it with VZERO+VSUM
496 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
497
498 // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
499 // and inverting the result as necessary.
500 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
501
502 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT,
503 Action: Legal);
504 }
505 }
506
507 if (Subtarget.hasVector()) {
508 // There should be no need to check for float types other than v2f64
509 // since <2 x f32> isn't a legal type.
510 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
511 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
512 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
513 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
514 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
515 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
516 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
517 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
518
519 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
520 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
521 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
522 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
523 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
524 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
525 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
526 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
527 }
528
529 if (Subtarget.hasVectorEnhancements2()) {
530 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
531 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
532 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
533 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
534 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
535 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
536 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
537 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
538
539 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
540 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
541 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
542 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
543 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
544 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
545 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
546 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
547 }
548
549 // Handle floating-point types.
550 if (!useSoftFloat()) {
551 // Promote all f16 operations to float, with some exceptions below.
552 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
553 setOperationAction(Op: Opc, VT: MVT::f16, Action: Promote);
554 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Expand);
555 for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) {
556 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f16, Action: Expand);
557 setTruncStoreAction(ValVT: VT, MemVT: MVT::f16, Action: Expand);
558 }
559 for (auto Op : {ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE})
560 setOperationAction(Op, VT: MVT::f16, Action: Subtarget.hasVector() ? Legal : Custom);
561 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f16, Action: LibCall);
562 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: LibCall);
563 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
564 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f16, Action: Custom);
565 for (auto Op : {ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN})
566 setOperationAction(Op, VT: MVT::f16, Action: Legal);
567 }
568
569 for (unsigned I = MVT::FIRST_FP_VALUETYPE;
570 I <= MVT::LAST_FP_VALUETYPE;
571 ++I) {
572 MVT VT = MVT::SimpleValueType(I);
573 if (isTypeLegal(VT) && VT != MVT::f16) {
574 // We can use FI for FRINT.
575 setOperationAction(Op: ISD::FRINT, VT, Action: Legal);
576
577 // We can use the extended form of FI for other rounding operations.
578 if (Subtarget.hasFPExtension()) {
579 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Legal);
580 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
581 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
582 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
583 setOperationAction(Op: ISD::FROUND, VT, Action: Legal);
584 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
585 }
586
587 // No special instructions for these.
588 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
589 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
590 setOperationAction(Op: ISD::FSINCOS, VT, Action: Expand);
591 setOperationAction(Op: ISD::FREM, VT, Action: LibCall);
592 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
593
594 // Special treatment.
595 setOperationAction(Op: ISD::IS_FPCLASS, VT, Action: Custom);
596
597 // Handle constrained floating-point operations.
598 setOperationAction(Op: ISD::STRICT_FADD, VT, Action: Legal);
599 setOperationAction(Op: ISD::STRICT_FSUB, VT, Action: Legal);
600 setOperationAction(Op: ISD::STRICT_FMUL, VT, Action: Legal);
601 setOperationAction(Op: ISD::STRICT_FDIV, VT, Action: Legal);
602 setOperationAction(Op: ISD::STRICT_FMA, VT, Action: Legal);
603 setOperationAction(Op: ISD::STRICT_FSQRT, VT, Action: Legal);
604 setOperationAction(Op: ISD::STRICT_FRINT, VT, Action: Legal);
605 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT, Action: Legal);
606 if (Subtarget.hasFPExtension()) {
607 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT, Action: Legal);
608 setOperationAction(Op: ISD::STRICT_FFLOOR, VT, Action: Legal);
609 setOperationAction(Op: ISD::STRICT_FCEIL, VT, Action: Legal);
610 setOperationAction(Op: ISD::STRICT_FTRUNC, VT, Action: Legal);
611 setOperationAction(Op: ISD::STRICT_FROUND, VT, Action: Legal);
612 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT, Action: Legal);
613 }
614
615 // Extension from f16 needs libcall.
616 setOperationAction(Op: ISD::FP_EXTEND, VT, Action: Custom);
617 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT, Action: Custom);
618 }
619 }
620
621 // Handle floating-point vector types.
622 if (Subtarget.hasVector()) {
623 // Scalar-to-vector conversion is just a subreg.
624 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8f16, Action: Legal);
625 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Legal);
626 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
627
628 // Some insertions and extractions can be done directly but others
629 // need to go via integers.
630 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8f16, Action: Custom);
631 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
632 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
633 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v8f16, Action: Custom);
634 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
635 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
636
637 // These operations have direct equivalents.
638 setOperationAction(Op: ISD::FADD, VT: MVT::v2f64, Action: Legal);
639 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Legal);
640 setOperationAction(Op: ISD::FSUB, VT: MVT::v2f64, Action: Legal);
641 setOperationAction(Op: ISD::FMUL, VT: MVT::v2f64, Action: Legal);
642 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Legal);
643 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Legal);
644 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Legal);
645 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Legal);
646 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Legal);
647 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f64, Action: Legal);
648 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Legal);
649 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Legal);
650 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Legal);
651 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f64, Action: Legal);
652 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
653
654 // Handle constrained floating-point operations.
655 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v2f64, Action: Legal);
656 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v2f64, Action: Legal);
657 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v2f64, Action: Legal);
658 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v2f64, Action: Legal);
659 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v2f64, Action: Legal);
660 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v2f64, Action: Legal);
661 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v2f64, Action: Legal);
662 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v2f64, Action: Legal);
663 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v2f64, Action: Legal);
664 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v2f64, Action: Legal);
665 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v2f64, Action: Legal);
666 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v2f64, Action: Legal);
667 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
668
669 setOperationAction(Op: ISD::SETCC, VT: MVT::v2f64, Action: Custom);
670 setOperationAction(Op: ISD::SETCC, VT: MVT::v4f32, Action: Custom);
671 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v2f64, Action: Custom);
672 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v4f32, Action: Custom);
673 if (Subtarget.hasVectorEnhancements1()) {
674 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v2f64, Action: Custom);
675 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v4f32, Action: Custom);
676 }
677 }
678
679 // The vector enhancements facility 1 has instructions for these.
680 if (Subtarget.hasVectorEnhancements1()) {
681 setOperationAction(Op: ISD::FADD, VT: MVT::v4f32, Action: Legal);
682 setOperationAction(Op: ISD::FNEG, VT: MVT::v4f32, Action: Legal);
683 setOperationAction(Op: ISD::FSUB, VT: MVT::v4f32, Action: Legal);
684 setOperationAction(Op: ISD::FMUL, VT: MVT::v4f32, Action: Legal);
685 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Legal);
686 setOperationAction(Op: ISD::FDIV, VT: MVT::v4f32, Action: Legal);
687 setOperationAction(Op: ISD::FABS, VT: MVT::v4f32, Action: Legal);
688 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Legal);
689 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Legal);
690 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Legal);
691 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Legal);
692 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Legal);
693 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Legal);
694 setOperationAction(Op: ISD::FROUND, VT: MVT::v4f32, Action: Legal);
695 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
696
697 for (MVT Type : {MVT::f64, MVT::v2f64, MVT::f32, MVT::v4f32, MVT::f128}) {
698 setOperationAction(Op: ISD::FMAXNUM, VT: Type, Action: Legal);
699 setOperationAction(Op: ISD::FMAXIMUM, VT: Type, Action: Legal);
700 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: Type, Action: Legal);
701 setOperationAction(Op: ISD::FMINNUM, VT: Type, Action: Legal);
702 setOperationAction(Op: ISD::FMINIMUM, VT: Type, Action: Legal);
703 setOperationAction(Op: ISD::FMINIMUMNUM, VT: Type, Action: Legal);
704 }
705
706 // Handle constrained floating-point operations.
707 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v4f32, Action: Legal);
708 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v4f32, Action: Legal);
709 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v4f32, Action: Legal);
710 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v4f32, Action: Legal);
711 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v4f32, Action: Legal);
712 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v4f32, Action: Legal);
713 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v4f32, Action: Legal);
714 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v4f32, Action: Legal);
715 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v4f32, Action: Legal);
716 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v4f32, Action: Legal);
717 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v4f32, Action: Legal);
718 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v4f32, Action: Legal);
719 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
720 for (auto VT : { MVT::f32, MVT::f64, MVT::f128,
721 MVT::v4f32, MVT::v2f64 }) {
722 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT, Action: Legal);
723 setOperationAction(Op: ISD::STRICT_FMINNUM, VT, Action: Legal);
724 setOperationAction(Op: ISD::STRICT_FMAXIMUM, VT, Action: Legal);
725 setOperationAction(Op: ISD::STRICT_FMINIMUM, VT, Action: Legal);
726 }
727 }
728
729 // We only have fused f128 multiply-addition on vector registers.
730 if (!Subtarget.hasVectorEnhancements1()) {
731 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Expand);
732 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f128, Action: Expand);
733 }
734
735 // We don't have a copysign instruction on vector registers.
736 if (Subtarget.hasVectorEnhancements1())
737 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f128, Action: Expand);
738
739 // Needed so that we don't try to implement f128 constant loads using
740 // a load-and-extend of a f80 constant (in cases where the constant
741 // would fit in an f80).
742 for (MVT VT : MVT::fp_valuetypes())
743 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f80, Action: Expand);
744
745 // We don't have extending load instruction on vector registers.
746 if (Subtarget.hasVectorEnhancements1()) {
747 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
748 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
749 }
750
751 // Floating-point truncation and stores need to be done separately.
752 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
753 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
754 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
755
756 // We have 64-bit FPR<->GPR moves, but need special handling for
757 // 32-bit forms.
758 if (!Subtarget.hasVector()) {
759 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
760 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Custom);
761 }
762
763 // VASTART and VACOPY need to deal with the SystemZ-specific varargs
764 // structure, but VAEND is a no-op.
765 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
766 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Custom);
767 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
768
769 if (Subtarget.isTargetzOS()) {
770 // Handle address space casts between mixed sized pointers.
771 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i32, Action: Custom);
772 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i64, Action: Custom);
773 }
774
775 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
776
777 // Codes for which we want to perform some z-specific combinations.
778 setTargetDAGCombine({ISD::ZERO_EXTEND,
779 ISD::SIGN_EXTEND,
780 ISD::SIGN_EXTEND_INREG,
781 ISD::LOAD,
782 ISD::STORE,
783 ISD::VECTOR_SHUFFLE,
784 ISD::EXTRACT_VECTOR_ELT,
785 ISD::FP_ROUND,
786 ISD::STRICT_FP_ROUND,
787 ISD::FP_EXTEND,
788 ISD::SINT_TO_FP,
789 ISD::UINT_TO_FP,
790 ISD::STRICT_FP_EXTEND,
791 ISD::FCOPYSIGN,
792 ISD::BSWAP,
793 ISD::SETCC,
794 ISD::SRL,
795 ISD::SRA,
796 ISD::MUL,
797 ISD::SDIV,
798 ISD::UDIV,
799 ISD::SREM,
800 ISD::UREM,
801 ISD::INTRINSIC_VOID,
802 ISD::INTRINSIC_W_CHAIN});
803
804 // Handle intrinsics.
805 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
806 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
807
808 // We're not using SJLJ for exception handling, but they're implemented
809 // solely to support use of __builtin_setjmp / __builtin_longjmp.
810 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
811 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
812
813 // We want to use MVC in preference to even a single load/store pair.
814 MaxStoresPerMemcpy = Subtarget.hasVector() ? 2 : 0;
815 MaxStoresPerMemcpyOptSize = 0;
816
817 // The main memset sequence is a byte store followed by an MVC.
818 // Two STC or MV..I stores win over that, but the kind of fused stores
819 // generated by target-independent code don't when the byte value is
820 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better
821 // than "STC;MVC". Handle the choice in target-specific code instead.
822 MaxStoresPerMemset = Subtarget.hasVector() ? 2 : 0;
823 MaxStoresPerMemsetOptSize = 0;
824
825 // Default to having -disable-strictnode-mutation on
826 IsStrictFPEnabled = true;
827}
828
829bool SystemZTargetLowering::useSoftFloat() const {
830 return Subtarget.hasSoftFloat();
831}
832
833unsigned SystemZTargetLowering::getVectorTypeBreakdownForCallingConv(
834 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
835 unsigned &NumIntermediates, MVT &RegisterVT) const {
836 // Pass fp16 vectors in VR(s).
837 if (Subtarget.hasVector() && VT.isVector() && VT.getScalarType() == MVT::f16) {
838 IntermediateVT = RegisterVT = MVT::v8f16;
839 return NumIntermediates =
840 divideCeil(Numerator: VT.getVectorNumElements(), Denominator: SystemZ::VectorBytes / 2);
841 }
842 return TargetLowering::getVectorTypeBreakdownForCallingConv(
843 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
844}
845
846MVT SystemZTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
847 CallingConv::ID CC,
848 EVT VT) const {
849 // 128-bit single-element vector types are passed like other vectors,
850 // not like their element type.
851 if (VT.isVector() && VT.getSizeInBits() == 128 &&
852 VT.getVectorNumElements() == 1)
853 return MVT::v16i8;
854 // Pass fp16 vectors in VR(s).
855 if (Subtarget.hasVector() && VT.isVector() && VT.getScalarType() == MVT::f16)
856 return MVT::v8f16;
857 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
858}
859
860unsigned SystemZTargetLowering::getNumRegistersForCallingConv(
861 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
862 // Pass fp16 vectors in VR(s).
863 if (Subtarget.hasVector() && VT.isVector() && VT.getScalarType() == MVT::f16)
864 return divideCeil(Numerator: VT.getVectorNumElements(), Denominator: SystemZ::VectorBytes / 2);
865 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
866}
867
868EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,
869 LLVMContext &, EVT VT) const {
870 if (!VT.isVector())
871 return MVT::i32;
872 return VT.changeVectorElementTypeToInteger();
873}
874
875bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(
876 const MachineFunction &MF, EVT VT) const {
877 if (useSoftFloat())
878 return false;
879
880 VT = VT.getScalarType();
881
882 if (!VT.isSimple())
883 return false;
884
885 switch (VT.getSimpleVT().SimpleTy) {
886 case MVT::f32:
887 case MVT::f64:
888 return true;
889 case MVT::f128:
890 return Subtarget.hasVectorEnhancements1();
891 default:
892 break;
893 }
894
895 return false;
896}
897
898// Return true if the constant can be generated with a vector instruction,
899// such as VGM, VGMB or VREPI.
900bool SystemZVectorConstantInfo::isVectorConstantLegal(
901 const SystemZSubtarget &Subtarget) {
902 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
903 if (!Subtarget.hasVector() ||
904 (isFP128 && !Subtarget.hasVectorEnhancements1()))
905 return false;
906
907 // Try using VECTOR GENERATE BYTE MASK. This is the architecturally-
908 // preferred way of creating all-zero and all-one vectors so give it
909 // priority over other methods below.
910 unsigned Mask = 0;
911 unsigned I = 0;
912 for (; I < SystemZ::VectorBytes; ++I) {
913 uint64_t Byte = IntBits.lshr(shiftAmt: I * 8).trunc(width: 8).getZExtValue();
914 if (Byte == 0xff)
915 Mask |= 1ULL << I;
916 else if (Byte != 0)
917 break;
918 }
919 if (I == SystemZ::VectorBytes) {
920 Opcode = SystemZISD::BYTE_MASK;
921 OpVals.push_back(Elt: Mask);
922 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 8), NumElements: 16);
923 return true;
924 }
925
926 if (SplatBitSize > 64)
927 return false;
928
929 auto TryValue = [&](uint64_t Value) -> bool {
930 // Try VECTOR REPLICATE IMMEDIATE
931 int64_t SignedValue = SignExtend64(X: Value, B: SplatBitSize);
932 if (isInt<16>(x: SignedValue)) {
933 OpVals.push_back(Elt: ((unsigned) SignedValue));
934 Opcode = SystemZISD::REPLICATE;
935 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
936 NumElements: SystemZ::VectorBits / SplatBitSize);
937 return true;
938 }
939 // Try VECTOR GENERATE MASK
940 unsigned Start, End;
941 if (TII->isRxSBGMask(Mask: Value, BitSize: SplatBitSize, Start, End)) {
942 // isRxSBGMask returns the bit numbers for a full 64-bit value, with 0
943 // denoting 1 << 63 and 63 denoting 1. Convert them to bit numbers for
944 // an SplatBitSize value, so that 0 denotes 1 << (SplatBitSize-1).
945 OpVals.push_back(Elt: Start - (64 - SplatBitSize));
946 OpVals.push_back(Elt: End - (64 - SplatBitSize));
947 Opcode = SystemZISD::ROTATE_MASK;
948 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
949 NumElements: SystemZ::VectorBits / SplatBitSize);
950 return true;
951 }
952 return false;
953 };
954
955 // First try assuming that any undefined bits above the highest set bit
956 // and below the lowest set bit are 1s. This increases the likelihood of
957 // being able to use a sign-extended element value in VECTOR REPLICATE
958 // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
959 uint64_t SplatBitsZ = SplatBits.getZExtValue();
960 uint64_t SplatUndefZ = SplatUndef.getZExtValue();
961 unsigned LowerBits = llvm::countr_zero(Val: SplatBitsZ);
962 unsigned UpperBits = llvm::countl_zero(Val: SplatBitsZ);
963 uint64_t Lower = SplatUndefZ & maskTrailingOnes<uint64_t>(N: LowerBits);
964 uint64_t Upper = SplatUndefZ & maskLeadingOnes<uint64_t>(N: UpperBits);
965 if (TryValue(SplatBitsZ | Upper | Lower))
966 return true;
967
968 // Now try assuming that any undefined bits between the first and
969 // last defined set bits are set. This increases the chances of
970 // using a non-wraparound mask.
971 uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
972 return TryValue(SplatBitsZ | Middle);
973}
974
975SystemZVectorConstantInfo::SystemZVectorConstantInfo(APInt IntImm) {
976 if (IntImm.isSingleWord()) {
977 IntBits = APInt(128, IntImm.getZExtValue());
978 IntBits <<= (SystemZ::VectorBits - IntImm.getBitWidth());
979 } else
980 IntBits = IntImm;
981 assert(IntBits.getBitWidth() == 128 && "Unsupported APInt.");
982
983 // Find the smallest splat.
984 SplatBits = IntImm;
985 unsigned Width = SplatBits.getBitWidth();
986 while (Width > 8) {
987 unsigned HalfSize = Width / 2;
988 APInt HighValue = SplatBits.lshr(shiftAmt: HalfSize).trunc(width: HalfSize);
989 APInt LowValue = SplatBits.trunc(width: HalfSize);
990
991 // If the two halves do not match, stop here.
992 if (HighValue != LowValue || 8 > HalfSize)
993 break;
994
995 SplatBits = HighValue;
996 Width = HalfSize;
997 }
998 SplatUndef = 0;
999 SplatBitSize = Width;
1000}
1001
1002SystemZVectorConstantInfo::SystemZVectorConstantInfo(BuildVectorSDNode *BVN) {
1003 assert(BVN->isConstant() && "Expected a constant BUILD_VECTOR");
1004 bool HasAnyUndefs;
1005
1006 // Get IntBits by finding the 128 bit splat.
1007 BVN->isConstantSplat(SplatValue&: IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 128,
1008 isBigEndian: true);
1009
1010 // Get SplatBits by finding the 8 bit or greater splat.
1011 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 8,
1012 isBigEndian: true);
1013}
1014
1015bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1016 bool ForCodeSize) const {
1017 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
1018 if (Imm.isZero() || Imm.isNegZero())
1019 return true;
1020
1021 return SystemZVectorConstantInfo(Imm).isVectorConstantLegal(Subtarget);
1022}
1023
1024MachineBasicBlock *
1025SystemZTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
1026 MachineBasicBlock *MBB) const {
1027 DebugLoc DL = MI.getDebugLoc();
1028 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1029 const SystemZRegisterInfo *TRI = Subtarget.getRegisterInfo();
1030
1031 MachineFunction *MF = MBB->getParent();
1032 MachineRegisterInfo &MRI = MF->getRegInfo();
1033
1034 const BasicBlock *BB = MBB->getBasicBlock();
1035 MachineFunction::iterator I = ++MBB->getIterator();
1036
1037 Register DstReg = MI.getOperand(i: 0).getReg();
1038 const TargetRegisterClass *RC = MRI.getRegClass(Reg: DstReg);
1039 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
1040 (void)TRI;
1041 Register MainDstReg = MRI.createVirtualRegister(RegClass: RC);
1042 Register RestoreDstReg = MRI.createVirtualRegister(RegClass: RC);
1043
1044 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1045 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1046 // For v = setjmp(buf), we generate.
1047 // Algorithm:
1048 //
1049 // ---------
1050 // | thisMBB |
1051 // ---------
1052 // |
1053 // ------------------------
1054 // | |
1055 // ---------- ---------------
1056 // | mainMBB | | restoreMBB |
1057 // | v = 0 | | v = 1 |
1058 // ---------- ---------------
1059 // | |
1060 // -------------------------
1061 // |
1062 // -----------------------------
1063 // | sinkMBB |
1064 // | phi(v_mainMBB,v_restoreMBB) |
1065 // -----------------------------
1066 // thisMBB:
1067 // buf[FPOffset] = Frame Pointer if hasFP.
1068 // buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB.
1069 // buf[BCOffset] = Backchain value if building with -mbackchain.
1070 // buf[SPOffset] = Stack Pointer.
1071 // buf[LPOffset] = We never write this slot with R13, gcc stores R13 always.
1072 // SjLjSetup restoreMBB
1073 // mainMBB:
1074 // v_main = 0
1075 // sinkMBB:
1076 // v = phi(v_main, v_restore)
1077 // restoreMBB:
1078 // v_restore = 1
1079
1080 MachineBasicBlock *ThisMBB = MBB;
1081 MachineBasicBlock *MainMBB = MF->CreateMachineBasicBlock(BB);
1082 MachineBasicBlock *SinkMBB = MF->CreateMachineBasicBlock(BB);
1083 MachineBasicBlock *RestoreMBB = MF->CreateMachineBasicBlock(BB);
1084
1085 MF->insert(MBBI: I, MBB: MainMBB);
1086 MF->insert(MBBI: I, MBB: SinkMBB);
1087 MF->push_back(MBB: RestoreMBB);
1088 RestoreMBB->setMachineBlockAddressTaken();
1089
1090 MachineInstrBuilder MIB;
1091
1092 // Transfer the remainder of BB and its successor edges to sinkMBB.
1093 SinkMBB->splice(Where: SinkMBB->begin(), Other: MBB,
1094 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
1095 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
1096
1097 // thisMBB:
1098 const int64_t FPOffset = 0; // Slot 1.
1099 const int64_t LabelOffset = 1 * PVT.getStoreSize(); // Slot 2.
1100 const int64_t BCOffset = 2 * PVT.getStoreSize(); // Slot 3.
1101 const int64_t SPOffset = 3 * PVT.getStoreSize(); // Slot 4.
1102
1103 // Buf address.
1104 Register BufReg = MI.getOperand(i: 1).getReg();
1105
1106 const TargetRegisterClass *PtrRC = getRegClassFor(VT: PVT);
1107 Register LabelReg = MRI.createVirtualRegister(RegClass: PtrRC);
1108
1109 // Prepare IP for longjmp.
1110 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LARL), DestReg: LabelReg)
1111 .addMBB(MBB: RestoreMBB);
1112 // Store IP for return from jmp, slot 2, offset = 1.
1113 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1114 .addReg(RegNo: LabelReg)
1115 .addReg(RegNo: BufReg)
1116 .addImm(Val: LabelOffset)
1117 .addReg(RegNo: 0);
1118
1119 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1120 bool HasFP = Subtarget.getFrameLowering()->hasFP(MF: *MF);
1121 if (HasFP) {
1122 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1123 .addReg(RegNo: SpecialRegs->getFramePointerRegister())
1124 .addReg(RegNo: BufReg)
1125 .addImm(Val: FPOffset)
1126 .addReg(RegNo: 0);
1127 }
1128
1129 // Store SP.
1130 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1131 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1132 .addReg(RegNo: BufReg)
1133 .addImm(Val: SPOffset)
1134 .addReg(RegNo: 0);
1135
1136 // Slot 3(Offset = 2) Backchain value (if building with -mbackchain).
1137 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1138 if (BackChain) {
1139 Register BCReg = MRI.createVirtualRegister(RegClass: PtrRC);
1140 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1141 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1142 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1143 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1144 .addReg(RegNo: 0);
1145
1146 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1147 .addReg(RegNo: BCReg)
1148 .addReg(RegNo: BufReg)
1149 .addImm(Val: BCOffset)
1150 .addReg(RegNo: 0);
1151 }
1152
1153 // Setup.
1154 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EH_SjLj_Setup))
1155 .addMBB(MBB: RestoreMBB);
1156
1157 const SystemZRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1158 MIB.addRegMask(Mask: RegInfo->getNoPreservedMask());
1159
1160 ThisMBB->addSuccessor(Succ: MainMBB);
1161 ThisMBB->addSuccessor(Succ: RestoreMBB);
1162
1163 // mainMBB:
1164 BuildMI(BB: MainMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: MainDstReg).addImm(Val: 0);
1165 MainMBB->addSuccessor(Succ: SinkMBB);
1166
1167 // sinkMBB:
1168 BuildMI(BB&: *SinkMBB, I: SinkMBB->begin(), MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: DstReg)
1169 .addReg(RegNo: MainDstReg)
1170 .addMBB(MBB: MainMBB)
1171 .addReg(RegNo: RestoreDstReg)
1172 .addMBB(MBB: RestoreMBB);
1173
1174 // restoreMBB.
1175 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: RestoreDstReg).addImm(Val: 1);
1176 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: SinkMBB);
1177 RestoreMBB->addSuccessor(Succ: SinkMBB);
1178
1179 MI.eraseFromParent();
1180
1181 return SinkMBB;
1182}
1183
1184MachineBasicBlock *
1185SystemZTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
1186 MachineBasicBlock *MBB) const {
1187
1188 DebugLoc DL = MI.getDebugLoc();
1189 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1190
1191 MachineFunction *MF = MBB->getParent();
1192 MachineRegisterInfo &MRI = MF->getRegInfo();
1193
1194 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1195 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1196 Register BufReg = MI.getOperand(i: 0).getReg();
1197 const TargetRegisterClass *RC = MRI.getRegClass(Reg: BufReg);
1198 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1199
1200 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
1201 Register BCReg = MRI.createVirtualRegister(RegClass: RC);
1202
1203 MachineInstrBuilder MIB;
1204
1205 const int64_t FPOffset = 0;
1206 const int64_t LabelOffset = 1 * PVT.getStoreSize();
1207 const int64_t BCOffset = 2 * PVT.getStoreSize();
1208 const int64_t SPOffset = 3 * PVT.getStoreSize();
1209 const int64_t LPOffset = 4 * PVT.getStoreSize();
1210
1211 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: Tmp)
1212 .addReg(RegNo: BufReg)
1213 .addImm(Val: LabelOffset)
1214 .addReg(RegNo: 0);
1215
1216 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1217 DestReg: SpecialRegs->getFramePointerRegister())
1218 .addReg(RegNo: BufReg)
1219 .addImm(Val: FPOffset)
1220 .addReg(RegNo: 0);
1221
1222 // We are restoring R13 even though we never stored in setjmp from llvm,
1223 // as gcc always stores R13 in builtin_setjmp. We could have mixed code
1224 // gcc setjmp and llvm longjmp.
1225 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: SystemZ::R13D)
1226 .addReg(RegNo: BufReg)
1227 .addImm(Val: LPOffset)
1228 .addReg(RegNo: 0);
1229
1230 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1231 if (BackChain) {
1232 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1233 .addReg(RegNo: BufReg)
1234 .addImm(Val: BCOffset)
1235 .addReg(RegNo: 0);
1236 }
1237
1238 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1239 DestReg: SpecialRegs->getStackPointerRegister())
1240 .addReg(RegNo: BufReg)
1241 .addImm(Val: SPOffset)
1242 .addReg(RegNo: 0);
1243
1244 if (BackChain) {
1245 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1246 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1247 .addReg(RegNo: BCReg)
1248 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1249 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1250 .addReg(RegNo: 0);
1251 }
1252
1253 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BR)).addReg(RegNo: Tmp);
1254
1255 MI.eraseFromParent();
1256 return MBB;
1257}
1258
1259/// Returns true if stack probing through inline assembly is requested.
1260bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
1261 // If the function specifically requests inline stack probes, emit them.
1262 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
1263 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
1264 "inline-asm";
1265 return false;
1266}
1267
1268TargetLowering::AtomicExpansionKind
1269SystemZTargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
1270 return AtomicExpansionKind::None;
1271}
1272
1273TargetLowering::AtomicExpansionKind
1274SystemZTargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
1275 return AtomicExpansionKind::None;
1276}
1277
1278TargetLowering::AtomicExpansionKind
1279SystemZTargetLowering::shouldExpandAtomicRMWInIR(
1280 const AtomicRMWInst *RMW) const {
1281 // Don't expand subword operations as they require special treatment.
1282 if (RMW->getType()->isIntegerTy(Bitwidth: 8) || RMW->getType()->isIntegerTy(Bitwidth: 16))
1283 return AtomicExpansionKind::None;
1284
1285 // Don't expand if there is a target instruction available.
1286 if (Subtarget.hasInterlockedAccess1() &&
1287 (RMW->getType()->isIntegerTy(Bitwidth: 32) || RMW->getType()->isIntegerTy(Bitwidth: 64)) &&
1288 (RMW->getOperation() == AtomicRMWInst::BinOp::Add ||
1289 RMW->getOperation() == AtomicRMWInst::BinOp::Sub ||
1290 RMW->getOperation() == AtomicRMWInst::BinOp::And ||
1291 RMW->getOperation() == AtomicRMWInst::BinOp::Or ||
1292 RMW->getOperation() == AtomicRMWInst::BinOp::Xor))
1293 return AtomicExpansionKind::None;
1294
1295 return AtomicExpansionKind::CmpXChg;
1296}
1297
1298bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1299 // We can use CGFI or CLGFI.
1300 return isInt<32>(x: Imm) || isUInt<32>(x: Imm);
1301}
1302
1303bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1304 // We can use ALGFI or SLGFI.
1305 return isUInt<32>(x: Imm) || isUInt<32>(x: -Imm);
1306}
1307
1308bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(
1309 EVT VT, unsigned, Align, MachineMemOperand::Flags, unsigned *Fast) const {
1310 // Unaligned accesses should never be slower than the expanded version.
1311 // We check specifically for aligned accesses in the few cases where
1312 // they are required.
1313 if (Fast)
1314 *Fast = 1;
1315 return true;
1316}
1317
1318bool SystemZTargetLowering::hasAndNot(SDValue Y) const {
1319 EVT VT = Y.getValueType();
1320
1321 // We can use NC(G)RK for types in GPRs ...
1322 if (VT == MVT::i32 || VT == MVT::i64)
1323 return Subtarget.hasMiscellaneousExtensions3();
1324
1325 // ... or VNC for types in VRs.
1326 if (VT.isVector() || VT == MVT::i128)
1327 return Subtarget.hasVector();
1328
1329 return false;
1330}
1331
1332// Information about the addressing mode for a memory access.
1333struct AddressingMode {
1334 // True if a long displacement is supported.
1335 bool LongDisplacement;
1336
1337 // True if use of index register is supported.
1338 bool IndexReg;
1339
1340 AddressingMode(bool LongDispl, bool IdxReg) :
1341 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
1342};
1343
1344// Return the desired addressing mode for a Load which has only one use (in
1345// the same block) which is a Store.
1346static AddressingMode getLoadStoreAddrMode(bool HasVector,
1347 Type *Ty) {
1348 // With vector support a Load->Store combination may be combined to either
1349 // an MVC or vector operations and it seems to work best to allow the
1350 // vector addressing mode.
1351 if (HasVector)
1352 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1353
1354 // Otherwise only the MVC case is special.
1355 bool MVC = Ty->isIntegerTy(Bitwidth: 8);
1356 return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/);
1357}
1358
1359// Return the addressing mode which seems most desirable given an LLVM
1360// Instruction pointer.
1361static AddressingMode
1362supportedAddressingMode(Instruction *I, bool HasVector) {
1363 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1364 switch (II->getIntrinsicID()) {
1365 default: break;
1366 case Intrinsic::memset:
1367 case Intrinsic::memmove:
1368 case Intrinsic::memcpy:
1369 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1370 }
1371 }
1372
1373 if (isa<LoadInst>(Val: I) && I->hasOneUse()) {
1374 auto *SingleUser = cast<Instruction>(Val: *I->user_begin());
1375 if (SingleUser->getParent() == I->getParent()) {
1376 if (isa<ICmpInst>(Val: SingleUser)) {
1377 if (auto *C = dyn_cast<ConstantInt>(Val: SingleUser->getOperand(i: 1)))
1378 if (C->getBitWidth() <= 64 &&
1379 (isInt<16>(x: C->getSExtValue()) || isUInt<16>(x: C->getZExtValue())))
1380 // Comparison of memory with 16 bit signed / unsigned immediate
1381 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1382 } else if (isa<StoreInst>(Val: SingleUser))
1383 // Load->Store
1384 return getLoadStoreAddrMode(HasVector, Ty: I->getType());
1385 }
1386 } else if (auto *StoreI = dyn_cast<StoreInst>(Val: I)) {
1387 if (auto *LoadI = dyn_cast<LoadInst>(Val: StoreI->getValueOperand()))
1388 if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())
1389 // Load->Store
1390 return getLoadStoreAddrMode(HasVector, Ty: LoadI->getType());
1391 }
1392
1393 if (HasVector && (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I))) {
1394
1395 // * Use LDE instead of LE/LEY for z13 to avoid partial register
1396 // dependencies (LDE only supports small offsets).
1397 // * Utilize the vector registers to hold floating point
1398 // values (vector load / store instructions only support small
1399 // offsets).
1400
1401 Type *MemAccessTy = (isa<LoadInst>(Val: I) ? I->getType() :
1402 I->getOperand(i: 0)->getType());
1403 bool IsFPAccess = MemAccessTy->isFloatingPointTy();
1404 bool IsVectorAccess = MemAccessTy->isVectorTy();
1405
1406 // A store of an extracted vector element will be combined into a VSTE type
1407 // instruction.
1408 if (!IsVectorAccess && isa<StoreInst>(Val: I)) {
1409 Value *DataOp = I->getOperand(i: 0);
1410 if (isa<ExtractElementInst>(Val: DataOp))
1411 IsVectorAccess = true;
1412 }
1413
1414 // A load which gets inserted into a vector element will be combined into a
1415 // VLE type instruction.
1416 if (!IsVectorAccess && isa<LoadInst>(Val: I) && I->hasOneUse()) {
1417 User *LoadUser = *I->user_begin();
1418 if (isa<InsertElementInst>(Val: LoadUser))
1419 IsVectorAccess = true;
1420 }
1421
1422 if (IsFPAccess || IsVectorAccess)
1423 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1424 }
1425
1426 return AddressingMode(true/*LongDispl*/, true/*IdxReg*/);
1427}
1428
1429bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1430 const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const {
1431 // Punt on globals for now, although they can be used in limited
1432 // RELATIVE LONG cases.
1433 if (AM.BaseGV)
1434 return false;
1435
1436 // Require a 20-bit signed offset.
1437 if (!isInt<20>(x: AM.BaseOffs))
1438 return false;
1439
1440 bool RequireD12 =
1441 Subtarget.hasVector() && (Ty->isVectorTy() || Ty->isIntegerTy(Bitwidth: 128));
1442 AddressingMode SupportedAM(!RequireD12, true);
1443 if (I != nullptr)
1444 SupportedAM = supportedAddressingMode(I, HasVector: Subtarget.hasVector());
1445
1446 if (!SupportedAM.LongDisplacement && !isUInt<12>(x: AM.BaseOffs))
1447 return false;
1448
1449 if (!SupportedAM.IndexReg)
1450 // No indexing allowed.
1451 return AM.Scale == 0;
1452 else
1453 // Indexing is OK but no scale factor can be applied.
1454 return AM.Scale == 0 || AM.Scale == 1;
1455}
1456
1457bool SystemZTargetLowering::findOptimalMemOpLowering(
1458 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
1459 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
1460 const AttributeList &FuncAttributes, EVT *LargestVT) const {
1461 const int MVCFastLen = 16;
1462
1463 if (Limit != ~unsigned(0)) {
1464 // Don't expand Op into scalar loads/stores in these cases:
1465 if (Op.isMemcpy() && Op.allowOverlap() && Op.size() <= MVCFastLen)
1466 return false; // Small memcpy: Use MVC
1467 if (Op.isMemset() && Op.size() - 1 <= MVCFastLen)
1468 return false; // Small memset (first byte with STC/MVI): Use MVC
1469 if (Op.isZeroMemset())
1470 return false; // Memset zero: Use XC
1471 }
1472
1473 return TargetLowering::findOptimalMemOpLowering(
1474 Context, MemOps, Limit, Op, DstAS, SrcAS, FuncAttributes, LargestVT);
1475}
1476
1477EVT SystemZTargetLowering::getOptimalMemOpType(
1478 LLVMContext &Context, const MemOp &Op,
1479 const AttributeList &FuncAttributes) const {
1480 return Subtarget.hasVector() ? MVT::v2i64 : MVT::Other;
1481}
1482
1483bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const {
1484 if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
1485 return false;
1486 unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();
1487 unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedValue();
1488 return FromBits > ToBits;
1489}
1490
1491bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const {
1492 if (!FromVT.isInteger() || !ToVT.isInteger())
1493 return false;
1494 unsigned FromBits = FromVT.getFixedSizeInBits();
1495 unsigned ToBits = ToVT.getFixedSizeInBits();
1496 return FromBits > ToBits;
1497}
1498
1499//===----------------------------------------------------------------------===//
1500// Inline asm support
1501//===----------------------------------------------------------------------===//
1502
1503TargetLowering::ConstraintType
1504SystemZTargetLowering::getConstraintType(StringRef Constraint) const {
1505 if (Constraint.size() == 1) {
1506 switch (Constraint[0]) {
1507 case 'a': // Address register
1508 case 'd': // Data register (equivalent to 'r')
1509 case 'f': // Floating-point register
1510 case 'h': // High-part register
1511 case 'r': // General-purpose register
1512 case 'v': // Vector register
1513 return C_RegisterClass;
1514
1515 case 'Q': // Memory with base and unsigned 12-bit displacement
1516 case 'R': // Likewise, plus an index
1517 case 'S': // Memory with base and signed 20-bit displacement
1518 case 'T': // Likewise, plus an index
1519 case 'm': // Equivalent to 'T'.
1520 return C_Memory;
1521
1522 case 'I': // Unsigned 8-bit constant
1523 case 'J': // Unsigned 12-bit constant
1524 case 'K': // Signed 16-bit constant
1525 case 'L': // Signed 20-bit displacement (on all targets we support)
1526 case 'M': // 0x7fffffff
1527 return C_Immediate;
1528
1529 default:
1530 break;
1531 }
1532 } else if (Constraint.size() == 2 && Constraint[0] == 'Z') {
1533 switch (Constraint[1]) {
1534 case 'Q': // Address with base and unsigned 12-bit displacement
1535 case 'R': // Likewise, plus an index
1536 case 'S': // Address with base and signed 20-bit displacement
1537 case 'T': // Likewise, plus an index
1538 return C_Address;
1539
1540 default:
1541 break;
1542 }
1543 } else if (Constraint.size() == 5 && Constraint.starts_with(Prefix: "{")) {
1544 if (StringRef("{@cc}").compare(RHS: Constraint) == 0)
1545 return C_Other;
1546 }
1547 return TargetLowering::getConstraintType(Constraint);
1548}
1549
1550TargetLowering::ConstraintWeight
1551SystemZTargetLowering::getSingleConstraintMatchWeight(
1552 AsmOperandInfo &Info, const char *Constraint) const {
1553 ConstraintWeight Weight = CW_Invalid;
1554 Value *CallOperandVal = Info.CallOperandVal;
1555 // If we don't have a value, we can't do a match,
1556 // but allow it at the lowest weight.
1557 if (!CallOperandVal)
1558 return CW_Default;
1559 Type *type = CallOperandVal->getType();
1560 // Look at the constraint type.
1561 switch (*Constraint) {
1562 default:
1563 Weight = TargetLowering::getSingleConstraintMatchWeight(info&: Info, constraint: Constraint);
1564 break;
1565
1566 case 'a': // Address register
1567 case 'd': // Data register (equivalent to 'r')
1568 case 'h': // High-part register
1569 case 'r': // General-purpose register
1570 Weight =
1571 CallOperandVal->getType()->isIntegerTy() ? CW_Register : CW_Default;
1572 break;
1573
1574 case 'f': // Floating-point register
1575 if (!useSoftFloat())
1576 Weight = type->isFloatingPointTy() ? CW_Register : CW_Default;
1577 break;
1578
1579 case 'v': // Vector register
1580 if (Subtarget.hasVector())
1581 Weight = (type->isVectorTy() || type->isFloatingPointTy()) ? CW_Register
1582 : CW_Default;
1583 break;
1584
1585 case 'I': // Unsigned 8-bit constant
1586 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1587 if (isUInt<8>(x: C->getZExtValue()))
1588 Weight = CW_Constant;
1589 break;
1590
1591 case 'J': // Unsigned 12-bit constant
1592 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1593 if (isUInt<12>(x: C->getZExtValue()))
1594 Weight = CW_Constant;
1595 break;
1596
1597 case 'K': // Signed 16-bit constant
1598 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1599 if (isInt<16>(x: C->getSExtValue()))
1600 Weight = CW_Constant;
1601 break;
1602
1603 case 'L': // Signed 20-bit displacement (on all targets we support)
1604 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1605 if (isInt<20>(x: C->getSExtValue()))
1606 Weight = CW_Constant;
1607 break;
1608
1609 case 'M': // 0x7fffffff
1610 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1611 if (C->getZExtValue() == 0x7fffffff)
1612 Weight = CW_Constant;
1613 break;
1614 }
1615 return Weight;
1616}
1617
1618// Parse a "{tNNN}" register constraint for which the register type "t"
1619// has already been verified. MC is the class associated with "t" and
1620// Map maps 0-based register numbers to LLVM register numbers.
1621static std::pair<unsigned, const TargetRegisterClass *>
1622parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC,
1623 const unsigned *Map, unsigned Size) {
1624 assert(*(Constraint.end()-1) == '}' && "Missing '}'");
1625 if (isdigit(Constraint[2])) {
1626 unsigned Index;
1627 bool Failed =
1628 Constraint.slice(Start: 2, End: Constraint.size() - 1).getAsInteger(Radix: 10, Result&: Index);
1629 if (!Failed && Index < Size && Map[Index])
1630 return std::make_pair(x: Map[Index], y&: RC);
1631 }
1632 return std::make_pair(x: 0U, y: nullptr);
1633}
1634
1635std::pair<unsigned, const TargetRegisterClass *>
1636SystemZTargetLowering::getRegForInlineAsmConstraint(
1637 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
1638 if (Constraint.size() == 1) {
1639 // GCC Constraint Letters
1640 switch (Constraint[0]) {
1641 default: break;
1642 case 'd': // Data register (equivalent to 'r')
1643 case 'r': // General-purpose register
1644 if (VT.getSizeInBits() == 64)
1645 return std::make_pair(x: 0U, y: &SystemZ::GR64BitRegClass);
1646 else if (VT.getSizeInBits() == 128)
1647 return std::make_pair(x: 0U, y: &SystemZ::GR128BitRegClass);
1648 return std::make_pair(x: 0U, y: &SystemZ::GR32BitRegClass);
1649
1650 case 'a': // Address register
1651 if (VT == MVT::i64)
1652 return std::make_pair(x: 0U, y: &SystemZ::ADDR64BitRegClass);
1653 else if (VT == MVT::i128)
1654 return std::make_pair(x: 0U, y: &SystemZ::ADDR128BitRegClass);
1655 return std::make_pair(x: 0U, y: &SystemZ::ADDR32BitRegClass);
1656
1657 case 'h': // High-part register (an LLVM extension)
1658 return std::make_pair(x: 0U, y: &SystemZ::GRH32BitRegClass);
1659
1660 case 'f': // Floating-point register
1661 if (!useSoftFloat()) {
1662 if (VT.getSizeInBits() == 16)
1663 return std::make_pair(x: 0U, y: &SystemZ::FP16BitRegClass);
1664 else if (VT.getSizeInBits() == 64)
1665 return std::make_pair(x: 0U, y: &SystemZ::FP64BitRegClass);
1666 else if (VT.getSizeInBits() == 128)
1667 return std::make_pair(x: 0U, y: &SystemZ::FP128BitRegClass);
1668 return std::make_pair(x: 0U, y: &SystemZ::FP32BitRegClass);
1669 }
1670 break;
1671
1672 case 'v': // Vector register
1673 if (Subtarget.hasVector()) {
1674 if (VT.getSizeInBits() == 16)
1675 return std::make_pair(x: 0U, y: &SystemZ::VR16BitRegClass);
1676 if (VT.getSizeInBits() == 32)
1677 return std::make_pair(x: 0U, y: &SystemZ::VR32BitRegClass);
1678 if (VT.getSizeInBits() == 64)
1679 return std::make_pair(x: 0U, y: &SystemZ::VR64BitRegClass);
1680 return std::make_pair(x: 0U, y: &SystemZ::VR128BitRegClass);
1681 }
1682 break;
1683 }
1684 }
1685 if (Constraint.starts_with(Prefix: "{")) {
1686
1687 // A clobber constraint (e.g. ~{f0}) will have MVT::Other which is illegal
1688 // to check the size on.
1689 auto getVTSizeInBits = [&VT]() {
1690 return VT == MVT::Other ? 0 : VT.getSizeInBits();
1691 };
1692
1693 // We need to override the default register parsing for GPRs and FPRs
1694 // because the interpretation depends on VT. The internal names of
1695 // the registers are also different from the external names
1696 // (F0D and F0S instead of F0, etc.).
1697 if (Constraint[1] == 'r') {
1698 if (getVTSizeInBits() == 32)
1699 return parseRegisterNumber(Constraint, RC: &SystemZ::GR32BitRegClass,
1700 Map: SystemZMC::GR32Regs, Size: 16);
1701 if (getVTSizeInBits() == 128)
1702 return parseRegisterNumber(Constraint, RC: &SystemZ::GR128BitRegClass,
1703 Map: SystemZMC::GR128Regs, Size: 16);
1704 return parseRegisterNumber(Constraint, RC: &SystemZ::GR64BitRegClass,
1705 Map: SystemZMC::GR64Regs, Size: 16);
1706 }
1707 if (Constraint[1] == 'f') {
1708 if (useSoftFloat())
1709 return std::make_pair(
1710 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1711 if (getVTSizeInBits() == 16)
1712 return parseRegisterNumber(Constraint, RC: &SystemZ::FP16BitRegClass,
1713 Map: SystemZMC::FP16Regs, Size: 16);
1714 if (getVTSizeInBits() == 32)
1715 return parseRegisterNumber(Constraint, RC: &SystemZ::FP32BitRegClass,
1716 Map: SystemZMC::FP32Regs, Size: 16);
1717 if (getVTSizeInBits() == 128)
1718 return parseRegisterNumber(Constraint, RC: &SystemZ::FP128BitRegClass,
1719 Map: SystemZMC::FP128Regs, Size: 16);
1720 return parseRegisterNumber(Constraint, RC: &SystemZ::FP64BitRegClass,
1721 Map: SystemZMC::FP64Regs, Size: 16);
1722 }
1723 if (Constraint[1] == 'v') {
1724 if (!Subtarget.hasVector())
1725 return std::make_pair(
1726 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1727 if (getVTSizeInBits() == 16)
1728 return parseRegisterNumber(Constraint, RC: &SystemZ::VR16BitRegClass,
1729 Map: SystemZMC::VR16Regs, Size: 32);
1730 if (getVTSizeInBits() == 32)
1731 return parseRegisterNumber(Constraint, RC: &SystemZ::VR32BitRegClass,
1732 Map: SystemZMC::VR32Regs, Size: 32);
1733 if (getVTSizeInBits() == 64)
1734 return parseRegisterNumber(Constraint, RC: &SystemZ::VR64BitRegClass,
1735 Map: SystemZMC::VR64Regs, Size: 32);
1736 return parseRegisterNumber(Constraint, RC: &SystemZ::VR128BitRegClass,
1737 Map: SystemZMC::VR128Regs, Size: 32);
1738 }
1739 if (Constraint[1] == '@') {
1740 if (StringRef("{@cc}").compare(RHS: Constraint) == 0)
1741 return std::make_pair(x: SystemZ::CC, y: &SystemZ::CCRRegClass);
1742 }
1743 }
1744 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
1745}
1746
1747// FIXME? Maybe this could be a TableGen attribute on some registers and
1748// this table could be generated automatically from RegInfo.
1749Register
1750SystemZTargetLowering::getRegisterByName(const char *RegName, LLT VT,
1751 const MachineFunction &MF) const {
1752 Register Reg =
1753 StringSwitch<Register>(RegName)
1754 .Case(S: "r4", Value: Subtarget.isTargetXPLINK64() ? SystemZ::R4D
1755 : SystemZ::NoRegister)
1756 .Case(S: "r15",
1757 Value: Subtarget.isTargetELF() ? SystemZ::R15D : SystemZ::NoRegister)
1758 .Default(Value: Register());
1759
1760 return Reg;
1761}
1762
1763Register SystemZTargetLowering::getExceptionPointerRegister(
1764 const Constant *PersonalityFn) const {
1765 return Subtarget.isTargetXPLINK64() ? SystemZ::R1D : SystemZ::R6D;
1766}
1767
1768Register SystemZTargetLowering::getExceptionSelectorRegister(
1769 const Constant *PersonalityFn) const {
1770 return Subtarget.isTargetXPLINK64() ? SystemZ::R2D : SystemZ::R7D;
1771}
1772
1773// Convert condition code in CCReg to an i32 value.
1774static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg) {
1775 SDLoc DL(CCReg);
1776 SDValue IPM = DAG.getNode(Opcode: SystemZISD::IPM, DL, VT: MVT::i32, Operand: CCReg);
1777 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: IPM,
1778 N2: DAG.getConstant(Val: SystemZ::IPM_CC, DL, VT: MVT::i32));
1779}
1780
1781// Lower @cc targets via setcc.
1782SDValue SystemZTargetLowering::LowerAsmOutputForConstraint(
1783 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
1784 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
1785 if (StringRef("{@cc}").compare(RHS: OpInfo.ConstraintCode) != 0)
1786 return SDValue();
1787
1788 // Check that return type is valid.
1789 if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
1790 OpInfo.ConstraintVT.getSizeInBits() < 8)
1791 report_fatal_error(reason: "Glue output operand is of invalid type");
1792
1793 if (Glue.getNode()) {
1794 Glue = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::CC, VT: MVT::i32, Glue);
1795 Chain = Glue.getValue(R: 1);
1796 } else
1797 Glue = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::CC, VT: MVT::i32);
1798 return getCCResult(DAG, CCReg: Glue);
1799}
1800
1801void SystemZTargetLowering::LowerAsmOperandForConstraint(
1802 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
1803 SelectionDAG &DAG) const {
1804 // Only support length 1 constraints for now.
1805 if (Constraint.size() == 1) {
1806 switch (Constraint[0]) {
1807 case 'I': // Unsigned 8-bit constant
1808 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1809 if (isUInt<8>(x: C->getZExtValue()))
1810 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1811 VT: Op.getValueType()));
1812 return;
1813
1814 case 'J': // Unsigned 12-bit constant
1815 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1816 if (isUInt<12>(x: C->getZExtValue()))
1817 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1818 VT: Op.getValueType()));
1819 return;
1820
1821 case 'K': // Signed 16-bit constant
1822 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1823 if (isInt<16>(x: C->getSExtValue()))
1824 Ops.push_back(x: DAG.getSignedTargetConstant(
1825 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1826 return;
1827
1828 case 'L': // Signed 20-bit displacement (on all targets we support)
1829 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1830 if (isInt<20>(x: C->getSExtValue()))
1831 Ops.push_back(x: DAG.getSignedTargetConstant(
1832 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1833 return;
1834
1835 case 'M': // 0x7fffffff
1836 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1837 if (C->getZExtValue() == 0x7fffffff)
1838 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1839 VT: Op.getValueType()));
1840 return;
1841 }
1842 }
1843 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
1844}
1845
1846//===----------------------------------------------------------------------===//
1847// Calling conventions
1848//===----------------------------------------------------------------------===//
1849
1850#include "SystemZGenCallingConv.inc"
1851
1852const MCPhysReg *SystemZTargetLowering::getScratchRegisters(
1853 CallingConv::ID) const {
1854 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1855 SystemZ::R14D, 0 };
1856 return ScratchRegs;
1857}
1858
1859bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType,
1860 Type *ToType) const {
1861 return isTruncateFree(FromType, ToType);
1862}
1863
1864bool SystemZTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
1865 return CI->isTailCall();
1866}
1867
1868// Value is a value that has been passed to us in the location described by VA
1869// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining
1870// any loads onto Chain.
1871static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
1872 CCValAssign &VA, SDValue Chain,
1873 SDValue Value) {
1874 // If the argument has been promoted from a smaller type, insert an
1875 // assertion to capture this.
1876 if (VA.getLocInfo() == CCValAssign::SExt)
1877 Value = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Value,
1878 N2: DAG.getValueType(VA.getValVT()));
1879 else if (VA.getLocInfo() == CCValAssign::ZExt)
1880 Value = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Value,
1881 N2: DAG.getValueType(VA.getValVT()));
1882
1883 if (VA.isExtInLoc())
1884 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Value);
1885 else if (VA.getLocInfo() == CCValAssign::BCvt) {
1886 // If this is a short vector argument loaded from the stack,
1887 // extend from i64 to full vector size and then bitcast.
1888 assert(VA.getLocVT() == MVT::i64);
1889 assert(VA.getValVT().isVector());
1890 Value = DAG.getBuildVector(VT: MVT::v2i64, DL, Ops: {Value, DAG.getUNDEF(VT: MVT::i64)});
1891 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Value);
1892 } else
1893 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo");
1894 return Value;
1895}
1896
1897// Value is a value of type VA.getValVT() that we need to copy into
1898// the location described by VA. Return a copy of Value converted to
1899// VA.getValVT(). The caller is responsible for handling indirect values.
1900static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
1901 CCValAssign &VA, SDValue Value) {
1902 switch (VA.getLocInfo()) {
1903 case CCValAssign::SExt:
1904 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1905 case CCValAssign::ZExt:
1906 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1907 case CCValAssign::AExt:
1908 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1909 case CCValAssign::BCvt: {
1910 assert(VA.getLocVT() == MVT::i64 || VA.getLocVT() == MVT::i128);
1911 assert(VA.getValVT().isVector() || VA.getValVT() == MVT::f32 ||
1912 VA.getValVT() == MVT::f64 || VA.getValVT() == MVT::f128);
1913 // For an f32 vararg we need to first promote it to an f64 and then
1914 // bitcast it to an i64.
1915 if (VA.getValVT() == MVT::f32 && VA.getLocVT() == MVT::i64)
1916 Value = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f64, Operand: Value);
1917 MVT BitCastToType = VA.getValVT().isVector() && VA.getLocVT() == MVT::i64
1918 ? MVT::v2i64
1919 : VA.getLocVT();
1920 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: BitCastToType, Operand: Value);
1921 // For ELF, this is a short vector argument to be stored to the stack,
1922 // bitcast to v2i64 and then extract first element.
1923 if (BitCastToType == MVT::v2i64)
1924 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: VA.getLocVT(), N1: Value,
1925 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
1926 return Value;
1927 }
1928 case CCValAssign::Full:
1929 return Value;
1930 default:
1931 llvm_unreachable("Unhandled getLocInfo()");
1932 }
1933}
1934
1935static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
1936 SDLoc DL(In);
1937 SDValue Lo, Hi;
1938 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
1939 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64, Operand: In);
1940 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64,
1941 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i128, N1: In,
1942 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32)));
1943 } else {
1944 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: In, DL, LoVT: MVT::i64, HiVT: MVT::i64);
1945 }
1946
1947 // FIXME: If v2i64 were a legal type, we could use it instead of
1948 // Untyped here. This might enable improved folding.
1949 SDNode *Pair = DAG.getMachineNode(Opcode: SystemZ::PAIR128, dl: DL,
1950 VT: MVT::Untyped, Op1: Hi, Op2: Lo);
1951 return SDValue(Pair, 0);
1952}
1953
1954static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
1955 SDLoc DL(In);
1956 SDValue Hi = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64,
1957 DL, VT: MVT::i64, Operand: In);
1958 SDValue Lo = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64,
1959 DL, VT: MVT::i64, Operand: In);
1960
1961 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
1962 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Lo);
1963 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Hi);
1964 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i128, N1: Hi,
1965 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32));
1966 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
1967 } else {
1968 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
1969 }
1970}
1971
1972bool SystemZTargetLowering::splitValueIntoRegisterParts(
1973 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
1974 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
1975 EVT ValueVT = Val.getValueType();
1976 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
1977 // Inline assembly operand.
1978 Parts[0] = lowerI128ToGR128(DAG, In: DAG.getBitcast(VT: MVT::i128, V: Val));
1979 return true;
1980 }
1981
1982 return false;
1983}
1984
1985SDValue SystemZTargetLowering::joinRegisterPartsIntoValue(
1986 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
1987 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
1988 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
1989 // Inline assembly operand.
1990 SDValue Res = lowerGR128ToI128(DAG, In: Parts[0]);
1991 return DAG.getBitcast(VT: ValueVT, V: Res);
1992 }
1993
1994 return SDValue();
1995}
1996
1997// The first part of a split stack argument is at index I in Args (and
1998// ArgLocs). Return the type of a part and the number of them by reference.
1999template <class ArgTy>
2000static bool analyzeArgSplit(const SmallVectorImpl<ArgTy> &Args,
2001 SmallVector<CCValAssign, 16> &ArgLocs, unsigned I,
2002 MVT &PartVT, unsigned &NumParts) {
2003 if (!Args[I].Flags.isSplit())
2004 return false;
2005 assert(I < ArgLocs.size() && ArgLocs.size() == Args.size() &&
2006 "ArgLocs havoc.");
2007 PartVT = ArgLocs[I].getValVT();
2008 NumParts = 1;
2009 for (unsigned PartIdx = I + 1;; ++PartIdx) {
2010 assert(PartIdx != ArgLocs.size() && "SplitEnd not found.");
2011 assert(ArgLocs[PartIdx].getValVT() == PartVT && "Unsupported split.");
2012 ++NumParts;
2013 if (Args[PartIdx].Flags.isSplitEnd())
2014 break;
2015 }
2016 return true;
2017}
2018
2019SDValue SystemZTargetLowering::LowerFormalArguments(
2020 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
2021 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2022 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2023 MachineFunction &MF = DAG.getMachineFunction();
2024 MachineFrameInfo &MFI = MF.getFrameInfo();
2025 MachineRegisterInfo &MRI = MF.getRegInfo();
2026 SystemZMachineFunctionInfo *FuncInfo =
2027 MF.getInfo<SystemZMachineFunctionInfo>();
2028 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
2029 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
2030
2031 // Assign locations to all of the incoming arguments.
2032 SmallVector<CCValAssign, 16> ArgLocs;
2033 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2034 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_SystemZ);
2035 FuncInfo->setSizeOfFnParams(CCInfo.getStackSize());
2036
2037 unsigned NumFixedGPRs = 0;
2038 unsigned NumFixedFPRs = 0;
2039 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2040 SDValue ArgValue;
2041 CCValAssign &VA = ArgLocs[I];
2042 EVT LocVT = VA.getLocVT();
2043 if (VA.isRegLoc()) {
2044 // Arguments passed in registers
2045 const TargetRegisterClass *RC;
2046 switch (LocVT.getSimpleVT().SimpleTy) {
2047 default:
2048 // Integers smaller than i64 should be promoted to i64.
2049 llvm_unreachable("Unexpected argument type");
2050 case MVT::i32:
2051 NumFixedGPRs += 1;
2052 RC = &SystemZ::GR32BitRegClass;
2053 break;
2054 case MVT::i64:
2055 NumFixedGPRs += 1;
2056 RC = &SystemZ::GR64BitRegClass;
2057 break;
2058 case MVT::f16:
2059 NumFixedFPRs += 1;
2060 RC = &SystemZ::FP16BitRegClass;
2061 break;
2062 case MVT::f32:
2063 NumFixedFPRs += 1;
2064 RC = &SystemZ::FP32BitRegClass;
2065 break;
2066 case MVT::f64:
2067 NumFixedFPRs += 1;
2068 RC = &SystemZ::FP64BitRegClass;
2069 break;
2070 case MVT::f128:
2071 NumFixedFPRs += 2;
2072 RC = &SystemZ::FP128BitRegClass;
2073 break;
2074 case MVT::v16i8:
2075 case MVT::v8i16:
2076 case MVT::v4i32:
2077 case MVT::v2i64:
2078 case MVT::v8f16:
2079 case MVT::v4f32:
2080 case MVT::v2f64:
2081 RC = &SystemZ::VR128BitRegClass;
2082 break;
2083 }
2084
2085 Register VReg = MRI.createVirtualRegister(RegClass: RC);
2086 MRI.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
2087 ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
2088 } else {
2089 assert(VA.isMemLoc() && "Argument not register or memory");
2090
2091 // Create the frame index object for this incoming parameter.
2092 // FIXME: Pre-include call frame size in the offset, should not
2093 // need to manually add it here.
2094 int64_t ArgSPOffset = VA.getLocMemOffset();
2095 if (Subtarget.isTargetXPLINK64()) {
2096 auto &XPRegs =
2097 Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
2098 ArgSPOffset += XPRegs.getCallFrameSize();
2099 }
2100 int FI =
2101 MFI.CreateFixedObject(Size: LocVT.getSizeInBits() / 8, SPOffset: ArgSPOffset, IsImmutable: true);
2102
2103 // Create the SelectionDAG nodes corresponding to a load
2104 // from this parameter. Unpromoted ints and floats are
2105 // passed as right-justified 8-byte values.
2106 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
2107 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32 ||
2108 VA.getLocVT() == MVT::f16) {
2109 unsigned SlotOffs = VA.getLocVT() == MVT::f16 ? 6 : 4;
2110 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FIN,
2111 N2: DAG.getIntPtrConstant(Val: SlotOffs, DL));
2112 }
2113 ArgValue = DAG.getLoad(VT: LocVT, dl: DL, Chain, Ptr: FIN,
2114 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2115 }
2116
2117 // Convert the value of the argument register into the value that's
2118 // being passed.
2119 if (VA.getLocInfo() == CCValAssign::Indirect) {
2120 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
2121 PtrInfo: MachinePointerInfo()));
2122 // If the original argument was split (e.g. i128), we need
2123 // to load all parts of it here (using the same address).
2124 MVT PartVT;
2125 unsigned NumParts;
2126 if (analyzeArgSplit(Args: Ins, ArgLocs, I, PartVT, NumParts)) {
2127 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2128 ++I;
2129 CCValAssign &PartVA = ArgLocs[I];
2130 unsigned PartOffset = Ins[I].PartOffset;
2131 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue,
2132 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2133 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
2134 PtrInfo: MachinePointerInfo()));
2135 assert(PartOffset && "Offset should be non-zero.");
2136 }
2137 }
2138 } else
2139 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: ArgValue));
2140 }
2141
2142 if (IsVarArg && Subtarget.isTargetXPLINK64()) {
2143 // Save the number of non-varargs registers for later use by va_start, etc.
2144 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2145 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2146
2147 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2148 Subtarget.getSpecialRegisters());
2149
2150 // Likewise the address (in the form of a frame index) of where the
2151 // first stack vararg would be. The 1-byte size here is arbitrary.
2152 // FIXME: Pre-include call frame size in the offset, should not
2153 // need to manually add it here.
2154 int64_t VarArgOffset = CCInfo.getStackSize() + Regs->getCallFrameSize();
2155 int FI = MFI.CreateFixedObject(Size: 1, SPOffset: VarArgOffset, IsImmutable: true);
2156 FuncInfo->setVarArgsFrameIndex(FI);
2157 }
2158
2159 if (IsVarArg && Subtarget.isTargetELF()) {
2160 // Save the number of non-varargs registers for later use by va_start, etc.
2161 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2162 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2163
2164 // Likewise the address (in the form of a frame index) of where the
2165 // first stack vararg would be. The 1-byte size here is arbitrary.
2166 int64_t VarArgsOffset = CCInfo.getStackSize();
2167 FuncInfo->setVarArgsFrameIndex(
2168 MFI.CreateFixedObject(Size: 1, SPOffset: VarArgsOffset, IsImmutable: true));
2169
2170 // ...and a similar frame index for the caller-allocated save area
2171 // that will be used to store the incoming registers.
2172 int64_t RegSaveOffset =
2173 -SystemZMC::ELFCallFrameSize + TFL->getRegSpillOffset(MF, Reg: SystemZ::R2D) - 16;
2174 unsigned RegSaveIndex = MFI.CreateFixedObject(Size: 1, SPOffset: RegSaveOffset, IsImmutable: true);
2175 FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
2176
2177 // Store the FPR varargs in the reserved frame slots. (We store the
2178 // GPRs as part of the prologue.)
2179 if (NumFixedFPRs < SystemZ::ELFNumArgFPRs && !useSoftFloat()) {
2180 SDValue MemOps[SystemZ::ELFNumArgFPRs];
2181 for (unsigned I = NumFixedFPRs; I < SystemZ::ELFNumArgFPRs; ++I) {
2182 unsigned Offset = TFL->getRegSpillOffset(MF, Reg: SystemZ::ELFArgFPRs[I]);
2183 int FI =
2184 MFI.CreateFixedObject(Size: 8, SPOffset: -SystemZMC::ELFCallFrameSize + Offset, IsImmutable: true);
2185 SDValue FIN = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
2186 Register VReg = MF.addLiveIn(PReg: SystemZ::ELFArgFPRs[I],
2187 RC: &SystemZ::FP64BitRegClass);
2188 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::f64);
2189 MemOps[I] = DAG.getStore(Chain: ArgValue.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: FIN,
2190 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2191 }
2192 // Join the stores, which are independent of one another.
2193 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
2194 Ops: ArrayRef(&MemOps[NumFixedFPRs],
2195 SystemZ::ELFNumArgFPRs - NumFixedFPRs));
2196 }
2197 }
2198
2199 if (Subtarget.isTargetXPLINK64()) {
2200 // Create virual register for handling incoming "ADA" special register (R5)
2201 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
2202 Register ADAvReg = MRI.createVirtualRegister(RegClass: RC);
2203 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2204 Subtarget.getSpecialRegisters());
2205 MRI.addLiveIn(Reg: Regs->getADARegister(), vreg: ADAvReg);
2206 FuncInfo->setADAVirtualRegister(ADAvReg);
2207 }
2208 return Chain;
2209}
2210
2211static bool canUseSiblingCall(const CCState &ArgCCInfo,
2212 SmallVectorImpl<CCValAssign> &ArgLocs,
2213 SmallVectorImpl<ISD::OutputArg> &Outs) {
2214 // Punt if there are any indirect or stack arguments, or if the call
2215 // needs the callee-saved argument register R6, or if the call uses
2216 // the callee-saved register arguments SwiftSelf and SwiftError.
2217 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2218 CCValAssign &VA = ArgLocs[I];
2219 if (VA.getLocInfo() == CCValAssign::Indirect)
2220 return false;
2221 if (!VA.isRegLoc())
2222 return false;
2223 Register Reg = VA.getLocReg();
2224 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
2225 return false;
2226 if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError())
2227 return false;
2228 }
2229 return true;
2230}
2231
2232static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL,
2233 unsigned Offset, bool LoadAdr = false) {
2234 MachineFunction &MF = DAG.getMachineFunction();
2235 SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
2236 Register ADAvReg = MFI->getADAVirtualRegister();
2237 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2238
2239 SDValue Reg = DAG.getRegister(Reg: ADAvReg, VT: PtrVT);
2240 SDValue Ofs = DAG.getTargetConstant(Val: Offset, DL, VT: PtrVT);
2241
2242 SDValue Result = DAG.getNode(Opcode: SystemZISD::ADA_ENTRY, DL, VT: PtrVT, N1: Val, N2: Reg, N3: Ofs);
2243 if (!LoadAdr)
2244 Result = DAG.getLoad(
2245 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result, PtrInfo: MachinePointerInfo(), Alignment: Align(8),
2246 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2247
2248 return Result;
2249}
2250
2251// ADA access using Global value
2252// Note: for functions, address of descriptor is returned
2253static SDValue getADAEntry(SelectionDAG &DAG, const GlobalValue *GV, SDLoc DL,
2254 EVT PtrVT) {
2255 unsigned ADAtype;
2256 bool LoadAddr = false;
2257 const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: GV);
2258 bool IsFunction =
2259 (isa<Function>(Val: GV)) || (GA && isa<Function>(Val: GA->getAliaseeObject()));
2260 bool IsInternal = (GV->hasInternalLinkage() || GV->hasPrivateLinkage());
2261
2262 if (IsFunction) {
2263 if (IsInternal) {
2264 ADAtype = SystemZII::MO_ADA_DIRECT_FUNC_DESC;
2265 LoadAddr = true;
2266 } else
2267 ADAtype = SystemZII::MO_ADA_INDIRECT_FUNC_DESC;
2268 } else {
2269 ADAtype = SystemZII::MO_ADA_DATA_SYMBOL_ADDR;
2270 }
2271 SDValue Val = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: ADAtype);
2272
2273 return getADAEntry(DAG, Val, DL, Offset: 0, LoadAdr: LoadAddr);
2274}
2275
2276static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA,
2277 SDLoc &DL, SDValue &Chain) {
2278 unsigned ADADelta = 0; // ADA offset in desc.
2279 unsigned EPADelta = 8; // EPA offset in desc.
2280 MachineFunction &MF = DAG.getMachineFunction();
2281 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2282
2283 // XPLink calling convention.
2284 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2285 bool IsInternal = (G->getGlobal()->hasInternalLinkage() ||
2286 G->getGlobal()->hasPrivateLinkage());
2287 if (IsInternal) {
2288 SystemZMachineFunctionInfo *MFI =
2289 MF.getInfo<SystemZMachineFunctionInfo>();
2290 Register ADAvReg = MFI->getADAVirtualRegister();
2291 ADA = DAG.getCopyFromReg(Chain, dl: DL, Reg: ADAvReg, VT: PtrVT);
2292 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2293 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2294 return true;
2295 } else {
2296 SDValue GA = DAG.getTargetGlobalAddress(
2297 GV: G->getGlobal(), DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2298 ADA = getADAEntry(DAG, Val: GA, DL, Offset: ADADelta);
2299 Callee = getADAEntry(DAG, Val: GA, DL, Offset: EPADelta);
2300 }
2301 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2302 SDValue ES = DAG.getTargetExternalSymbol(
2303 Sym: E->getSymbol(), VT: PtrVT, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2304 ADA = getADAEntry(DAG, Val: ES, DL, Offset: ADADelta);
2305 Callee = getADAEntry(DAG, Val: ES, DL, Offset: EPADelta);
2306 } else {
2307 // Function pointer case
2308 ADA = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2309 N2: DAG.getConstant(Val: ADADelta, DL, VT: PtrVT));
2310 ADA = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: ADA,
2311 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2312 Callee = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2313 N2: DAG.getConstant(Val: EPADelta, DL, VT: PtrVT));
2314 Callee = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Callee,
2315 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2316 }
2317 return false;
2318}
2319
2320SDValue
2321SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
2322 SmallVectorImpl<SDValue> &InVals) const {
2323 SelectionDAG &DAG = CLI.DAG;
2324 SDLoc &DL = CLI.DL;
2325 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2326 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2327 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2328 SDValue Chain = CLI.Chain;
2329 SDValue Callee = CLI.Callee;
2330 bool &IsTailCall = CLI.IsTailCall;
2331 CallingConv::ID CallConv = CLI.CallConv;
2332 bool IsVarArg = CLI.IsVarArg;
2333 MachineFunction &MF = DAG.getMachineFunction();
2334 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
2335 LLVMContext &Ctx = *DAG.getContext();
2336 SystemZCallingConventionRegisters *Regs = Subtarget.getSpecialRegisters();
2337
2338 // FIXME: z/OS support to be added in later.
2339 if (Subtarget.isTargetXPLINK64())
2340 IsTailCall = false;
2341
2342 // Integer args <=32 bits should have an extension attribute.
2343 verifyNarrowIntegerArgs_Call(Outs, F: &MF.getFunction(), Callee);
2344
2345 // Analyze the operands of the call, assigning locations to each operand.
2346 SmallVector<CCValAssign, 16> ArgLocs;
2347 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
2348 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_SystemZ);
2349
2350 // We don't support GuaranteedTailCallOpt, only automatically-detected
2351 // sibling calls.
2352 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs))
2353 IsTailCall = false;
2354
2355 // Get a count of how many bytes are to be pushed on the stack.
2356 unsigned NumBytes = ArgCCInfo.getStackSize();
2357
2358 // Mark the start of the call.
2359 if (!IsTailCall)
2360 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL);
2361
2362 // Copy argument values to their designated locations.
2363 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass;
2364 SmallVector<SDValue, 8> MemOpChains;
2365 SDValue StackPtr;
2366 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2367 CCValAssign &VA = ArgLocs[I];
2368 SDValue ArgValue = OutVals[I];
2369
2370 if (VA.getLocInfo() == CCValAssign::Indirect) {
2371 // Store the argument in a stack slot and pass its address.
2372 EVT SlotVT;
2373 MVT PartVT;
2374 unsigned NumParts = 1;
2375 if (analyzeArgSplit(Args: Outs, ArgLocs, I, PartVT, NumParts))
2376 SlotVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: PartVT.getSizeInBits() * NumParts);
2377 else
2378 SlotVT = Outs[I].VT;
2379 SDValue SpillSlot = DAG.CreateStackTemporary(VT: SlotVT);
2380 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
2381
2382 MachinePointerInfo StackPtrInfo =
2383 MachinePointerInfo::getFixedStack(MF, FI);
2384 MemOpChains.push_back(
2385 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot, PtrInfo: StackPtrInfo));
2386 // If the original argument was split (e.g. i128), we need
2387 // to store all parts of it here (and pass just one address).
2388 assert(Outs[I].PartOffset == 0);
2389 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2390 ++I;
2391 SDValue PartValue = OutVals[I];
2392 unsigned PartOffset = Outs[I].PartOffset;
2393 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot,
2394 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2395 MemOpChains.push_back(
2396 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
2397 PtrInfo: StackPtrInfo.getWithOffset(O: PartOffset)));
2398 assert(PartOffset && "Offset should be non-zero.");
2399 assert((PartOffset + PartValue.getValueType().getStoreSize() <=
2400 SlotVT.getStoreSize()) && "Not enough space for argument part!");
2401 }
2402 ArgValue = SpillSlot;
2403 } else
2404 ArgValue = convertValVTToLocVT(DAG, DL, VA, Value: ArgValue);
2405
2406 if (VA.isRegLoc()) {
2407 // In XPLINK64, for the 128-bit vararg case, ArgValue is bitcasted to a
2408 // MVT::i128 type. We decompose the 128-bit type to a pair of its high
2409 // and low values.
2410 if (VA.getLocVT() == MVT::i128)
2411 ArgValue = lowerI128ToGR128(DAG, In: ArgValue);
2412 // Queue up the argument copies and emit them at the end.
2413 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
2414 } else {
2415 assert(VA.isMemLoc() && "Argument not register or memory");
2416
2417 // Work out the address of the stack slot. Unpromoted ints and
2418 // floats are passed as right-justified 8-byte values.
2419 if (!StackPtr.getNode())
2420 StackPtr = DAG.getCopyFromReg(Chain, dl: DL,
2421 Reg: Regs->getStackPointerRegister(), VT: PtrVT);
2422 unsigned Offset = Regs->getStackPointerBias() + Regs->getCallFrameSize() +
2423 VA.getLocMemOffset();
2424 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
2425 Offset += 4;
2426 else if (VA.getLocVT() == MVT::f16)
2427 Offset += 6;
2428 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
2429 N2: DAG.getIntPtrConstant(Val: Offset, DL));
2430
2431 // Emit the store.
2432 MemOpChains.push_back(
2433 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address, PtrInfo: MachinePointerInfo()));
2434
2435 // Although long doubles or vectors are passed through the stack when
2436 // they are vararg (non-fixed arguments), if a long double or vector
2437 // occupies the third and fourth slot of the argument list GPR3 should
2438 // still shadow the third slot of the argument list.
2439 if (Subtarget.isTargetXPLINK64() && VA.needsCustom()) {
2440 SDValue ShadowArgValue =
2441 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: ArgValue,
2442 N2: DAG.getIntPtrConstant(Val: 1, DL));
2443 RegsToPass.push_back(Elt: std::make_pair(x: SystemZ::R3D, y&: ShadowArgValue));
2444 }
2445 }
2446 }
2447
2448 // Join the stores, which are independent of one another.
2449 if (!MemOpChains.empty())
2450 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
2451
2452 // Accept direct calls by converting symbolic call addresses to the
2453 // associated Target* opcodes. Force %r1 to be used for indirect
2454 // tail calls.
2455 SDValue Glue;
2456
2457 if (Subtarget.isTargetXPLINK64()) {
2458 SDValue ADA;
2459 bool IsBRASL = getzOSCalleeAndADA(DAG, Callee, ADA, DL, Chain);
2460 if (!IsBRASL) {
2461 unsigned CalleeReg = static_cast<SystemZXPLINK64Registers *>(Regs)
2462 ->getAddressOfCalleeRegister();
2463 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CalleeReg, N: Callee, Glue);
2464 Glue = Chain.getValue(R: 1);
2465 Callee = DAG.getRegister(Reg: CalleeReg, VT: Callee.getValueType());
2466 }
2467 RegsToPass.push_back(Elt: std::make_pair(
2468 x: static_cast<SystemZXPLINK64Registers *>(Regs)->getADARegister(), y&: ADA));
2469 } else {
2470 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2471 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2472 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2473 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2474 Callee = DAG.getTargetExternalSymbol(Sym: E->getSymbol(), VT: PtrVT);
2475 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2476 } else if (IsTailCall) {
2477 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R1D, N: Callee, Glue);
2478 Glue = Chain.getValue(R: 1);
2479 Callee = DAG.getRegister(Reg: SystemZ::R1D, VT: Callee.getValueType());
2480 }
2481 }
2482
2483 // Build a sequence of copy-to-reg nodes, chained and glued together.
2484 for (const auto &[Reg, N] : RegsToPass) {
2485 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N, Glue);
2486 Glue = Chain.getValue(R: 1);
2487 }
2488
2489 // The first call operand is the chain and the second is the target address.
2490 SmallVector<SDValue, 8> Ops;
2491 Ops.push_back(Elt: Chain);
2492 Ops.push_back(Elt: Callee);
2493
2494 // Add argument registers to the end of the list so that they are
2495 // known live into the call.
2496 for (const auto &[Reg, N] : RegsToPass)
2497 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
2498
2499 // Add a register mask operand representing the call-preserved registers.
2500 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2501 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2502 assert(Mask && "Missing call preserved mask for calling convention");
2503 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
2504
2505 // Glue the call to the argument copies, if any.
2506 if (Glue.getNode())
2507 Ops.push_back(Elt: Glue);
2508
2509 // Emit the call.
2510 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2511 if (IsTailCall) {
2512 SDValue Ret = DAG.getNode(Opcode: SystemZISD::SIBCALL, DL, VTList: NodeTys, Ops);
2513 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
2514 return Ret;
2515 }
2516 Chain = DAG.getNode(Opcode: SystemZISD::CALL, DL, VTList: NodeTys, Ops);
2517 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
2518 Glue = Chain.getValue(R: 1);
2519
2520 // Mark the end of the call, which is glued to the call itself.
2521 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
2522 Glue = Chain.getValue(R: 1);
2523
2524 // Assign locations to each value returned by this call.
2525 SmallVector<CCValAssign, 16> RetLocs;
2526 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
2527 RetCCInfo.AnalyzeCallResult(Ins, Fn: RetCC_SystemZ);
2528
2529 // Copy all of the result registers out of their specified physreg.
2530 for (CCValAssign &VA : RetLocs) {
2531 // Copy the value out, gluing the copy to the end of the call sequence.
2532 SDValue RetValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(),
2533 VT: VA.getLocVT(), Glue);
2534 Chain = RetValue.getValue(R: 1);
2535 Glue = RetValue.getValue(R: 2);
2536
2537 // Convert the value of the return register into the value that's
2538 // being returned.
2539 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: RetValue));
2540 }
2541
2542 return Chain;
2543}
2544
2545// Generate a call taking the given operands as arguments and returning a
2546// result of type RetVT.
2547std::pair<SDValue, SDValue> SystemZTargetLowering::makeExternalCall(
2548 SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT,
2549 ArrayRef<SDValue> Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL,
2550 bool DoesNotReturn, bool IsReturnValueUsed) const {
2551 TargetLowering::ArgListTy Args;
2552 Args.reserve(n: Ops.size());
2553
2554 for (SDValue Op : Ops) {
2555 TargetLowering::ArgListEntry Entry(
2556 Op, Op.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
2557 Entry.IsSExt = shouldSignExtendTypeInLibCall(Ty: Entry.Ty, IsSigned);
2558 Entry.IsZExt = !Entry.IsSExt;
2559 Args.push_back(x: Entry);
2560 }
2561
2562 SDValue Callee =
2563 DAG.getExternalSymbol(Sym: CalleeName, VT: getPointerTy(DL: DAG.getDataLayout()));
2564
2565 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2566 TargetLowering::CallLoweringInfo CLI(DAG);
2567 bool SignExtend = shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2568 CLI.setDebugLoc(DL)
2569 .setChain(Chain)
2570 .setCallee(CC: CallConv, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2571 .setNoReturn(DoesNotReturn)
2572 .setDiscardResult(!IsReturnValueUsed)
2573 .setSExtResult(SignExtend)
2574 .setZExtResult(!SignExtend);
2575 return LowerCallTo(CLI);
2576}
2577
2578bool SystemZTargetLowering::CanLowerReturn(
2579 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2580 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
2581 const Type *RetTy) const {
2582 // Special case that we cannot easily detect in RetCC_SystemZ since
2583 // i128 may not be a legal type.
2584 for (auto &Out : Outs)
2585 if (Out.ArgVT.isScalarInteger() && Out.ArgVT.getSizeInBits() > 64)
2586 return false;
2587
2588 SmallVector<CCValAssign, 16> RetLocs;
2589 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Context);
2590 return RetCCInfo.CheckReturn(Outs, Fn: RetCC_SystemZ);
2591}
2592
2593SDValue
2594SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2595 bool IsVarArg,
2596 const SmallVectorImpl<ISD::OutputArg> &Outs,
2597 const SmallVectorImpl<SDValue> &OutVals,
2598 const SDLoc &DL, SelectionDAG &DAG) const {
2599 MachineFunction &MF = DAG.getMachineFunction();
2600
2601 // Integer args <=32 bits should have an extension attribute.
2602 verifyNarrowIntegerArgs_Ret(Outs, F: &MF.getFunction());
2603
2604 // Assign locations to each returned value.
2605 SmallVector<CCValAssign, 16> RetLocs;
2606 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
2607 RetCCInfo.AnalyzeReturn(Outs, Fn: RetCC_SystemZ);
2608
2609 // Quick exit for void returns
2610 if (RetLocs.empty())
2611 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Operand: Chain);
2612
2613 if (CallConv == CallingConv::GHC)
2614 report_fatal_error(reason: "GHC functions return void only");
2615
2616 // Copy the result values into the output registers.
2617 SDValue Glue;
2618 SmallVector<SDValue, 4> RetOps;
2619 RetOps.push_back(Elt: Chain);
2620 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
2621 CCValAssign &VA = RetLocs[I];
2622 SDValue RetValue = OutVals[I];
2623
2624 // Make the return register live on exit.
2625 assert(VA.isRegLoc() && "Can only return in registers!");
2626
2627 // Promote the value as required.
2628 RetValue = convertValVTToLocVT(DAG, DL, VA, Value: RetValue);
2629
2630 // Chain and glue the copies together.
2631 Register Reg = VA.getLocReg();
2632 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: RetValue, Glue);
2633 Glue = Chain.getValue(R: 1);
2634 RetOps.push_back(Elt: DAG.getRegister(Reg, VT: VA.getLocVT()));
2635 }
2636
2637 // Update chain and glue.
2638 RetOps[0] = Chain;
2639 if (Glue.getNode())
2640 RetOps.push_back(Elt: Glue);
2641
2642 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Ops: RetOps);
2643}
2644
2645// Return true if Op is an intrinsic node with chain that returns the CC value
2646// as its only (other) argument. Provide the associated SystemZISD opcode and
2647// the mask of valid CC values if so.
2648static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
2649 unsigned &CCValid) {
2650 unsigned Id = Op.getConstantOperandVal(i: 1);
2651 switch (Id) {
2652 case Intrinsic::s390_tbegin:
2653 Opcode = SystemZISD::TBEGIN;
2654 CCValid = SystemZ::CCMASK_TBEGIN;
2655 return true;
2656
2657 case Intrinsic::s390_tbegin_nofloat:
2658 Opcode = SystemZISD::TBEGIN_NOFLOAT;
2659 CCValid = SystemZ::CCMASK_TBEGIN;
2660 return true;
2661
2662 case Intrinsic::s390_tend:
2663 Opcode = SystemZISD::TEND;
2664 CCValid = SystemZ::CCMASK_TEND;
2665 return true;
2666
2667 default:
2668 return false;
2669 }
2670}
2671
2672// Return true if Op is an intrinsic node without chain that returns the
2673// CC value as its final argument. Provide the associated SystemZISD
2674// opcode and the mask of valid CC values if so.
2675static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
2676 unsigned Id = Op.getConstantOperandVal(i: 0);
2677 switch (Id) {
2678 case Intrinsic::s390_vpkshs:
2679 case Intrinsic::s390_vpksfs:
2680 case Intrinsic::s390_vpksgs:
2681 Opcode = SystemZISD::PACKS_CC;
2682 CCValid = SystemZ::CCMASK_VCMP;
2683 return true;
2684
2685 case Intrinsic::s390_vpklshs:
2686 case Intrinsic::s390_vpklsfs:
2687 case Intrinsic::s390_vpklsgs:
2688 Opcode = SystemZISD::PACKLS_CC;
2689 CCValid = SystemZ::CCMASK_VCMP;
2690 return true;
2691
2692 case Intrinsic::s390_vceqbs:
2693 case Intrinsic::s390_vceqhs:
2694 case Intrinsic::s390_vceqfs:
2695 case Intrinsic::s390_vceqgs:
2696 case Intrinsic::s390_vceqqs:
2697 Opcode = SystemZISD::VICMPES;
2698 CCValid = SystemZ::CCMASK_VCMP;
2699 return true;
2700
2701 case Intrinsic::s390_vchbs:
2702 case Intrinsic::s390_vchhs:
2703 case Intrinsic::s390_vchfs:
2704 case Intrinsic::s390_vchgs:
2705 case Intrinsic::s390_vchqs:
2706 Opcode = SystemZISD::VICMPHS;
2707 CCValid = SystemZ::CCMASK_VCMP;
2708 return true;
2709
2710 case Intrinsic::s390_vchlbs:
2711 case Intrinsic::s390_vchlhs:
2712 case Intrinsic::s390_vchlfs:
2713 case Intrinsic::s390_vchlgs:
2714 case Intrinsic::s390_vchlqs:
2715 Opcode = SystemZISD::VICMPHLS;
2716 CCValid = SystemZ::CCMASK_VCMP;
2717 return true;
2718
2719 case Intrinsic::s390_vtm:
2720 Opcode = SystemZISD::VTM;
2721 CCValid = SystemZ::CCMASK_VCMP;
2722 return true;
2723
2724 case Intrinsic::s390_vfaebs:
2725 case Intrinsic::s390_vfaehs:
2726 case Intrinsic::s390_vfaefs:
2727 Opcode = SystemZISD::VFAE_CC;
2728 CCValid = SystemZ::CCMASK_ANY;
2729 return true;
2730
2731 case Intrinsic::s390_vfaezbs:
2732 case Intrinsic::s390_vfaezhs:
2733 case Intrinsic::s390_vfaezfs:
2734 Opcode = SystemZISD::VFAEZ_CC;
2735 CCValid = SystemZ::CCMASK_ANY;
2736 return true;
2737
2738 case Intrinsic::s390_vfeebs:
2739 case Intrinsic::s390_vfeehs:
2740 case Intrinsic::s390_vfeefs:
2741 Opcode = SystemZISD::VFEE_CC;
2742 CCValid = SystemZ::CCMASK_ANY;
2743 return true;
2744
2745 case Intrinsic::s390_vfeezbs:
2746 case Intrinsic::s390_vfeezhs:
2747 case Intrinsic::s390_vfeezfs:
2748 Opcode = SystemZISD::VFEEZ_CC;
2749 CCValid = SystemZ::CCMASK_ANY;
2750 return true;
2751
2752 case Intrinsic::s390_vfenebs:
2753 case Intrinsic::s390_vfenehs:
2754 case Intrinsic::s390_vfenefs:
2755 Opcode = SystemZISD::VFENE_CC;
2756 CCValid = SystemZ::CCMASK_ANY;
2757 return true;
2758
2759 case Intrinsic::s390_vfenezbs:
2760 case Intrinsic::s390_vfenezhs:
2761 case Intrinsic::s390_vfenezfs:
2762 Opcode = SystemZISD::VFENEZ_CC;
2763 CCValid = SystemZ::CCMASK_ANY;
2764 return true;
2765
2766 case Intrinsic::s390_vistrbs:
2767 case Intrinsic::s390_vistrhs:
2768 case Intrinsic::s390_vistrfs:
2769 Opcode = SystemZISD::VISTR_CC;
2770 CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3;
2771 return true;
2772
2773 case Intrinsic::s390_vstrcbs:
2774 case Intrinsic::s390_vstrchs:
2775 case Intrinsic::s390_vstrcfs:
2776 Opcode = SystemZISD::VSTRC_CC;
2777 CCValid = SystemZ::CCMASK_ANY;
2778 return true;
2779
2780 case Intrinsic::s390_vstrczbs:
2781 case Intrinsic::s390_vstrczhs:
2782 case Intrinsic::s390_vstrczfs:
2783 Opcode = SystemZISD::VSTRCZ_CC;
2784 CCValid = SystemZ::CCMASK_ANY;
2785 return true;
2786
2787 case Intrinsic::s390_vstrsb:
2788 case Intrinsic::s390_vstrsh:
2789 case Intrinsic::s390_vstrsf:
2790 Opcode = SystemZISD::VSTRS_CC;
2791 CCValid = SystemZ::CCMASK_ANY;
2792 return true;
2793
2794 case Intrinsic::s390_vstrszb:
2795 case Intrinsic::s390_vstrszh:
2796 case Intrinsic::s390_vstrszf:
2797 Opcode = SystemZISD::VSTRSZ_CC;
2798 CCValid = SystemZ::CCMASK_ANY;
2799 return true;
2800
2801 case Intrinsic::s390_vfcedbs:
2802 case Intrinsic::s390_vfcesbs:
2803 Opcode = SystemZISD::VFCMPES;
2804 CCValid = SystemZ::CCMASK_VCMP;
2805 return true;
2806
2807 case Intrinsic::s390_vfchdbs:
2808 case Intrinsic::s390_vfchsbs:
2809 Opcode = SystemZISD::VFCMPHS;
2810 CCValid = SystemZ::CCMASK_VCMP;
2811 return true;
2812
2813 case Intrinsic::s390_vfchedbs:
2814 case Intrinsic::s390_vfchesbs:
2815 Opcode = SystemZISD::VFCMPHES;
2816 CCValid = SystemZ::CCMASK_VCMP;
2817 return true;
2818
2819 case Intrinsic::s390_vftcidb:
2820 case Intrinsic::s390_vftcisb:
2821 Opcode = SystemZISD::VFTCI;
2822 CCValid = SystemZ::CCMASK_VCMP;
2823 return true;
2824
2825 case Intrinsic::s390_tdc:
2826 Opcode = SystemZISD::TDC;
2827 CCValid = SystemZ::CCMASK_TDC;
2828 return true;
2829
2830 default:
2831 return false;
2832 }
2833}
2834
2835// Emit an intrinsic with chain and an explicit CC register result.
2836static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op,
2837 unsigned Opcode) {
2838 // Copy all operands except the intrinsic ID.
2839 unsigned NumOps = Op.getNumOperands();
2840 SmallVector<SDValue, 6> Ops;
2841 Ops.reserve(N: NumOps - 1);
2842 Ops.push_back(Elt: Op.getOperand(i: 0));
2843 for (unsigned I = 2; I < NumOps; ++I)
2844 Ops.push_back(Elt: Op.getOperand(i: I));
2845
2846 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
2847 SDVTList RawVTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
2848 SDValue Intr = DAG.getNode(Opcode, DL: SDLoc(Op), VTList: RawVTs, Ops);
2849 SDValue OldChain = SDValue(Op.getNode(), 1);
2850 SDValue NewChain = SDValue(Intr.getNode(), 1);
2851 DAG.ReplaceAllUsesOfValueWith(From: OldChain, To: NewChain);
2852 return Intr.getNode();
2853}
2854
2855// Emit an intrinsic with an explicit CC register result.
2856static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op,
2857 unsigned Opcode) {
2858 // Copy all operands except the intrinsic ID.
2859 SDLoc DL(Op);
2860 unsigned NumOps = Op.getNumOperands();
2861 SmallVector<SDValue, 6> Ops;
2862 Ops.reserve(N: NumOps - 1);
2863 for (unsigned I = 1; I < NumOps; ++I) {
2864 SDValue CurrOper = Op.getOperand(i: I);
2865 if (CurrOper.getValueType() == MVT::f16) {
2866 assert((Op.getConstantOperandVal(0) == Intrinsic::s390_tdc && I == 1) &&
2867 "Unhandled intrinsic with f16 operand.");
2868 CurrOper = DAG.getFPExtendOrRound(Op: CurrOper, DL, VT: MVT::f32);
2869 }
2870 Ops.push_back(Elt: CurrOper);
2871 }
2872
2873 SDValue Intr = DAG.getNode(Opcode, DL, VTList: Op->getVTList(), Ops);
2874 return Intr.getNode();
2875}
2876
2877// CC is a comparison that will be implemented using an integer or
2878// floating-point comparison. Return the condition code mask for
2879// a branch on true. In the integer case, CCMASK_CMP_UO is set for
2880// unsigned comparisons and clear for signed ones. In the floating-point
2881// case, CCMASK_CMP_UO has its normal mask meaning (unordered).
2882static unsigned CCMaskForCondCode(ISD::CondCode CC) {
2883#define CONV(X) \
2884 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
2885 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
2886 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
2887
2888 switch (CC) {
2889 default:
2890 llvm_unreachable("Invalid integer condition!");
2891
2892 CONV(EQ);
2893 CONV(NE);
2894 CONV(GT);
2895 CONV(GE);
2896 CONV(LT);
2897 CONV(LE);
2898
2899 case ISD::SETO: return SystemZ::CCMASK_CMP_O;
2900 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO;
2901 }
2902#undef CONV
2903}
2904
2905// If C can be converted to a comparison against zero, adjust the operands
2906// as necessary.
2907static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2908 if (C.ICmpType == SystemZICMP::UnsignedOnly)
2909 return;
2910
2911 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1.getNode());
2912 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
2913 return;
2914
2915 int64_t Value = ConstOp1->getSExtValue();
2916 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
2917 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
2918 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
2919 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
2920 C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
2921 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op1.getValueType());
2922 }
2923}
2924
2925// If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
2926// adjust the operands as necessary.
2927static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
2928 Comparison &C) {
2929 // For us to make any changes, it must a comparison between a single-use
2930 // load and a constant.
2931 if (!C.Op0.hasOneUse() ||
2932 C.Op0.getOpcode() != ISD::LOAD ||
2933 C.Op1.getOpcode() != ISD::Constant)
2934 return;
2935
2936 // We must have an 8- or 16-bit load.
2937 auto *Load = cast<LoadSDNode>(Val&: C.Op0);
2938 unsigned NumBits = Load->getMemoryVT().getSizeInBits();
2939 if ((NumBits != 8 && NumBits != 16) ||
2940 NumBits != Load->getMemoryVT().getStoreSizeInBits())
2941 return;
2942
2943 // The load must be an extending one and the constant must be within the
2944 // range of the unextended value.
2945 auto *ConstOp1 = cast<ConstantSDNode>(Val&: C.Op1);
2946 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
2947 return;
2948 uint64_t Value = ConstOp1->getZExtValue();
2949 uint64_t Mask = (1 << NumBits) - 1;
2950 if (Load->getExtensionType() == ISD::SEXTLOAD) {
2951 // Make sure that ConstOp1 is in range of C.Op0.
2952 int64_t SignedValue = ConstOp1->getSExtValue();
2953 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)
2954 return;
2955 if (C.ICmpType != SystemZICMP::SignedOnly) {
2956 // Unsigned comparison between two sign-extended values is equivalent
2957 // to unsigned comparison between two zero-extended values.
2958 Value &= Mask;
2959 } else if (NumBits == 8) {
2960 // Try to treat the comparison as unsigned, so that we can use CLI.
2961 // Adjust CCMask and Value as necessary.
2962 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
2963 // Test whether the high bit of the byte is set.
2964 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
2965 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
2966 // Test whether the high bit of the byte is clear.
2967 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
2968 else
2969 // No instruction exists for this combination.
2970 return;
2971 C.ICmpType = SystemZICMP::UnsignedOnly;
2972 }
2973 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {
2974 if (Value > Mask)
2975 return;
2976 // If the constant is in range, we can use any comparison.
2977 C.ICmpType = SystemZICMP::Any;
2978 } else
2979 return;
2980
2981 // Make sure that the first operand is an i32 of the right extension type.
2982 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ?
2983 ISD::SEXTLOAD :
2984 ISD::ZEXTLOAD);
2985 if (C.Op0.getValueType() != MVT::i32 ||
2986 Load->getExtensionType() != ExtType) {
2987 C.Op0 = DAG.getExtLoad(ExtType, dl: SDLoc(Load), VT: MVT::i32, Chain: Load->getChain(),
2988 Ptr: Load->getBasePtr(), PtrInfo: Load->getPointerInfo(),
2989 MemVT: Load->getMemoryVT(), Alignment: Load->getAlign(),
2990 MMOFlags: Load->getMemOperand()->getFlags());
2991 // Update the chain uses.
2992 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 1), To: C.Op0.getValue(R: 1));
2993 }
2994
2995 // Make sure that the second operand is an i32 with the right value.
2996 if (C.Op1.getValueType() != MVT::i32 ||
2997 Value != ConstOp1->getZExtValue())
2998 C.Op1 = DAG.getConstant(Val: (uint32_t)Value, DL, VT: MVT::i32);
2999}
3000
3001// Return true if Op is either an unextended load, or a load suitable
3002// for integer register-memory comparisons of type ICmpType.
3003static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
3004 auto *Load = dyn_cast<LoadSDNode>(Val: Op.getNode());
3005 if (Load) {
3006 // There are no instructions to compare a register with a memory byte.
3007 if (Load->getMemoryVT() == MVT::i8)
3008 return false;
3009 // Otherwise decide on extension type.
3010 switch (Load->getExtensionType()) {
3011 case ISD::NON_EXTLOAD:
3012 return true;
3013 case ISD::SEXTLOAD:
3014 return ICmpType != SystemZICMP::UnsignedOnly;
3015 case ISD::ZEXTLOAD:
3016 return ICmpType != SystemZICMP::SignedOnly;
3017 default:
3018 break;
3019 }
3020 }
3021 return false;
3022}
3023
3024// Return true if it is better to swap the operands of C.
3025static bool shouldSwapCmpOperands(const Comparison &C) {
3026 // Leave i128 and f128 comparisons alone, since they have no memory forms.
3027 if (C.Op0.getValueType() == MVT::i128)
3028 return false;
3029 if (C.Op0.getValueType() == MVT::f128)
3030 return false;
3031
3032 // Always keep a floating-point constant second, since comparisons with
3033 // zero can use LOAD TEST and comparisons with other constants make a
3034 // natural memory operand.
3035 if (isa<ConstantFPSDNode>(Val: C.Op1))
3036 return false;
3037
3038 // Never swap comparisons with zero since there are many ways to optimize
3039 // those later.
3040 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1);
3041 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
3042 return false;
3043
3044 // Also keep natural memory operands second if the loaded value is
3045 // only used here. Several comparisons have memory forms.
3046 if (isNaturalMemoryOperand(Op: C.Op1, ICmpType: C.ICmpType) && C.Op1.hasOneUse())
3047 return false;
3048
3049 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.
3050 // In that case we generally prefer the memory to be second.
3051 if (isNaturalMemoryOperand(Op: C.Op0, ICmpType: C.ICmpType) && C.Op0.hasOneUse()) {
3052 // The only exceptions are when the second operand is a constant and
3053 // we can use things like CHHSI.
3054 if (!ConstOp1)
3055 return true;
3056 // The unsigned memory-immediate instructions can handle 16-bit
3057 // unsigned integers.
3058 if (C.ICmpType != SystemZICMP::SignedOnly &&
3059 isUInt<16>(x: ConstOp1->getZExtValue()))
3060 return false;
3061 // The signed memory-immediate instructions can handle 16-bit
3062 // signed integers.
3063 if (C.ICmpType != SystemZICMP::UnsignedOnly &&
3064 isInt<16>(x: ConstOp1->getSExtValue()))
3065 return false;
3066 return true;
3067 }
3068
3069 // Try to promote the use of CGFR and CLGFR.
3070 unsigned Opcode0 = C.Op0.getOpcode();
3071 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND)
3072 return true;
3073 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND)
3074 return true;
3075 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::AND &&
3076 C.Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
3077 C.Op0.getConstantOperandVal(i: 1) == 0xffffffff)
3078 return true;
3079
3080 return false;
3081}
3082
3083// Check whether C tests for equality between X and Y and whether X - Y
3084// or Y - X is also computed. In that case it's better to compare the
3085// result of the subtraction against zero.
3086static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
3087 Comparison &C) {
3088 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3089 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3090 for (SDNode *N : C.Op0->users()) {
3091 if (N->getOpcode() == ISD::SUB &&
3092 ((N->getOperand(Num: 0) == C.Op0 && N->getOperand(Num: 1) == C.Op1) ||
3093 (N->getOperand(Num: 0) == C.Op1 && N->getOperand(Num: 1) == C.Op0))) {
3094 // Disable the nsw and nuw flags: the backend needs to handle
3095 // overflow as well during comparison elimination.
3096 N->dropFlags(Mask: SDNodeFlags::NoWrap);
3097 C.Op0 = SDValue(N, 0);
3098 C.Op1 = DAG.getConstant(Val: 0, DL, VT: N->getValueType(ResNo: 0));
3099 return;
3100 }
3101 }
3102 }
3103}
3104
3105// Check whether C compares a floating-point value with zero and if that
3106// floating-point value is also negated. In this case we can use the
3107// negation to set CC, so avoiding separate LOAD AND TEST and
3108// LOAD (NEGATIVE/COMPLEMENT) instructions.
3109static void adjustForFNeg(Comparison &C) {
3110 // This optimization is invalid for strict comparisons, since FNEG
3111 // does not raise any exceptions.
3112 if (C.Chain)
3113 return;
3114 auto *C1 = dyn_cast<ConstantFPSDNode>(Val&: C.Op1);
3115 if (C1 && C1->isZero()) {
3116 for (SDNode *N : C.Op0->users()) {
3117 if (N->getOpcode() == ISD::FNEG) {
3118 C.Op0 = SDValue(N, 0);
3119 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3120 return;
3121 }
3122 }
3123 }
3124}
3125
3126// Check whether C compares (shl X, 32) with 0 and whether X is
3127// also sign-extended. In that case it is better to test the result
3128// of the sign extension using LTGFR.
3129//
3130// This case is important because InstCombine transforms a comparison
3131// with (sext (trunc X)) into a comparison with (shl X, 32).
3132static void adjustForLTGFR(Comparison &C) {
3133 // Check for a comparison between (shl X, 32) and 0.
3134 if (C.Op0.getOpcode() == ISD::SHL && C.Op0.getValueType() == MVT::i64 &&
3135 C.Op1.getOpcode() == ISD::Constant && C.Op1->getAsZExtVal() == 0) {
3136 auto *C1 = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3137 if (C1 && C1->getZExtValue() == 32) {
3138 SDValue ShlOp0 = C.Op0.getOperand(i: 0);
3139 // See whether X has any SIGN_EXTEND_INREG uses.
3140 for (SDNode *N : ShlOp0->users()) {
3141 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3142 cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT() == MVT::i32) {
3143 C.Op0 = SDValue(N, 0);
3144 return;
3145 }
3146 }
3147 }
3148 }
3149}
3150
3151// If C compares the truncation of an extending load, try to compare
3152// the untruncated value instead. This exposes more opportunities to
3153// reuse CC.
3154static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
3155 Comparison &C) {
3156 if (C.Op0.getOpcode() == ISD::TRUNCATE &&
3157 C.Op0.getOperand(i: 0).getOpcode() == ISD::LOAD &&
3158 C.Op1.getOpcode() == ISD::Constant &&
3159 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
3160 C.Op1->getAsZExtVal() == 0) {
3161 auto *L = cast<LoadSDNode>(Val: C.Op0.getOperand(i: 0));
3162 if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=
3163 C.Op0.getValueSizeInBits().getFixedValue()) {
3164 unsigned Type = L->getExtensionType();
3165 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
3166 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
3167 C.Op0 = C.Op0.getOperand(i: 0);
3168 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op0.getValueType());
3169 }
3170 }
3171 }
3172}
3173
3174// Return true if shift operation N has an in-range constant shift value.
3175// Store it in ShiftVal if so.
3176static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
3177 auto *Shift = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
3178 if (!Shift)
3179 return false;
3180
3181 uint64_t Amount = Shift->getZExtValue();
3182 if (Amount >= N.getValueSizeInBits())
3183 return false;
3184
3185 ShiftVal = Amount;
3186 return true;
3187}
3188
3189// Check whether an AND with Mask is suitable for a TEST UNDER MASK
3190// instruction and whether the CC value is descriptive enough to handle
3191// a comparison of type Opcode between the AND result and CmpVal.
3192// CCMask says which comparison result is being tested and BitSize is
3193// the number of bits in the operands. If TEST UNDER MASK can be used,
3194// return the corresponding CC mask, otherwise return 0.
3195static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
3196 uint64_t Mask, uint64_t CmpVal,
3197 unsigned ICmpType) {
3198 assert(Mask != 0 && "ANDs with zero should have been removed by now");
3199
3200 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL.
3201 if (!SystemZ::isImmLL(Val: Mask) && !SystemZ::isImmLH(Val: Mask) &&
3202 !SystemZ::isImmHL(Val: Mask) && !SystemZ::isImmHH(Val: Mask))
3203 return 0;
3204
3205 // Work out the masks for the lowest and highest bits.
3206 uint64_t High = llvm::bit_floor(Value: Mask);
3207 uint64_t Low = uint64_t(1) << llvm::countr_zero(Val: Mask);
3208
3209 // Signed ordered comparisons are effectively unsigned if the sign
3210 // bit is dropped.
3211 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly);
3212
3213 // Check for equality comparisons with 0, or the equivalent.
3214 if (CmpVal == 0) {
3215 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3216 return SystemZ::CCMASK_TM_ALL_0;
3217 if (CCMask == SystemZ::CCMASK_CMP_NE)
3218 return SystemZ::CCMASK_TM_SOME_1;
3219 }
3220 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) {
3221 if (CCMask == SystemZ::CCMASK_CMP_LT)
3222 return SystemZ::CCMASK_TM_ALL_0;
3223 if (CCMask == SystemZ::CCMASK_CMP_GE)
3224 return SystemZ::CCMASK_TM_SOME_1;
3225 }
3226 if (EffectivelyUnsigned && CmpVal < Low) {
3227 if (CCMask == SystemZ::CCMASK_CMP_LE)
3228 return SystemZ::CCMASK_TM_ALL_0;
3229 if (CCMask == SystemZ::CCMASK_CMP_GT)
3230 return SystemZ::CCMASK_TM_SOME_1;
3231 }
3232
3233 // Check for equality comparisons with the mask, or the equivalent.
3234 if (CmpVal == Mask) {
3235 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3236 return SystemZ::CCMASK_TM_ALL_1;
3237 if (CCMask == SystemZ::CCMASK_CMP_NE)
3238 return SystemZ::CCMASK_TM_SOME_0;
3239 }
3240 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {
3241 if (CCMask == SystemZ::CCMASK_CMP_GT)
3242 return SystemZ::CCMASK_TM_ALL_1;
3243 if (CCMask == SystemZ::CCMASK_CMP_LE)
3244 return SystemZ::CCMASK_TM_SOME_0;
3245 }
3246 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {
3247 if (CCMask == SystemZ::CCMASK_CMP_GE)
3248 return SystemZ::CCMASK_TM_ALL_1;
3249 if (CCMask == SystemZ::CCMASK_CMP_LT)
3250 return SystemZ::CCMASK_TM_SOME_0;
3251 }
3252
3253 // Check for ordered comparisons with the top bit.
3254 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {
3255 if (CCMask == SystemZ::CCMASK_CMP_LE)
3256 return SystemZ::CCMASK_TM_MSB_0;
3257 if (CCMask == SystemZ::CCMASK_CMP_GT)
3258 return SystemZ::CCMASK_TM_MSB_1;
3259 }
3260 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {
3261 if (CCMask == SystemZ::CCMASK_CMP_LT)
3262 return SystemZ::CCMASK_TM_MSB_0;
3263 if (CCMask == SystemZ::CCMASK_CMP_GE)
3264 return SystemZ::CCMASK_TM_MSB_1;
3265 }
3266
3267 // If there are just two bits, we can do equality checks for Low and High
3268 // as well.
3269 if (Mask == Low + High) {
3270 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
3271 return SystemZ::CCMASK_TM_MIXED_MSB_0;
3272 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
3273 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY;
3274 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
3275 return SystemZ::CCMASK_TM_MIXED_MSB_1;
3276 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
3277 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY;
3278 }
3279
3280 // Looks like we've exhausted our options.
3281 return 0;
3282}
3283
3284// See whether C can be implemented as a TEST UNDER MASK instruction.
3285// Update the arguments with the TM version if so.
3286static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
3287 Comparison &C) {
3288 // Use VECTOR TEST UNDER MASK for i128 operations.
3289 if (C.Op0.getValueType() == MVT::i128) {
3290 // We can use VTM for EQ/NE comparisons of x & y against 0.
3291 if (C.Op0.getOpcode() == ISD::AND &&
3292 (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3293 C.CCMask == SystemZ::CCMASK_CMP_NE)) {
3294 auto *Mask = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3295 if (Mask && Mask->getAPIntValue() == 0) {
3296 C.Opcode = SystemZISD::VTM;
3297 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 1));
3298 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 0));
3299 C.CCValid = SystemZ::CCMASK_VCMP;
3300 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3301 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3302 else
3303 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3304 }
3305 }
3306 return;
3307 }
3308
3309 // Check that we have a comparison with a constant.
3310 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3311 if (!ConstOp1)
3312 return;
3313 uint64_t CmpVal = ConstOp1->getZExtValue();
3314
3315 // Check whether the nonconstant input is an AND with a constant mask.
3316 Comparison NewC(C);
3317 uint64_t MaskVal;
3318 ConstantSDNode *Mask = nullptr;
3319 if (C.Op0.getOpcode() == ISD::AND) {
3320 NewC.Op0 = C.Op0.getOperand(i: 0);
3321 NewC.Op1 = C.Op0.getOperand(i: 1);
3322 Mask = dyn_cast<ConstantSDNode>(Val&: NewC.Op1);
3323 if (!Mask)
3324 return;
3325 MaskVal = Mask->getZExtValue();
3326 } else {
3327 // There is no instruction to compare with a 64-bit immediate
3328 // so use TMHH instead if possible. We need an unsigned ordered
3329 // comparison with an i64 immediate.
3330 if (NewC.Op0.getValueType() != MVT::i64 ||
3331 NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
3332 NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
3333 NewC.ICmpType == SystemZICMP::SignedOnly)
3334 return;
3335 // Convert LE and GT comparisons into LT and GE.
3336 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
3337 NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
3338 if (CmpVal == uint64_t(-1))
3339 return;
3340 CmpVal += 1;
3341 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
3342 }
3343 // If the low N bits of Op1 are zero than the low N bits of Op0 can
3344 // be masked off without changing the result.
3345 MaskVal = -(CmpVal & -CmpVal);
3346 NewC.ICmpType = SystemZICMP::UnsignedOnly;
3347 }
3348 if (!MaskVal)
3349 return;
3350
3351 // Check whether the combination of mask, comparison value and comparison
3352 // type are suitable.
3353 unsigned BitSize = NewC.Op0.getValueSizeInBits();
3354 unsigned NewCCMask, ShiftVal;
3355 if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3356 NewC.Op0.getOpcode() == ISD::SHL &&
3357 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3358 (MaskVal >> ShiftVal != 0) &&
3359 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
3360 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3361 Mask: MaskVal >> ShiftVal,
3362 CmpVal: CmpVal >> ShiftVal,
3363 ICmpType: SystemZICMP::Any))) {
3364 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3365 MaskVal >>= ShiftVal;
3366 } else if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3367 NewC.Op0.getOpcode() == ISD::SRL &&
3368 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3369 (MaskVal << ShiftVal != 0) &&
3370 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
3371 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3372 Mask: MaskVal << ShiftVal,
3373 CmpVal: CmpVal << ShiftVal,
3374 ICmpType: SystemZICMP::UnsignedOnly))) {
3375 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3376 MaskVal <<= ShiftVal;
3377 } else {
3378 NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask, Mask: MaskVal, CmpVal,
3379 ICmpType: NewC.ICmpType);
3380 if (!NewCCMask)
3381 return;
3382 }
3383
3384 // Go ahead and make the change.
3385 C.Opcode = SystemZISD::TM;
3386 C.Op0 = NewC.Op0;
3387 if (Mask && Mask->getZExtValue() == MaskVal)
3388 C.Op1 = SDValue(Mask, 0);
3389 else
3390 C.Op1 = DAG.getConstant(Val: MaskVal, DL, VT: C.Op0.getValueType());
3391 C.CCValid = SystemZ::CCMASK_TM;
3392 C.CCMask = NewCCMask;
3393}
3394
3395// Implement i128 comparison in vector registers.
3396static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL,
3397 Comparison &C) {
3398 if (C.Opcode != SystemZISD::ICMP)
3399 return;
3400 if (C.Op0.getValueType() != MVT::i128)
3401 return;
3402
3403 // Recognize vector comparison reductions.
3404 if ((C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3405 C.CCMask == SystemZ::CCMASK_CMP_NE) &&
3406 (isNullConstant(V: C.Op1) || isAllOnesConstant(V: C.Op1))) {
3407 bool CmpEq = C.CCMask == SystemZ::CCMASK_CMP_EQ;
3408 bool CmpNull = isNullConstant(V: C.Op1);
3409 SDValue Src = peekThroughBitcasts(V: C.Op0);
3410 if (Src.hasOneUse() && isBitwiseNot(V: Src)) {
3411 Src = Src.getOperand(i: 0);
3412 CmpNull = !CmpNull;
3413 }
3414 unsigned Opcode = 0;
3415 if (Src.hasOneUse()) {
3416 switch (Src.getOpcode()) {
3417 case SystemZISD::VICMPE: Opcode = SystemZISD::VICMPES; break;
3418 case SystemZISD::VICMPH: Opcode = SystemZISD::VICMPHS; break;
3419 case SystemZISD::VICMPHL: Opcode = SystemZISD::VICMPHLS; break;
3420 case SystemZISD::VFCMPE: Opcode = SystemZISD::VFCMPES; break;
3421 case SystemZISD::VFCMPH: Opcode = SystemZISD::VFCMPHS; break;
3422 case SystemZISD::VFCMPHE: Opcode = SystemZISD::VFCMPHES; break;
3423 default: break;
3424 }
3425 }
3426 if (Opcode) {
3427 C.Opcode = Opcode;
3428 C.Op0 = Src->getOperand(Num: 0);
3429 C.Op1 = Src->getOperand(Num: 1);
3430 C.CCValid = SystemZ::CCMASK_VCMP;
3431 C.CCMask = CmpNull ? SystemZ::CCMASK_VCMP_NONE : SystemZ::CCMASK_VCMP_ALL;
3432 if (!CmpEq)
3433 C.CCMask ^= C.CCValid;
3434 return;
3435 }
3436 }
3437
3438 // Everything below here is not useful if we have native i128 compares.
3439 if (DAG.getSubtarget<SystemZSubtarget>().hasVectorEnhancements3())
3440 return;
3441
3442 // (In-)Equality comparisons can be implemented via VCEQGS.
3443 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3444 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3445 C.Opcode = SystemZISD::VICMPES;
3446 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op0);
3447 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op1);
3448 C.CCValid = SystemZ::CCMASK_VCMP;
3449 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3450 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3451 else
3452 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3453 return;
3454 }
3455
3456 // Normalize other comparisons to GT.
3457 bool Swap = false, Invert = false;
3458 switch (C.CCMask) {
3459 case SystemZ::CCMASK_CMP_GT: break;
3460 case SystemZ::CCMASK_CMP_LT: Swap = true; break;
3461 case SystemZ::CCMASK_CMP_LE: Invert = true; break;
3462 case SystemZ::CCMASK_CMP_GE: Swap = Invert = true; break;
3463 default: llvm_unreachable("Invalid integer condition!");
3464 }
3465 if (Swap)
3466 std::swap(a&: C.Op0, b&: C.Op1);
3467
3468 if (C.ICmpType == SystemZICMP::UnsignedOnly)
3469 C.Opcode = SystemZISD::UCMP128HI;
3470 else
3471 C.Opcode = SystemZISD::SCMP128HI;
3472 C.CCValid = SystemZ::CCMASK_ANY;
3473 C.CCMask = SystemZ::CCMASK_1;
3474
3475 if (Invert)
3476 C.CCMask ^= C.CCValid;
3477}
3478
3479// See whether the comparison argument contains a redundant AND
3480// and remove it if so. This sometimes happens due to the generic
3481// BRCOND expansion.
3482static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
3483 Comparison &C) {
3484 if (C.Op0.getOpcode() != ISD::AND)
3485 return;
3486 auto *Mask = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3487 if (!Mask || Mask->getValueSizeInBits(ResNo: 0) > 64)
3488 return;
3489 KnownBits Known = DAG.computeKnownBits(Op: C.Op0.getOperand(i: 0));
3490 if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
3491 return;
3492
3493 C.Op0 = C.Op0.getOperand(i: 0);
3494}
3495
3496// Return a Comparison that tests the condition-code result of intrinsic
3497// node Call against constant integer CC using comparison code Cond.
3498// Opcode is the opcode of the SystemZISD operation for the intrinsic
3499// and CCValid is the set of possible condition-code results.
3500static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
3501 SDValue Call, unsigned CCValid, uint64_t CC,
3502 ISD::CondCode Cond) {
3503 Comparison C(Call, SDValue(), SDValue());
3504 C.Opcode = Opcode;
3505 C.CCValid = CCValid;
3506 if (Cond == ISD::SETEQ)
3507 // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3.
3508 C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
3509 else if (Cond == ISD::SETNE)
3510 // ...and the inverse of that.
3511 C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
3512 else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
3513 // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
3514 // always true for CC>3.
3515 C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
3516 else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
3517 // ...and the inverse of that.
3518 C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
3519 else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
3520 // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
3521 // always true for CC>3.
3522 C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
3523 else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
3524 // ...and the inverse of that.
3525 C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
3526 else
3527 llvm_unreachable("Unexpected integer comparison type");
3528 C.CCMask &= CCValid;
3529 return C;
3530}
3531
3532// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
3533static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
3534 ISD::CondCode Cond, const SDLoc &DL,
3535 SDValue Chain = SDValue(),
3536 bool IsSignaling = false) {
3537 if (CmpOp1.getOpcode() == ISD::Constant) {
3538 assert(!Chain);
3539 unsigned Opcode, CCValid;
3540 if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
3541 CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(NUses: 1, Value: 0) &&
3542 isIntrinsicWithCCAndChain(Op: CmpOp0, Opcode, CCValid))
3543 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3544 CC: CmpOp1->getAsZExtVal(), Cond);
3545 if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
3546 CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
3547 isIntrinsicWithCC(Op: CmpOp0, Opcode, CCValid))
3548 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3549 CC: CmpOp1->getAsZExtVal(), Cond);
3550 }
3551 Comparison C(CmpOp0, CmpOp1, Chain);
3552 C.CCMask = CCMaskForCondCode(CC: Cond);
3553 if (C.Op0.getValueType().isFloatingPoint()) {
3554 C.CCValid = SystemZ::CCMASK_FCMP;
3555 if (!C.Chain)
3556 C.Opcode = SystemZISD::FCMP;
3557 else if (!IsSignaling)
3558 C.Opcode = SystemZISD::STRICT_FCMP;
3559 else
3560 C.Opcode = SystemZISD::STRICT_FCMPS;
3561 adjustForFNeg(C);
3562 } else {
3563 assert(!C.Chain);
3564 C.CCValid = SystemZ::CCMASK_ICMP;
3565 C.Opcode = SystemZISD::ICMP;
3566 // Choose the type of comparison. Equality and inequality tests can
3567 // use either signed or unsigned comparisons. The choice also doesn't
3568 // matter if both sign bits are known to be clear. In those cases we
3569 // want to give the main isel code the freedom to choose whichever
3570 // form fits best.
3571 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3572 C.CCMask == SystemZ::CCMASK_CMP_NE ||
3573 (DAG.SignBitIsZero(Op: C.Op0) && DAG.SignBitIsZero(Op: C.Op1)))
3574 C.ICmpType = SystemZICMP::Any;
3575 else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
3576 C.ICmpType = SystemZICMP::UnsignedOnly;
3577 else
3578 C.ICmpType = SystemZICMP::SignedOnly;
3579 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
3580 adjustForRedundantAnd(DAG, DL, C);
3581 adjustZeroCmp(DAG, DL, C);
3582 adjustSubwordCmp(DAG, DL, C);
3583 adjustForSubtraction(DAG, DL, C);
3584 adjustForLTGFR(C);
3585 adjustICmpTruncate(DAG, DL, C);
3586 }
3587
3588 if (shouldSwapCmpOperands(C)) {
3589 std::swap(a&: C.Op0, b&: C.Op1);
3590 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3591 }
3592
3593 adjustForTestUnderMask(DAG, DL, C);
3594 adjustICmp128(DAG, DL, C);
3595 return C;
3596}
3597
3598// Emit the comparison instruction described by C.
3599static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
3600 if (!C.Op1.getNode()) {
3601 SDNode *Node;
3602 switch (C.Op0.getOpcode()) {
3603 case ISD::INTRINSIC_W_CHAIN:
3604 Node = emitIntrinsicWithCCAndChain(DAG, Op: C.Op0, Opcode: C.Opcode);
3605 return SDValue(Node, 0);
3606 case ISD::INTRINSIC_WO_CHAIN:
3607 Node = emitIntrinsicWithCC(DAG, Op: C.Op0, Opcode: C.Opcode);
3608 return SDValue(Node, Node->getNumValues() - 1);
3609 default:
3610 llvm_unreachable("Invalid comparison operands");
3611 }
3612 }
3613 if (C.Opcode == SystemZISD::ICMP)
3614 return DAG.getNode(Opcode: SystemZISD::ICMP, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3615 N3: DAG.getTargetConstant(Val: C.ICmpType, DL, VT: MVT::i32));
3616 if (C.Opcode == SystemZISD::TM) {
3617 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
3618 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
3619 return DAG.getNode(Opcode: SystemZISD::TM, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3620 N3: DAG.getTargetConstant(Val: RegisterOnly, DL, VT: MVT::i32));
3621 }
3622 if (C.Opcode == SystemZISD::VICMPES ||
3623 C.Opcode == SystemZISD::VICMPHS ||
3624 C.Opcode == SystemZISD::VICMPHLS ||
3625 C.Opcode == SystemZISD::VFCMPES ||
3626 C.Opcode == SystemZISD::VFCMPHS ||
3627 C.Opcode == SystemZISD::VFCMPHES) {
3628 EVT IntVT = C.Op0.getValueType().changeVectorElementTypeToInteger();
3629 SDVTList VTs = DAG.getVTList(VT1: IntVT, VT2: MVT::i32);
3630 SDValue Val = DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Op0, N2: C.Op1);
3631 return SDValue(Val.getNode(), 1);
3632 }
3633 if (C.Chain) {
3634 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
3635 return DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Chain, N2: C.Op0, N3: C.Op1);
3636 }
3637 return DAG.getNode(Opcode: C.Opcode, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1);
3638}
3639
3640// Implement a 32-bit *MUL_LOHI operation by extending both operands to
3641// 64 bits. Extend is the extension type to use. Store the high part
3642// in Hi and the low part in Lo.
3643static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
3644 SDValue Op0, SDValue Op1, SDValue &Hi,
3645 SDValue &Lo) {
3646 Op0 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op0);
3647 Op1 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op1);
3648 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
3649 Hi = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Mul,
3650 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3651 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Hi);
3652 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Mul);
3653}
3654
3655// Lower a binary operation that produces two VT results, one in each
3656// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation,
3657// and Opcode performs the GR128 operation. Store the even register result
3658// in Even and the odd register result in Odd.
3659static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
3660 unsigned Opcode, SDValue Op0, SDValue Op1,
3661 SDValue &Even, SDValue &Odd) {
3662 SDValue Result = DAG.getNode(Opcode, DL, VT: MVT::Untyped, N1: Op0, N2: Op1);
3663 bool Is32Bit = is32Bit(VT);
3664 Even = DAG.getTargetExtractSubreg(SRIdx: SystemZ::even128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3665 Odd = DAG.getTargetExtractSubreg(SRIdx: SystemZ::odd128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3666}
3667
3668// Return an i32 value that is 1 if the CC value produced by CCReg is
3669// in the mask CCMask and 0 otherwise. CC is known to have a value
3670// in CCValid, so other values can be ignored.
3671static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
3672 unsigned CCValid, unsigned CCMask) {
3673 SDValue Ops[] = {DAG.getConstant(Val: 1, DL, VT: MVT::i32),
3674 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
3675 DAG.getTargetConstant(Val: CCValid, DL, VT: MVT::i32),
3676 DAG.getTargetConstant(Val: CCMask, DL, VT: MVT::i32), CCReg};
3677 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: MVT::i32, Ops);
3678}
3679
3680// Return the SystemISD vector comparison operation for CC, or 0 if it cannot
3681// be done directly. Mode is CmpMode::Int for integer comparisons, CmpMode::FP
3682// for regular floating-point comparisons, CmpMode::StrictFP for strict (quiet)
3683// floating-point comparisons, and CmpMode::SignalingFP for strict signaling
3684// floating-point comparisons.
3685enum class CmpMode { Int, FP, StrictFP, SignalingFP };
3686static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode) {
3687 switch (CC) {
3688 case ISD::SETOEQ:
3689 case ISD::SETEQ:
3690 switch (Mode) {
3691 case CmpMode::Int: return SystemZISD::VICMPE;
3692 case CmpMode::FP: return SystemZISD::VFCMPE;
3693 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPE;
3694 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPES;
3695 }
3696 llvm_unreachable("Bad mode");
3697
3698 case ISD::SETOGE:
3699 case ISD::SETGE:
3700 switch (Mode) {
3701 case CmpMode::Int: return 0;
3702 case CmpMode::FP: return SystemZISD::VFCMPHE;
3703 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPHE;
3704 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHES;
3705 }
3706 llvm_unreachable("Bad mode");
3707
3708 case ISD::SETOGT:
3709 case ISD::SETGT:
3710 switch (Mode) {
3711 case CmpMode::Int: return SystemZISD::VICMPH;
3712 case CmpMode::FP: return SystemZISD::VFCMPH;
3713 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPH;
3714 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHS;
3715 }
3716 llvm_unreachable("Bad mode");
3717
3718 case ISD::SETUGT:
3719 switch (Mode) {
3720 case CmpMode::Int: return SystemZISD::VICMPHL;
3721 case CmpMode::FP: return 0;
3722 case CmpMode::StrictFP: return 0;
3723 case CmpMode::SignalingFP: return 0;
3724 }
3725 llvm_unreachable("Bad mode");
3726
3727 default:
3728 return 0;
3729 }
3730}
3731
3732// Return the SystemZISD vector comparison operation for CC or its inverse,
3733// or 0 if neither can be done directly. Indicate in Invert whether the
3734// result is for the inverse of CC. Mode is as above.
3735static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode,
3736 bool &Invert) {
3737 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3738 Invert = false;
3739 return Opcode;
3740 }
3741
3742 CC = ISD::getSetCCInverse(Operation: CC, Type: Mode == CmpMode::Int ? MVT::i32 : MVT::f32);
3743 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3744 Invert = true;
3745 return Opcode;
3746 }
3747
3748 return 0;
3749}
3750
3751// Return a v2f64 that contains the extended form of elements Start and Start+1
3752// of v4f32 value Op. If Chain is nonnull, return the strict form.
3753static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
3754 SDValue Op, SDValue Chain) {
3755 int Mask[] = { Start, -1, Start + 1, -1 };
3756 Op = DAG.getVectorShuffle(VT: MVT::v4f32, dl: DL, N1: Op, N2: DAG.getUNDEF(VT: MVT::v4f32), Mask);
3757 if (Chain) {
3758 SDVTList VTs = DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other);
3759 return DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL, VTList: VTs, N1: Chain, N2: Op);
3760 }
3761 return DAG.getNode(Opcode: SystemZISD::VEXTEND, DL, VT: MVT::v2f64, Operand: Op);
3762}
3763
3764// Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
3765// producing a result of type VT. If Chain is nonnull, return the strict form.
3766SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
3767 const SDLoc &DL, EVT VT,
3768 SDValue CmpOp0,
3769 SDValue CmpOp1,
3770 SDValue Chain) const {
3771 // There is no hardware support for v4f32 (unless we have the vector
3772 // enhancements facility 1), so extend the vector into two v2f64s
3773 // and compare those.
3774 if (CmpOp0.getValueType() == MVT::v4f32 &&
3775 !Subtarget.hasVectorEnhancements1()) {
3776 SDValue H0 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp0, Chain);
3777 SDValue L0 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp0, Chain);
3778 SDValue H1 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp1, Chain);
3779 SDValue L1 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp1, Chain);
3780 if (Chain) {
3781 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i64, VT2: MVT::Other);
3782 SDValue HRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: H0, N3: H1);
3783 SDValue LRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: L0, N3: L1);
3784 SDValue Res = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3785 SDValue Chains[6] = { H0.getValue(R: 1), L0.getValue(R: 1),
3786 H1.getValue(R: 1), L1.getValue(R: 1),
3787 HRes.getValue(R: 1), LRes.getValue(R: 1) };
3788 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3789 SDValue Ops[2] = { Res, NewChain };
3790 return DAG.getMergeValues(Ops, dl: DL);
3791 }
3792 SDValue HRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: H0, N2: H1);
3793 SDValue LRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: L0, N2: L1);
3794 return DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3795 }
3796 if (Chain) {
3797 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::Other);
3798 return DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: CmpOp0, N3: CmpOp1);
3799 }
3800 return DAG.getNode(Opcode, DL, VT, N1: CmpOp0, N2: CmpOp1);
3801}
3802
3803// Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
3804// an integer mask of type VT. If Chain is nonnull, we have a strict
3805// floating-point comparison. If in addition IsSignaling is true, we have
3806// a strict signaling floating-point comparison.
3807SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
3808 const SDLoc &DL, EVT VT,
3809 ISD::CondCode CC,
3810 SDValue CmpOp0,
3811 SDValue CmpOp1,
3812 SDValue Chain,
3813 bool IsSignaling) const {
3814 bool IsFP = CmpOp0.getValueType().isFloatingPoint();
3815 assert (!Chain || IsFP);
3816 assert (!IsSignaling || Chain);
3817 CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
3818 Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
3819 bool Invert = false;
3820 SDValue Cmp;
3821 switch (CC) {
3822 // Handle tests for order using (or (ogt y x) (oge x y)).
3823 case ISD::SETUO:
3824 Invert = true;
3825 [[fallthrough]];
3826 case ISD::SETO: {
3827 assert(IsFP && "Unexpected integer comparison");
3828 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3829 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3830 SDValue GE = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGE, Mode),
3831 DL, VT, CmpOp0, CmpOp1, Chain);
3832 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GE);
3833 if (Chain)
3834 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
3835 N1: LT.getValue(R: 1), N2: GE.getValue(R: 1));
3836 break;
3837 }
3838
3839 // Handle <> tests using (or (ogt y x) (ogt x y)).
3840 case ISD::SETUEQ:
3841 Invert = true;
3842 [[fallthrough]];
3843 case ISD::SETONE: {
3844 assert(IsFP && "Unexpected integer comparison");
3845 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3846 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3847 SDValue GT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3848 DL, VT, CmpOp0, CmpOp1, Chain);
3849 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GT);
3850 if (Chain)
3851 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
3852 N1: LT.getValue(R: 1), N2: GT.getValue(R: 1));
3853 break;
3854 }
3855
3856 // Otherwise a single comparison is enough. It doesn't really
3857 // matter whether we try the inversion or the swap first, since
3858 // there are no cases where both work.
3859 default:
3860 // Optimize sign-bit comparisons to signed compares.
3861 if (Mode == CmpMode::Int && (CC == ISD::SETEQ || CC == ISD::SETNE) &&
3862 ISD::isConstantSplatVectorAllZeros(N: CmpOp1.getNode())) {
3863 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
3864 APInt Mask;
3865 if (CmpOp0.getOpcode() == ISD::AND
3866 && ISD::isConstantSplatVector(N: CmpOp0.getOperand(i: 1).getNode(), SplatValue&: Mask)
3867 && Mask == APInt::getSignMask(BitWidth: EltSize)) {
3868 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
3869 CmpOp0 = CmpOp0.getOperand(i: 0);
3870 }
3871 }
3872 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3873 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
3874 else {
3875 CC = ISD::getSetCCSwappedOperands(Operation: CC);
3876 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3877 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3878 else
3879 llvm_unreachable("Unhandled comparison");
3880 }
3881 if (Chain)
3882 Chain = Cmp.getValue(R: 1);
3883 break;
3884 }
3885 if (Invert) {
3886 SDValue Mask =
3887 DAG.getSplatBuildVector(VT, DL, Op: DAG.getAllOnesConstant(DL, VT: MVT::i64));
3888 Cmp = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Cmp, N2: Mask);
3889 }
3890 if (Chain && Chain.getNode() != Cmp.getNode()) {
3891 SDValue Ops[2] = { Cmp, Chain };
3892 Cmp = DAG.getMergeValues(Ops, dl: DL);
3893 }
3894 return Cmp;
3895}
3896
3897SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
3898 SelectionDAG &DAG) const {
3899 SDValue CmpOp0 = Op.getOperand(i: 0);
3900 SDValue CmpOp1 = Op.getOperand(i: 1);
3901 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
3902 SDLoc DL(Op);
3903 EVT VT = Op.getValueType();
3904 if (VT.isVector())
3905 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
3906
3907 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
3908 SDValue CCReg = emitCmp(DAG, DL, C);
3909 return emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
3910}
3911
3912SDValue SystemZTargetLowering::lowerSTRICT_FSETCC(SDValue Op,
3913 SelectionDAG &DAG,
3914 bool IsSignaling) const {
3915 SDValue Chain = Op.getOperand(i: 0);
3916 SDValue CmpOp0 = Op.getOperand(i: 1);
3917 SDValue CmpOp1 = Op.getOperand(i: 2);
3918 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 3))->get();
3919 SDLoc DL(Op);
3920 EVT VT = Op.getNode()->getValueType(ResNo: 0);
3921 if (VT.isVector()) {
3922 SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
3923 Chain, IsSignaling);
3924 return Res.getValue(R: Op.getResNo());
3925 }
3926
3927 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL, Chain, IsSignaling));
3928 SDValue CCReg = emitCmp(DAG, DL, C);
3929 CCReg->setFlags(Op->getFlags());
3930 SDValue Result = emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
3931 SDValue Ops[2] = { Result, CCReg.getValue(R: 1) };
3932 return DAG.getMergeValues(Ops, dl: DL);
3933}
3934
3935SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
3936 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
3937 SDValue CmpOp0 = Op.getOperand(i: 2);
3938 SDValue CmpOp1 = Op.getOperand(i: 3);
3939 SDValue Dest = Op.getOperand(i: 4);
3940 SDLoc DL(Op);
3941
3942 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
3943 SDValue CCReg = emitCmp(DAG, DL, C);
3944 return DAG.getNode(
3945 Opcode: SystemZISD::BR_CCMASK, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
3946 N2: DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
3947 N3: DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), N4: Dest, N5: CCReg);
3948}
3949
3950// Return true if Pos is CmpOp and Neg is the negative of CmpOp,
3951// allowing Pos and Neg to be wider than CmpOp.
3952static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
3953 return (Neg.getOpcode() == ISD::SUB &&
3954 Neg.getOperand(i: 0).getOpcode() == ISD::Constant &&
3955 Neg.getConstantOperandVal(i: 0) == 0 && Neg.getOperand(i: 1) == Pos &&
3956 (Pos == CmpOp || (Pos.getOpcode() == ISD::SIGN_EXTEND &&
3957 Pos.getOperand(i: 0) == CmpOp)));
3958}
3959
3960// Return the absolute or negative absolute of Op; IsNegative decides which.
3961static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
3962 bool IsNegative) {
3963 Op = DAG.getNode(Opcode: ISD::ABS, DL, VT: Op.getValueType(), Operand: Op);
3964 if (IsNegative)
3965 Op = DAG.getNode(Opcode: ISD::SUB, DL, VT: Op.getValueType(),
3966 N1: DAG.getConstant(Val: 0, DL, VT: Op.getValueType()), N2: Op);
3967 return Op;
3968}
3969
3970static SDValue getI128Select(SelectionDAG &DAG, const SDLoc &DL,
3971 Comparison C, SDValue TrueOp, SDValue FalseOp) {
3972 EVT VT = MVT::i128;
3973 unsigned Op;
3974
3975 if (C.CCMask == SystemZ::CCMASK_CMP_NE ||
3976 C.CCMask == SystemZ::CCMASK_CMP_GE ||
3977 C.CCMask == SystemZ::CCMASK_CMP_LE) {
3978 std::swap(a&: TrueOp, b&: FalseOp);
3979 C.CCMask ^= C.CCValid;
3980 }
3981 if (C.CCMask == SystemZ::CCMASK_CMP_LT) {
3982 std::swap(a&: C.Op0, b&: C.Op1);
3983 C.CCMask = SystemZ::CCMASK_CMP_GT;
3984 }
3985 switch (C.CCMask) {
3986 case SystemZ::CCMASK_CMP_EQ:
3987 Op = SystemZISD::VICMPE;
3988 break;
3989 case SystemZ::CCMASK_CMP_GT:
3990 if (C.ICmpType == SystemZICMP::UnsignedOnly)
3991 Op = SystemZISD::VICMPHL;
3992 else
3993 Op = SystemZISD::VICMPH;
3994 break;
3995 default:
3996 llvm_unreachable("Unhandled comparison");
3997 break;
3998 }
3999
4000 SDValue Mask = DAG.getNode(Opcode: Op, DL, VT, N1: C.Op0, N2: C.Op1);
4001 TrueOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: TrueOp, N2: Mask);
4002 FalseOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: FalseOp, N2: DAG.getNOT(DL, Val: Mask, VT));
4003 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: TrueOp, N2: FalseOp);
4004}
4005
4006SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
4007 SelectionDAG &DAG) const {
4008 SDValue CmpOp0 = Op.getOperand(i: 0);
4009 SDValue CmpOp1 = Op.getOperand(i: 1);
4010 SDValue TrueOp = Op.getOperand(i: 2);
4011 SDValue FalseOp = Op.getOperand(i: 3);
4012 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
4013 SDLoc DL(Op);
4014
4015 // SELECT_CC involving f16 will not have the cmp-ops promoted by the
4016 // legalizer, as it will be handled according to the type of the resulting
4017 // value. Extend them here if needed.
4018 if (CmpOp0.getSimpleValueType() == MVT::f16) {
4019 CmpOp0 = DAG.getFPExtendOrRound(Op: CmpOp0, DL: SDLoc(CmpOp0), VT: MVT::f32);
4020 CmpOp1 = DAG.getFPExtendOrRound(Op: CmpOp1, DL: SDLoc(CmpOp1), VT: MVT::f32);
4021 }
4022
4023 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
4024
4025 // Check for absolute and negative-absolute selections, including those
4026 // where the comparison value is sign-extended (for LPGFR and LNGFR).
4027 // This check supplements the one in DAGCombiner.
4028 if (C.Opcode == SystemZISD::ICMP && C.CCMask != SystemZ::CCMASK_CMP_EQ &&
4029 C.CCMask != SystemZ::CCMASK_CMP_NE &&
4030 C.Op1.getOpcode() == ISD::Constant &&
4031 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
4032 C.Op1->getAsZExtVal() == 0) {
4033 if (isAbsolute(CmpOp: C.Op0, Pos: TrueOp, Neg: FalseOp))
4034 return getAbsolute(DAG, DL, Op: TrueOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_LT);
4035 if (isAbsolute(CmpOp: C.Op0, Pos: FalseOp, Neg: TrueOp))
4036 return getAbsolute(DAG, DL, Op: FalseOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_GT);
4037 }
4038
4039 if (Subtarget.hasVectorEnhancements3() &&
4040 C.Opcode == SystemZISD::ICMP &&
4041 C.Op0.getValueType() == MVT::i128 &&
4042 TrueOp.getValueType() == MVT::i128) {
4043 return getI128Select(DAG, DL, C, TrueOp, FalseOp);
4044 }
4045
4046 SDValue CCReg = emitCmp(DAG, DL, C);
4047 SDValue Ops[] = {TrueOp, FalseOp,
4048 DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
4049 DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), CCReg};
4050
4051 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: Op.getValueType(), Ops);
4052}
4053
4054SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
4055 SelectionDAG &DAG) const {
4056 SDLoc DL(Node);
4057 const GlobalValue *GV = Node->getGlobal();
4058 int64_t Offset = Node->getOffset();
4059 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4060 CodeModel::Model CM = DAG.getTarget().getCodeModel();
4061
4062 SDValue Result;
4063 if (Subtarget.isPC32DBLSymbol(GV, CM)) {
4064 if (isInt<32>(x: Offset)) {
4065 // Assign anchors at 1<<12 byte boundaries.
4066 uint64_t Anchor = Offset & ~uint64_t(0xfff);
4067 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor);
4068 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4069
4070 // The offset can be folded into the address if it is aligned to a
4071 // halfword.
4072 Offset -= Anchor;
4073 if (Offset != 0 && (Offset & 1) == 0) {
4074 SDValue Full =
4075 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor + Offset);
4076 Result = DAG.getNode(Opcode: SystemZISD::PCREL_OFFSET, DL, VT: PtrVT, N1: Full, N2: Result);
4077 Offset = 0;
4078 }
4079 } else {
4080 // Conservatively load a constant offset greater than 32 bits into a
4081 // register below.
4082 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT);
4083 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4084 }
4085 } else if (Subtarget.isTargetELF()) {
4086 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_GOT);
4087 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4088 Result = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result,
4089 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4090 } else if (Subtarget.isTargetzOS()) {
4091 Result = getADAEntry(DAG, GV, DL, PtrVT);
4092 } else
4093 llvm_unreachable("Unexpected Subtarget");
4094
4095 // If there was a non-zero offset that we didn't fold, create an explicit
4096 // addition for it.
4097 if (Offset != 0)
4098 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4099 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4100
4101 return Result;
4102}
4103
4104SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
4105 SelectionDAG &DAG,
4106 unsigned Opcode,
4107 SDValue GOTOffset) const {
4108 SDLoc DL(Node);
4109 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4110 SDValue Chain = DAG.getEntryNode();
4111 SDValue Glue;
4112
4113 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4114 CallingConv::GHC)
4115 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4116
4117 // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12.
4118 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(VT: PtrVT);
4119 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R12D, N: GOT, Glue);
4120 Glue = Chain.getValue(R: 1);
4121 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R2D, N: GOTOffset, Glue);
4122 Glue = Chain.getValue(R: 1);
4123
4124 // The first call operand is the chain and the second is the TLS symbol.
4125 SmallVector<SDValue, 8> Ops;
4126 Ops.push_back(Elt: Chain);
4127 Ops.push_back(Elt: DAG.getTargetGlobalAddress(GV: Node->getGlobal(), DL,
4128 VT: Node->getValueType(ResNo: 0),
4129 offset: 0, TargetFlags: 0));
4130
4131 // Add argument registers to the end of the list so that they are
4132 // known live into the call.
4133 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R2D, VT: PtrVT));
4134 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R12D, VT: PtrVT));
4135
4136 // Add a register mask operand representing the call-preserved registers.
4137 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
4138 const uint32_t *Mask =
4139 TRI->getCallPreservedMask(MF: DAG.getMachineFunction(), CallingConv::C);
4140 assert(Mask && "Missing call preserved mask for calling convention");
4141 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
4142
4143 // Glue the call to the argument copies.
4144 Ops.push_back(Elt: Glue);
4145
4146 // Emit the call.
4147 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
4148 Chain = DAG.getNode(Opcode, DL, VTList: NodeTys, Ops);
4149 Glue = Chain.getValue(R: 1);
4150
4151 // Copy the return value from %r2.
4152 return DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::R2D, VT: PtrVT, Glue);
4153}
4154
4155SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
4156 SelectionDAG &DAG) const {
4157 SDValue Chain = DAG.getEntryNode();
4158 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4159
4160 // The high part of the thread pointer is in access register 0.
4161 SDValue TPHi = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A0, VT: MVT::i32);
4162 TPHi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PtrVT, Operand: TPHi);
4163
4164 // The low part of the thread pointer is in access register 1.
4165 SDValue TPLo = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A1, VT: MVT::i32);
4166 TPLo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: PtrVT, Operand: TPLo);
4167
4168 // Merge them into a single 64-bit address.
4169 SDValue TPHiShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: TPHi,
4170 N2: DAG.getConstant(Val: 32, DL, VT: PtrVT));
4171 return DAG.getNode(Opcode: ISD::OR, DL, VT: PtrVT, N1: TPHiShifted, N2: TPLo);
4172}
4173
4174SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
4175 SelectionDAG &DAG) const {
4176 if (DAG.getTarget().useEmulatedTLS())
4177 return LowerToTLSEmulatedModel(GA: Node, DAG);
4178 SDLoc DL(Node);
4179 const GlobalValue *GV = Node->getGlobal();
4180 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4181 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
4182
4183 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4184 CallingConv::GHC)
4185 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4186
4187 SDValue TP = lowerThreadPointer(DL, DAG);
4188
4189 // Get the offset of GA from the thread pointer, based on the TLS model.
4190 SDValue Offset;
4191 switch (model) {
4192 case TLSModel::GeneralDynamic: {
4193 // Load the GOT offset of the tls_index (module ID / per-symbol offset).
4194 SystemZConstantPoolValue *CPV =
4195 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSGD);
4196
4197 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4198 Offset = DAG.getLoad(
4199 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4200 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4201
4202 // Call __tls_get_offset to retrieve the offset.
4203 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_GDCALL, GOTOffset: Offset);
4204 break;
4205 }
4206
4207 case TLSModel::LocalDynamic: {
4208 // Load the GOT offset of the module ID.
4209 SystemZConstantPoolValue *CPV =
4210 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSLDM);
4211
4212 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4213 Offset = DAG.getLoad(
4214 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4215 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4216
4217 // Call __tls_get_offset to retrieve the module base offset.
4218 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_LDCALL, GOTOffset: Offset);
4219
4220 // Note: The SystemZLDCleanupPass will remove redundant computations
4221 // of the module base offset. Count total number of local-dynamic
4222 // accesses to trigger execution of that pass.
4223 SystemZMachineFunctionInfo* MFI =
4224 DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>();
4225 MFI->incNumLocalDynamicTLSAccesses();
4226
4227 // Add the per-symbol offset.
4228 CPV = SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::DTPOFF);
4229
4230 SDValue DTPOffset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4231 DTPOffset = DAG.getLoad(
4232 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: DTPOffset,
4233 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4234
4235 Offset = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Offset, N2: DTPOffset);
4236 break;
4237 }
4238
4239 case TLSModel::InitialExec: {
4240 // Load the offset from the GOT.
4241 Offset = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0,
4242 TargetFlags: SystemZII::MO_INDNTPOFF);
4243 Offset = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Offset);
4244 Offset =
4245 DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4246 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4247 break;
4248 }
4249
4250 case TLSModel::LocalExec: {
4251 // Force the offset into the constant pool and load it from there.
4252 SystemZConstantPoolValue *CPV =
4253 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::NTPOFF);
4254
4255 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4256 Offset = DAG.getLoad(
4257 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4258 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4259 break;
4260 }
4261 }
4262
4263 // Add the base and offset together.
4264 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TP, N2: Offset);
4265}
4266
4267SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
4268 SelectionDAG &DAG) const {
4269 SDLoc DL(Node);
4270 const BlockAddress *BA = Node->getBlockAddress();
4271 int64_t Offset = Node->getOffset();
4272 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4273
4274 SDValue Result = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset);
4275 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4276 return Result;
4277}
4278
4279SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
4280 SelectionDAG &DAG) const {
4281 SDLoc DL(JT);
4282 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4283 SDValue Result = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT);
4284
4285 // Use LARL to load the address of the table.
4286 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4287}
4288
4289SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
4290 SelectionDAG &DAG) const {
4291 SDLoc DL(CP);
4292 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4293
4294 SDValue Result;
4295 if (CP->isMachineConstantPoolEntry())
4296 Result =
4297 DAG.getTargetConstantPool(C: CP->getMachineCPVal(), VT: PtrVT, Align: CP->getAlign());
4298 else
4299 Result = DAG.getTargetConstantPool(C: CP->getConstVal(), VT: PtrVT, Align: CP->getAlign(),
4300 Offset: CP->getOffset());
4301
4302 // Use LARL to load the address of the constant pool entry.
4303 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4304}
4305
4306SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
4307 SelectionDAG &DAG) const {
4308 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4309 MachineFunction &MF = DAG.getMachineFunction();
4310 MachineFrameInfo &MFI = MF.getFrameInfo();
4311 MFI.setFrameAddressIsTaken(true);
4312
4313 SDLoc DL(Op);
4314 unsigned Depth = Op.getConstantOperandVal(i: 0);
4315 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4316
4317 // By definition, the frame address is the address of the back chain. (In
4318 // the case of packed stack without backchain, return the address where the
4319 // backchain would have been stored. This will either be an unused space or
4320 // contain a saved register).
4321 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
4322 SDValue BackChain = DAG.getFrameIndex(FI: BackChainIdx, VT: PtrVT);
4323
4324 if (Depth > 0) {
4325 // FIXME The frontend should detect this case.
4326 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4327 report_fatal_error(reason: "Unsupported stack frame traversal count");
4328
4329 SDValue Offset = DAG.getConstant(Val: TFL->getBackchainOffset(MF), DL, VT: PtrVT);
4330 while (Depth--) {
4331 BackChain = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: BackChain,
4332 PtrInfo: MachinePointerInfo());
4333 BackChain = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: BackChain, N2: Offset);
4334 }
4335 }
4336
4337 return BackChain;
4338}
4339
4340SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
4341 SelectionDAG &DAG) const {
4342 MachineFunction &MF = DAG.getMachineFunction();
4343 MachineFrameInfo &MFI = MF.getFrameInfo();
4344 MFI.setReturnAddressIsTaken(true);
4345
4346 SDLoc DL(Op);
4347 unsigned Depth = Op.getConstantOperandVal(i: 0);
4348 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4349
4350 if (Depth > 0) {
4351 // FIXME The frontend should detect this case.
4352 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4353 report_fatal_error(reason: "Unsupported stack frame traversal count");
4354
4355 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
4356 const auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4357 int Offset = TFL->getReturnAddressOffset(MF);
4358 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FrameAddr,
4359 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4360 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr,
4361 PtrInfo: MachinePointerInfo());
4362 }
4363
4364 // Return R14D (Elf) / R7D (XPLINK), which has the return address. Mark it an
4365 // implicit live-in.
4366 SystemZCallingConventionRegisters *CCR = Subtarget.getSpecialRegisters();
4367 Register LinkReg = MF.addLiveIn(PReg: CCR->getReturnFunctionAddressRegister(),
4368 RC: &SystemZ::GR64BitRegClass);
4369 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: LinkReg, VT: PtrVT);
4370}
4371
4372SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
4373 SelectionDAG &DAG) const {
4374 SDLoc DL(Op);
4375 SDValue In = Op.getOperand(i: 0);
4376 EVT InVT = In.getValueType();
4377 EVT ResVT = Op.getValueType();
4378
4379 // Convert loads directly. This is normally done by DAGCombiner,
4380 // but we need this case for bitcasts that are created during lowering
4381 // and which are then lowered themselves.
4382 if (auto *LoadN = dyn_cast<LoadSDNode>(Val&: In))
4383 if (ISD::isNormalLoad(N: LoadN)) {
4384 SDValue NewLoad = DAG.getLoad(VT: ResVT, dl: DL, Chain: LoadN->getChain(),
4385 Ptr: LoadN->getBasePtr(), MMO: LoadN->getMemOperand());
4386 // Update the chain uses.
4387 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LoadN, 1), To: NewLoad.getValue(R: 1));
4388 return NewLoad;
4389 }
4390
4391 if (InVT == MVT::i32 && ResVT == MVT::f32) {
4392 SDValue In64;
4393 if (Subtarget.hasHighWord()) {
4394 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL,
4395 VT: MVT::i64);
4396 In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4397 VT: MVT::i64, Operand: SDValue(U64, 0), Subreg: In);
4398 } else {
4399 In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: In);
4400 In64 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: In64,
4401 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4402 }
4403 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: In64);
4404 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32,
4405 DL, VT: MVT::f32, Operand: Out64);
4406 }
4407 if (InVT == MVT::f32 && ResVT == MVT::i32) {
4408 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
4409 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4410 VT: MVT::f64, Operand: SDValue(U64, 0), Subreg: In);
4411 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
4412 if (Subtarget.hasHighWord())
4413 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32, DL,
4414 VT: MVT::i32, Operand: Out64);
4415 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Out64,
4416 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4417 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Shift);
4418 }
4419 llvm_unreachable("Unexpected bitcast combination");
4420}
4421
4422SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
4423 SelectionDAG &DAG) const {
4424
4425 if (Subtarget.isTargetXPLINK64())
4426 return lowerVASTART_XPLINK(Op, DAG);
4427 else
4428 return lowerVASTART_ELF(Op, DAG);
4429}
4430
4431SDValue SystemZTargetLowering::lowerVASTART_XPLINK(SDValue Op,
4432 SelectionDAG &DAG) const {
4433 MachineFunction &MF = DAG.getMachineFunction();
4434 SystemZMachineFunctionInfo *FuncInfo =
4435 MF.getInfo<SystemZMachineFunctionInfo>();
4436
4437 SDLoc DL(Op);
4438
4439 // vastart just stores the address of the VarArgsFrameIndex slot into the
4440 // memory location argument.
4441 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4442 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4443 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4444 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FR, Ptr: Op.getOperand(i: 1),
4445 PtrInfo: MachinePointerInfo(SV));
4446}
4447
4448SDValue SystemZTargetLowering::lowerVASTART_ELF(SDValue Op,
4449 SelectionDAG &DAG) const {
4450 MachineFunction &MF = DAG.getMachineFunction();
4451 SystemZMachineFunctionInfo *FuncInfo =
4452 MF.getInfo<SystemZMachineFunctionInfo>();
4453 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4454
4455 SDValue Chain = Op.getOperand(i: 0);
4456 SDValue Addr = Op.getOperand(i: 1);
4457 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4458 SDLoc DL(Op);
4459
4460 // The initial values of each field.
4461 const unsigned NumFields = 4;
4462 SDValue Fields[NumFields] = {
4463 DAG.getConstant(Val: FuncInfo->getVarArgsFirstGPR(), DL, VT: PtrVT),
4464 DAG.getConstant(Val: FuncInfo->getVarArgsFirstFPR(), DL, VT: PtrVT),
4465 DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT),
4466 DAG.getFrameIndex(FI: FuncInfo->getRegSaveFrameIndex(), VT: PtrVT)
4467 };
4468
4469 // Store each field into its respective slot.
4470 SDValue MemOps[NumFields];
4471 unsigned Offset = 0;
4472 for (unsigned I = 0; I < NumFields; ++I) {
4473 SDValue FieldAddr = Addr;
4474 if (Offset != 0)
4475 FieldAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FieldAddr,
4476 N2: DAG.getIntPtrConstant(Val: Offset, DL));
4477 MemOps[I] = DAG.getStore(Chain, dl: DL, Val: Fields[I], Ptr: FieldAddr,
4478 PtrInfo: MachinePointerInfo(SV, Offset));
4479 Offset += 8;
4480 }
4481 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
4482}
4483
4484SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
4485 SelectionDAG &DAG) const {
4486 SDValue Chain = Op.getOperand(i: 0);
4487 SDValue DstPtr = Op.getOperand(i: 1);
4488 SDValue SrcPtr = Op.getOperand(i: 2);
4489 const Value *DstSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 3))->getValue();
4490 const Value *SrcSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
4491 SDLoc DL(Op);
4492
4493 uint32_t Sz =
4494 Subtarget.isTargetXPLINK64() ? getTargetMachine().getPointerSize(AS: 0) : 32;
4495 return DAG.getMemcpy(Chain, dl: DL, Dst: DstPtr, Src: SrcPtr, Size: DAG.getIntPtrConstant(Val: Sz, DL),
4496 Alignment: Align(8), /*isVolatile*/ isVol: false, /*AlwaysInline*/ false,
4497 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(DstSV),
4498 SrcPtrInfo: MachinePointerInfo(SrcSV));
4499}
4500
4501SDValue
4502SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
4503 SelectionDAG &DAG) const {
4504 if (Subtarget.isTargetXPLINK64())
4505 return lowerDYNAMIC_STACKALLOC_XPLINK(Op, DAG);
4506 else
4507 return lowerDYNAMIC_STACKALLOC_ELF(Op, DAG);
4508}
4509
4510SDValue
4511SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op,
4512 SelectionDAG &DAG) const {
4513 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4514 MachineFunction &MF = DAG.getMachineFunction();
4515 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4516 SDValue Chain = Op.getOperand(i: 0);
4517 SDValue Size = Op.getOperand(i: 1);
4518 SDValue Align = Op.getOperand(i: 2);
4519 SDLoc DL(Op);
4520
4521 // If user has set the no alignment function attribute, ignore
4522 // alloca alignments.
4523 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4524
4525 uint64_t StackAlign = TFI->getStackAlignment();
4526 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4527 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4528
4529 SDValue NeededSpace = Size;
4530
4531 // Add extra space for alignment if needed.
4532 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4533 if (ExtraAlignSpace)
4534 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: NeededSpace,
4535 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4536
4537 bool IsSigned = false;
4538 bool DoesNotReturn = false;
4539 bool IsReturnValueUsed = false;
4540 EVT VT = Op.getValueType();
4541 SDValue AllocaCall =
4542 makeExternalCall(Chain, DAG, CalleeName: "@@ALCAXP", RetVT: VT, Ops: ArrayRef(NeededSpace),
4543 CallConv: CallingConv::C, IsSigned, DL, DoesNotReturn,
4544 IsReturnValueUsed)
4545 .first;
4546
4547 // Perform a CopyFromReg from %GPR4 (stack pointer register). Chain and Glue
4548 // to end of call in order to ensure it isn't broken up from the call
4549 // sequence.
4550 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
4551 Register SPReg = Regs.getStackPointerRegister();
4552 Chain = AllocaCall.getValue(R: 1);
4553 SDValue Glue = AllocaCall.getValue(R: 2);
4554 SDValue NewSPRegNode = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: PtrVT, Glue);
4555 Chain = NewSPRegNode.getValue(R: 1);
4556
4557 MVT PtrMVT = getPointerMemTy(DL: MF.getDataLayout());
4558 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: PtrMVT);
4559 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrMVT, N1: NewSPRegNode, N2: ArgAdjust);
4560
4561 // Dynamically realign if needed.
4562 if (ExtraAlignSpace) {
4563 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4564 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4565 Result = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Result,
4566 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: PtrVT));
4567 }
4568
4569 SDValue Ops[2] = {Result, Chain};
4570 return DAG.getMergeValues(Ops, dl: DL);
4571}
4572
4573SDValue
4574SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(SDValue Op,
4575 SelectionDAG &DAG) const {
4576 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4577 MachineFunction &MF = DAG.getMachineFunction();
4578 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4579 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
4580
4581 SDValue Chain = Op.getOperand(i: 0);
4582 SDValue Size = Op.getOperand(i: 1);
4583 SDValue Align = Op.getOperand(i: 2);
4584 SDLoc DL(Op);
4585
4586 // If user has set the no alignment function attribute, ignore
4587 // alloca alignments.
4588 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4589
4590 uint64_t StackAlign = TFI->getStackAlignment();
4591 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4592 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4593
4594 Register SPReg = getStackPointerRegisterToSaveRestore();
4595 SDValue NeededSpace = Size;
4596
4597 // Get a reference to the stack pointer.
4598 SDValue OldSP = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: MVT::i64);
4599
4600 // If we need a backchain, save it now.
4601 SDValue Backchain;
4602 if (StoreBackchain)
4603 Backchain = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
4604 PtrInfo: MachinePointerInfo());
4605
4606 // Add extra space for alignment if needed.
4607 if (ExtraAlignSpace)
4608 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NeededSpace,
4609 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4610
4611 // Get the new stack pointer value.
4612 SDValue NewSP;
4613 if (hasInlineStackProbe(MF)) {
4614 NewSP = DAG.getNode(Opcode: SystemZISD::PROBED_ALLOCA, DL,
4615 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N1: Chain, N2: OldSP, N3: NeededSpace);
4616 Chain = NewSP.getValue(R: 1);
4617 }
4618 else {
4619 NewSP = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: OldSP, N2: NeededSpace);
4620 // Copy the new stack pointer back.
4621 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SPReg, N: NewSP);
4622 }
4623
4624 // The allocated data lives above the 160 bytes allocated for the standard
4625 // frame, plus any outgoing stack arguments. We don't know how much that
4626 // amounts to yet, so emit a special ADJDYNALLOC placeholder.
4627 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4628 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NewSP, N2: ArgAdjust);
4629
4630 // Dynamically realign if needed.
4631 if (RequiredAlign > StackAlign) {
4632 Result =
4633 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Result,
4634 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4635 Result =
4636 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, N1: Result,
4637 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: MVT::i64));
4638 }
4639
4640 if (StoreBackchain)
4641 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
4642 PtrInfo: MachinePointerInfo());
4643
4644 SDValue Ops[2] = { Result, Chain };
4645 return DAG.getMergeValues(Ops, dl: DL);
4646}
4647
4648SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
4649 SDValue Op, SelectionDAG &DAG) const {
4650 SDLoc DL(Op);
4651
4652 return DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4653}
4654
4655SDValue SystemZTargetLowering::lowerMULH(SDValue Op,
4656 SelectionDAG &DAG,
4657 unsigned Opcode) const {
4658 EVT VT = Op.getValueType();
4659 SDLoc DL(Op);
4660 SDValue Even, Odd;
4661
4662 // This custom expander is only used on z17 and later for 64-bit types.
4663 assert(!is32Bit(VT));
4664 assert(Subtarget.hasMiscellaneousExtensions2());
4665
4666 // SystemZISD::xMUL_LOHI returns the low result in the odd register and
4667 // the high result in the even register. Return the latter.
4668 lowerGR128Binary(DAG, DL, VT, Opcode,
4669 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even, Odd);
4670 return Even;
4671}
4672
4673SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
4674 SelectionDAG &DAG) const {
4675 EVT VT = Op.getValueType();
4676 SDLoc DL(Op);
4677 SDValue Ops[2];
4678 if (is32Bit(VT))
4679 // Just do a normal 64-bit multiplication and extract the results.
4680 // We define this so that it can be used for constant division.
4681 lowerMUL_LOHI32(DAG, DL, Extend: ISD::SIGN_EXTEND, Op0: Op.getOperand(i: 0),
4682 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4683 else if (Subtarget.hasMiscellaneousExtensions2())
4684 // SystemZISD::SMUL_LOHI returns the low result in the odd register and
4685 // the high result in the even register. ISD::SMUL_LOHI is defined to
4686 // return the low half first, so the results are in reverse order.
4687 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SMUL_LOHI,
4688 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4689 else {
4690 // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:
4691 //
4692 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64)
4693 //
4694 // but using the fact that the upper halves are either all zeros
4695 // or all ones:
4696 //
4697 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)
4698 //
4699 // and grouping the right terms together since they are quicker than the
4700 // multiplication:
4701 //
4702 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
4703 SDValue C63 = DAG.getConstant(Val: 63, DL, VT: MVT::i64);
4704 SDValue LL = Op.getOperand(i: 0);
4705 SDValue RL = Op.getOperand(i: 1);
4706 SDValue LH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LL, N2: C63);
4707 SDValue RH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: RL, N2: C63);
4708 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4709 // the high result in the even register. ISD::SMUL_LOHI is defined to
4710 // return the low half first, so the results are in reverse order.
4711 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4712 Op0: LL, Op1: RL, Even&: Ops[1], Odd&: Ops[0]);
4713 SDValue NegLLTimesRH = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LL, N2: RH);
4714 SDValue NegLHTimesRL = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LH, N2: RL);
4715 SDValue NegSum = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NegLLTimesRH, N2: NegLHTimesRL);
4716 Ops[1] = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Ops[1], N2: NegSum);
4717 }
4718 return DAG.getMergeValues(Ops, dl: DL);
4719}
4720
4721SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
4722 SelectionDAG &DAG) const {
4723 EVT VT = Op.getValueType();
4724 SDLoc DL(Op);
4725 SDValue Ops[2];
4726 if (is32Bit(VT))
4727 // Just do a normal 64-bit multiplication and extract the results.
4728 // We define this so that it can be used for constant division.
4729 lowerMUL_LOHI32(DAG, DL, Extend: ISD::ZERO_EXTEND, Op0: Op.getOperand(i: 0),
4730 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4731 else
4732 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4733 // the high result in the even register. ISD::UMUL_LOHI is defined to
4734 // return the low half first, so the results are in reverse order.
4735 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4736 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4737 return DAG.getMergeValues(Ops, dl: DL);
4738}
4739
4740SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
4741 SelectionDAG &DAG) const {
4742 SDValue Op0 = Op.getOperand(i: 0);
4743 SDValue Op1 = Op.getOperand(i: 1);
4744 EVT VT = Op.getValueType();
4745 SDLoc DL(Op);
4746
4747 // We use DSGF for 32-bit division. This means the first operand must
4748 // always be 64-bit, and the second operand should be 32-bit whenever
4749 // that is possible, to improve performance.
4750 if (is32Bit(VT))
4751 Op0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: Op0);
4752 else if (DAG.ComputeNumSignBits(Op: Op1) > 32)
4753 Op1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Op1);
4754
4755 // DSG(F) returns the remainder in the even register and the
4756 // quotient in the odd register.
4757 SDValue Ops[2];
4758 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SDIVREM, Op0, Op1, Even&: Ops[1], Odd&: Ops[0]);
4759 return DAG.getMergeValues(Ops, dl: DL);
4760}
4761
4762SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
4763 SelectionDAG &DAG) const {
4764 EVT VT = Op.getValueType();
4765 SDLoc DL(Op);
4766
4767 // DL(G) returns the remainder in the even register and the
4768 // quotient in the odd register.
4769 SDValue Ops[2];
4770 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UDIVREM,
4771 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4772 return DAG.getMergeValues(Ops, dl: DL);
4773}
4774
4775SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
4776 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");
4777
4778 // Get the known-zero masks for each operand.
4779 SDValue Ops[] = {Op.getOperand(i: 0), Op.getOperand(i: 1)};
4780 KnownBits Known[2] = {DAG.computeKnownBits(Op: Ops[0]),
4781 DAG.computeKnownBits(Op: Ops[1])};
4782
4783 // See if the upper 32 bits of one operand and the lower 32 bits of the
4784 // other are known zero. They are the low and high operands respectively.
4785 uint64_t Masks[] = { Known[0].Zero.getZExtValue(),
4786 Known[1].Zero.getZExtValue() };
4787 unsigned High, Low;
4788 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
4789 High = 1, Low = 0;
4790 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
4791 High = 0, Low = 1;
4792 else
4793 return Op;
4794
4795 SDValue LowOp = Ops[Low];
4796 SDValue HighOp = Ops[High];
4797
4798 // If the high part is a constant, we're better off using IILH.
4799 if (HighOp.getOpcode() == ISD::Constant)
4800 return Op;
4801
4802 // If the low part is a constant that is outside the range of LHI,
4803 // then we're better off using IILF.
4804 if (LowOp.getOpcode() == ISD::Constant) {
4805 int64_t Value = int32_t(LowOp->getAsZExtVal());
4806 if (!isInt<16>(x: Value))
4807 return Op;
4808 }
4809
4810 // Check whether the high part is an AND that doesn't change the
4811 // high 32 bits and just masks out low bits. We can skip it if so.
4812 if (HighOp.getOpcode() == ISD::AND &&
4813 HighOp.getOperand(i: 1).getOpcode() == ISD::Constant) {
4814 SDValue HighOp0 = HighOp.getOperand(i: 0);
4815 uint64_t Mask = HighOp.getConstantOperandVal(i: 1);
4816 if (DAG.MaskedValueIsZero(Op: HighOp0, Mask: APInt(64, ~(Mask | 0xffffffff))))
4817 HighOp = HighOp0;
4818 }
4819
4820 // Take advantage of the fact that all GR32 operations only change the
4821 // low 32 bits by truncating Low to an i32 and inserting it directly
4822 // using a subreg. The interesting cases are those where the truncation
4823 // can be folded.
4824 SDLoc DL(Op);
4825 SDValue Low32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: LowOp);
4826 return DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_l32, DL,
4827 VT: MVT::i64, Operand: HighOp, Subreg: Low32);
4828}
4829
4830// Lower SADDO/SSUBO/UADDO/USUBO nodes.
4831SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
4832 SelectionDAG &DAG) const {
4833 SDNode *N = Op.getNode();
4834 SDValue LHS = N->getOperand(Num: 0);
4835 SDValue RHS = N->getOperand(Num: 1);
4836 SDLoc DL(N);
4837
4838 if (N->getValueType(ResNo: 0) == MVT::i128) {
4839 unsigned BaseOp = 0;
4840 unsigned FlagOp = 0;
4841 bool IsBorrow = false;
4842 switch (Op.getOpcode()) {
4843 default: llvm_unreachable("Unknown instruction!");
4844 case ISD::UADDO:
4845 BaseOp = ISD::ADD;
4846 FlagOp = SystemZISD::VACC;
4847 break;
4848 case ISD::USUBO:
4849 BaseOp = ISD::SUB;
4850 FlagOp = SystemZISD::VSCBI;
4851 IsBorrow = true;
4852 break;
4853 }
4854 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
4855 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
4856 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
4857 N2: DAG.getValueType(MVT::i1));
4858 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
4859 if (IsBorrow)
4860 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
4861 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
4862 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
4863 }
4864
4865 unsigned BaseOp = 0;
4866 unsigned CCValid = 0;
4867 unsigned CCMask = 0;
4868
4869 switch (Op.getOpcode()) {
4870 default: llvm_unreachable("Unknown instruction!");
4871 case ISD::SADDO:
4872 BaseOp = SystemZISD::SADDO;
4873 CCValid = SystemZ::CCMASK_ARITH;
4874 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4875 break;
4876 case ISD::SSUBO:
4877 BaseOp = SystemZISD::SSUBO;
4878 CCValid = SystemZ::CCMASK_ARITH;
4879 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4880 break;
4881 case ISD::UADDO:
4882 BaseOp = SystemZISD::UADDO;
4883 CCValid = SystemZ::CCMASK_LOGICAL;
4884 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
4885 break;
4886 case ISD::USUBO:
4887 BaseOp = SystemZISD::USUBO;
4888 CCValid = SystemZ::CCMASK_LOGICAL;
4889 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
4890 break;
4891 }
4892
4893 SDVTList VTs = DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::i32);
4894 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS);
4895
4896 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
4897 if (N->getValueType(ResNo: 1) == MVT::i1)
4898 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
4899
4900 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
4901}
4902
4903static bool isAddCarryChain(SDValue Carry) {
4904 while (Carry.getOpcode() == ISD::UADDO_CARRY &&
4905 Carry->getValueType(ResNo: 0) != MVT::i128)
4906 Carry = Carry.getOperand(i: 2);
4907 return Carry.getOpcode() == ISD::UADDO &&
4908 Carry->getValueType(ResNo: 0) != MVT::i128;
4909}
4910
4911static bool isSubBorrowChain(SDValue Carry) {
4912 while (Carry.getOpcode() == ISD::USUBO_CARRY &&
4913 Carry->getValueType(ResNo: 0) != MVT::i128)
4914 Carry = Carry.getOperand(i: 2);
4915 return Carry.getOpcode() == ISD::USUBO &&
4916 Carry->getValueType(ResNo: 0) != MVT::i128;
4917}
4918
4919// Lower UADDO_CARRY/USUBO_CARRY nodes.
4920SDValue SystemZTargetLowering::lowerUADDSUBO_CARRY(SDValue Op,
4921 SelectionDAG &DAG) const {
4922
4923 SDNode *N = Op.getNode();
4924 MVT VT = N->getSimpleValueType(ResNo: 0);
4925
4926 // Let legalize expand this if it isn't a legal type yet.
4927 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
4928 return SDValue();
4929
4930 SDValue LHS = N->getOperand(Num: 0);
4931 SDValue RHS = N->getOperand(Num: 1);
4932 SDValue Carry = Op.getOperand(i: 2);
4933 SDLoc DL(N);
4934
4935 if (VT == MVT::i128) {
4936 unsigned BaseOp = 0;
4937 unsigned FlagOp = 0;
4938 bool IsBorrow = false;
4939 switch (Op.getOpcode()) {
4940 default: llvm_unreachable("Unknown instruction!");
4941 case ISD::UADDO_CARRY:
4942 BaseOp = SystemZISD::VAC;
4943 FlagOp = SystemZISD::VACCC;
4944 break;
4945 case ISD::USUBO_CARRY:
4946 BaseOp = SystemZISD::VSBI;
4947 FlagOp = SystemZISD::VSBCBI;
4948 IsBorrow = true;
4949 break;
4950 }
4951 if (IsBorrow)
4952 Carry = DAG.getNode(Opcode: ISD::XOR, DL, VT: Carry.getValueType(),
4953 N1: Carry, N2: DAG.getConstant(Val: 1, DL, VT: Carry.getValueType()));
4954 Carry = DAG.getZExtOrTrunc(Op: Carry, DL, VT: MVT::i128);
4955 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
4956 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
4957 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
4958 N2: DAG.getValueType(MVT::i1));
4959 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
4960 if (IsBorrow)
4961 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
4962 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
4963 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
4964 }
4965
4966 unsigned BaseOp = 0;
4967 unsigned CCValid = 0;
4968 unsigned CCMask = 0;
4969
4970 switch (Op.getOpcode()) {
4971 default: llvm_unreachable("Unknown instruction!");
4972 case ISD::UADDO_CARRY:
4973 if (!isAddCarryChain(Carry))
4974 return SDValue();
4975
4976 BaseOp = SystemZISD::ADDCARRY;
4977 CCValid = SystemZ::CCMASK_LOGICAL;
4978 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
4979 break;
4980 case ISD::USUBO_CARRY:
4981 if (!isSubBorrowChain(Carry))
4982 return SDValue();
4983
4984 BaseOp = SystemZISD::SUBCARRY;
4985 CCValid = SystemZ::CCMASK_LOGICAL;
4986 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
4987 break;
4988 }
4989
4990 // Set the condition code from the carry flag.
4991 Carry = DAG.getNode(Opcode: SystemZISD::GET_CCMASK, DL, VT: MVT::i32, N1: Carry,
4992 N2: DAG.getConstant(Val: CCValid, DL, VT: MVT::i32),
4993 N3: DAG.getConstant(Val: CCMask, DL, VT: MVT::i32));
4994
4995 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
4996 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS, N3: Carry);
4997
4998 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
4999 if (N->getValueType(ResNo: 1) == MVT::i1)
5000 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
5001
5002 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
5003}
5004
5005SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
5006 SelectionDAG &DAG) const {
5007 EVT VT = Op.getValueType();
5008 SDLoc DL(Op);
5009 Op = Op.getOperand(i: 0);
5010
5011 if (VT.getScalarSizeInBits() == 128) {
5012 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: Op);
5013 Op = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: MVT::v2i64, Operand: Op);
5014 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v2i64, DL,
5015 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i64));
5016 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5017 return Op;
5018 }
5019
5020 // Handle vector types via VPOPCT.
5021 if (VT.isVector()) {
5022 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Op);
5023 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::v16i8, Operand: Op);
5024 switch (VT.getScalarSizeInBits()) {
5025 case 8:
5026 break;
5027 case 16: {
5028 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
5029 SDValue Shift = DAG.getConstant(Val: 8, DL, VT: MVT::i32);
5030 SDValue Tmp = DAG.getNode(Opcode: SystemZISD::VSHL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
5031 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
5032 Op = DAG.getNode(Opcode: SystemZISD::VSRL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
5033 break;
5034 }
5035 case 32: {
5036 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
5037 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5038 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5039 break;
5040 }
5041 case 64: {
5042 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
5043 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5044 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Tmp);
5045 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5046 break;
5047 }
5048 default:
5049 llvm_unreachable("Unexpected type");
5050 }
5051 return Op;
5052 }
5053
5054 // Get the known-zero mask for the operand.
5055 KnownBits Known = DAG.computeKnownBits(Op);
5056 unsigned NumSignificantBits = Known.getMaxValue().getActiveBits();
5057 if (NumSignificantBits == 0)
5058 return DAG.getConstant(Val: 0, DL, VT);
5059
5060 // Skip known-zero high parts of the operand.
5061 int64_t OrigBitSize = VT.getSizeInBits();
5062 int64_t BitSize = llvm::bit_ceil(Value: NumSignificantBits);
5063 BitSize = std::min(a: BitSize, b: OrigBitSize);
5064
5065 // The POPCNT instruction counts the number of bits in each byte.
5066 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op);
5067 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::i64, Operand: Op);
5068 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
5069
5070 // Add up per-byte counts in a binary tree. All bits of Op at
5071 // position larger than BitSize remain zero throughout.
5072 for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
5073 SDValue Tmp = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Op, N2: DAG.getConstant(Val: I, DL, VT));
5074 if (BitSize != OrigBitSize)
5075 Tmp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Tmp,
5076 N2: DAG.getConstant(Val: ((uint64_t)1 << BitSize) - 1, DL, VT));
5077 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
5078 }
5079
5080 // Extract overall result from high byte.
5081 if (BitSize > 8)
5082 Op = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Op,
5083 N2: DAG.getConstant(Val: BitSize - 8, DL, VT));
5084
5085 return Op;
5086}
5087
5088SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
5089 SelectionDAG &DAG) const {
5090 SDLoc DL(Op);
5091 AtomicOrdering FenceOrdering =
5092 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
5093 SyncScope::ID FenceSSID =
5094 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
5095
5096 // The only fence that needs an instruction is a sequentially-consistent
5097 // cross-thread fence.
5098 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
5099 FenceSSID == SyncScope::System) {
5100 return SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL, VT: MVT::Other,
5101 Op1: Op.getOperand(i: 0)),
5102 0);
5103 }
5104
5105 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
5106 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
5107}
5108
5109SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
5110 SelectionDAG &DAG) const {
5111 EVT RegVT = Op.getValueType();
5112 if (RegVT.getSizeInBits() == 128)
5113 return lowerATOMIC_LDST_I128(Op, DAG);
5114 return lowerLoadF16(Op, DAG);
5115}
5116
5117SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
5118 SelectionDAG &DAG) const {
5119 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5120 if (Node->getMemoryVT().getSizeInBits() == 128)
5121 return lowerATOMIC_LDST_I128(Op, DAG);
5122 return lowerStoreF16(Op, DAG);
5123}
5124
5125SDValue SystemZTargetLowering::lowerATOMIC_LDST_I128(SDValue Op,
5126 SelectionDAG &DAG) const {
5127 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5128 assert(
5129 (Node->getMemoryVT() == MVT::i128 || Node->getMemoryVT() == MVT::f128) &&
5130 "Only custom lowering i128 or f128.");
5131 // Use same code to handle both legal and non-legal i128 types.
5132 SmallVector<SDValue, 2> Results;
5133 LowerOperationWrapper(N: Node, Results, DAG);
5134 return DAG.getMergeValues(Ops: Results, dl: SDLoc(Op));
5135}
5136
5137// Prepare for a Compare And Swap for a subword operation. This needs to be
5138// done in memory with 4 bytes at natural alignment.
5139static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
5140 SDValue &AlignedAddr, SDValue &BitShift,
5141 SDValue &NegBitShift) {
5142 EVT PtrVT = Addr.getValueType();
5143 EVT WideVT = MVT::i32;
5144
5145 // Get the address of the containing word.
5146 AlignedAddr = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Addr,
5147 N2: DAG.getSignedConstant(Val: -4, DL, VT: PtrVT));
5148
5149 // Get the number of bits that the word must be rotated left in order
5150 // to bring the field to the top bits of a GR32.
5151 BitShift = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: Addr,
5152 N2: DAG.getConstant(Val: 3, DL, VT: PtrVT));
5153 BitShift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: WideVT, Operand: BitShift);
5154
5155 // Get the complementing shift amount, for rotating a field in the top
5156 // bits back to its proper position.
5157 NegBitShift = DAG.getNode(Opcode: ISD::SUB, DL, VT: WideVT,
5158 N1: DAG.getConstant(Val: 0, DL, VT: WideVT), N2: BitShift);
5159
5160}
5161
5162// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first
5163// two into the fullword ATOMIC_LOADW_* operation given by Opcode.
5164SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
5165 SelectionDAG &DAG,
5166 unsigned Opcode) const {
5167 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5168
5169 // 32-bit operations need no special handling.
5170 EVT NarrowVT = Node->getMemoryVT();
5171 EVT WideVT = MVT::i32;
5172 if (NarrowVT == WideVT)
5173 return Op;
5174
5175 int64_t BitSize = NarrowVT.getSizeInBits();
5176 SDValue ChainIn = Node->getChain();
5177 SDValue Addr = Node->getBasePtr();
5178 SDValue Src2 = Node->getVal();
5179 MachineMemOperand *MMO = Node->getMemOperand();
5180 SDLoc DL(Node);
5181
5182 // Convert atomic subtracts of constants into additions.
5183 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
5184 if (auto *Const = dyn_cast<ConstantSDNode>(Val&: Src2)) {
5185 Opcode = SystemZISD::ATOMIC_LOADW_ADD;
5186 Src2 = DAG.getSignedConstant(Val: -Const->getSExtValue(), DL,
5187 VT: Src2.getValueType());
5188 }
5189
5190 SDValue AlignedAddr, BitShift, NegBitShift;
5191 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5192
5193 // Extend the source operand to 32 bits and prepare it for the inner loop.
5194 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
5195 // operations require the source to be shifted in advance. (This shift
5196 // can be folded if the source is constant.) For AND and NAND, the lower
5197 // bits must be set, while for other opcodes they should be left clear.
5198 if (Opcode != SystemZISD::ATOMIC_SWAPW)
5199 Src2 = DAG.getNode(Opcode: ISD::SHL, DL, VT: WideVT, N1: Src2,
5200 N2: DAG.getConstant(Val: 32 - BitSize, DL, VT: WideVT));
5201 if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
5202 Opcode == SystemZISD::ATOMIC_LOADW_NAND)
5203 Src2 = DAG.getNode(Opcode: ISD::OR, DL, VT: WideVT, N1: Src2,
5204 N2: DAG.getConstant(Val: uint32_t(-1) >> BitSize, DL, VT: WideVT));
5205
5206 // Construct the ATOMIC_LOADW_* node.
5207 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::Other);
5208 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
5209 DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5210 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops,
5211 MemVT: NarrowVT, MMO);
5212
5213 // Rotate the result of the final CS so that the field is in the lower
5214 // bits of a GR32, then truncate it.
5215 SDValue ResultShift = DAG.getNode(Opcode: ISD::ADD, DL, VT: WideVT, N1: BitShift,
5216 N2: DAG.getConstant(Val: BitSize, DL, VT: WideVT));
5217 SDValue Result = DAG.getNode(Opcode: ISD::ROTL, DL, VT: WideVT, N1: AtomicOp, N2: ResultShift);
5218
5219 SDValue RetOps[2] = { Result, AtomicOp.getValue(R: 1) };
5220 return DAG.getMergeValues(Ops: RetOps, dl: DL);
5221}
5222
5223// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations into
5224// ATOMIC_LOADW_SUBs and convert 32- and 64-bit operations into additions.
5225SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
5226 SelectionDAG &DAG) const {
5227 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5228 EVT MemVT = Node->getMemoryVT();
5229 if (MemVT == MVT::i32 || MemVT == MVT::i64) {
5230 // A full-width operation: negate and use LAA(G).
5231 assert(Op.getValueType() == MemVT && "Mismatched VTs");
5232 assert(Subtarget.hasInterlockedAccess1() &&
5233 "Should have been expanded by AtomicExpand pass.");
5234 SDValue Src2 = Node->getVal();
5235 SDLoc DL(Src2);
5236 SDValue NegSrc2 =
5237 DAG.getNode(Opcode: ISD::SUB, DL, VT: MemVT, N1: DAG.getConstant(Val: 0, DL, VT: MemVT), N2: Src2);
5238 return DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT,
5239 Chain: Node->getChain(), Ptr: Node->getBasePtr(), Val: NegSrc2,
5240 MMO: Node->getMemOperand());
5241 }
5242
5243 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_SUB);
5244}
5245
5246// Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
5247SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
5248 SelectionDAG &DAG) const {
5249 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5250 SDValue ChainIn = Node->getOperand(Num: 0);
5251 SDValue Addr = Node->getOperand(Num: 1);
5252 SDValue CmpVal = Node->getOperand(Num: 2);
5253 SDValue SwapVal = Node->getOperand(Num: 3);
5254 MachineMemOperand *MMO = Node->getMemOperand();
5255 SDLoc DL(Node);
5256
5257 if (Node->getMemoryVT() == MVT::i128) {
5258 // Use same code to handle both legal and non-legal i128 types.
5259 SmallVector<SDValue, 3> Results;
5260 LowerOperationWrapper(N: Node, Results, DAG);
5261 return DAG.getMergeValues(Ops: Results, dl: DL);
5262 }
5263
5264 // We have native support for 32-bit and 64-bit compare and swap, but we
5265 // still need to expand extracting the "success" result from the CC.
5266 EVT NarrowVT = Node->getMemoryVT();
5267 EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
5268 if (NarrowVT == WideVT) {
5269 SDVTList Tys = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5270 SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
5271 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP,
5272 dl: DL, VTList: Tys, Ops, MemVT: NarrowVT, MMO);
5273 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5274 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
5275
5276 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: AtomicOp.getValue(R: 0));
5277 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5278 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5279 return SDValue();
5280 }
5281
5282 // Convert 8-bit and 16-bit compare and swap to a loop, implemented
5283 // via a fullword ATOMIC_CMP_SWAPW operation.
5284 int64_t BitSize = NarrowVT.getSizeInBits();
5285
5286 SDValue AlignedAddr, BitShift, NegBitShift;
5287 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5288
5289 // Construct the ATOMIC_CMP_SWAPW node.
5290 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5291 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
5292 NegBitShift, DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5293 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAPW, dl: DL,
5294 VTList, Ops, MemVT: NarrowVT, MMO);
5295 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5296 CCValid: SystemZ::CCMASK_ICMP, CCMask: SystemZ::CCMASK_CMP_EQ);
5297
5298 // emitAtomicCmpSwapW() will zero extend the result (original value).
5299 SDValue OrigVal = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: WideVT, N1: AtomicOp.getValue(R: 0),
5300 N2: DAG.getValueType(NarrowVT));
5301 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: OrigVal);
5302 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5303 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5304 return SDValue();
5305}
5306
5307MachineMemOperand::Flags
5308SystemZTargetLowering::getTargetMMOFlags(const Instruction &I) const {
5309 // Because of how we convert atomic_load and atomic_store to normal loads and
5310 // stores in the DAG, we need to ensure that the MMOs are marked volatile
5311 // since DAGCombine hasn't been updated to account for atomic, but non
5312 // volatile loads. (See D57601)
5313 if (auto *SI = dyn_cast<StoreInst>(Val: &I))
5314 if (SI->isAtomic())
5315 return MachineMemOperand::MOVolatile;
5316 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
5317 if (LI->isAtomic())
5318 return MachineMemOperand::MOVolatile;
5319 if (auto *AI = dyn_cast<AtomicRMWInst>(Val: &I))
5320 if (AI->isAtomic())
5321 return MachineMemOperand::MOVolatile;
5322 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
5323 if (AI->isAtomic())
5324 return MachineMemOperand::MOVolatile;
5325 return MachineMemOperand::MONone;
5326}
5327
5328SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
5329 SelectionDAG &DAG) const {
5330 MachineFunction &MF = DAG.getMachineFunction();
5331 auto *Regs = Subtarget.getSpecialRegisters();
5332 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5333 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5334 "in GHC calling convention");
5335 return DAG.getCopyFromReg(Chain: Op.getOperand(i: 0), dl: SDLoc(Op),
5336 Reg: Regs->getStackPointerRegister(), VT: Op.getValueType());
5337}
5338
5339SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
5340 SelectionDAG &DAG) const {
5341 MachineFunction &MF = DAG.getMachineFunction();
5342 auto *Regs = Subtarget.getSpecialRegisters();
5343 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
5344
5345 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5346 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5347 "in GHC calling convention");
5348
5349 SDValue Chain = Op.getOperand(i: 0);
5350 SDValue NewSP = Op.getOperand(i: 1);
5351 SDValue Backchain;
5352 SDLoc DL(Op);
5353
5354 if (StoreBackchain) {
5355 SDValue OldSP = DAG.getCopyFromReg(
5356 Chain, dl: DL, Reg: Regs->getStackPointerRegister(), VT: MVT::i64);
5357 Backchain = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
5358 PtrInfo: MachinePointerInfo());
5359 }
5360
5361 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Regs->getStackPointerRegister(), N: NewSP);
5362
5363 if (StoreBackchain)
5364 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
5365 PtrInfo: MachinePointerInfo());
5366
5367 return Chain;
5368}
5369
5370SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
5371 SelectionDAG &DAG) const {
5372 bool IsData = Op.getConstantOperandVal(i: 4);
5373 if (!IsData)
5374 // Just preserve the chain.
5375 return Op.getOperand(i: 0);
5376
5377 SDLoc DL(Op);
5378 bool IsWrite = Op.getConstantOperandVal(i: 2);
5379 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
5380 auto *Node = cast<MemIntrinsicSDNode>(Val: Op.getNode());
5381 SDValue Ops[] = {Op.getOperand(i: 0), DAG.getTargetConstant(Val: Code, DL, VT: MVT::i32),
5382 Op.getOperand(i: 1)};
5383 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::PREFETCH, dl: DL,
5384 VTList: Node->getVTList(), Ops,
5385 MemVT: Node->getMemoryVT(), MMO: Node->getMemOperand());
5386}
5387
5388SDValue
5389SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
5390 SelectionDAG &DAG) const {
5391 unsigned Opcode, CCValid;
5392 if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) {
5393 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
5394 SDNode *Node = emitIntrinsicWithCCAndChain(DAG, Op, Opcode);
5395 SDValue CC = getCCResult(DAG, CCReg: SDValue(Node, 0));
5396 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Op.getNode(), 0), To: CC);
5397 return SDValue();
5398 }
5399
5400 return SDValue();
5401}
5402
5403SDValue
5404SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
5405 SelectionDAG &DAG) const {
5406 unsigned Opcode, CCValid;
5407 if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
5408 SDNode *Node = emitIntrinsicWithCC(DAG, Op, Opcode);
5409 if (Op->getNumValues() == 1)
5410 return getCCResult(DAG, CCReg: SDValue(Node, 0));
5411 assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");
5412 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op), VTList: Op->getVTList(),
5413 N1: SDValue(Node, 0), N2: getCCResult(DAG, CCReg: SDValue(Node, 1)));
5414 }
5415
5416 unsigned Id = Op.getConstantOperandVal(i: 0);
5417 switch (Id) {
5418 case Intrinsic::thread_pointer:
5419 return lowerThreadPointer(DL: SDLoc(Op), DAG);
5420
5421 case Intrinsic::s390_vpdi:
5422 return DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL: SDLoc(Op), VT: Op.getValueType(),
5423 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5424
5425 case Intrinsic::s390_vperm:
5426 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL: SDLoc(Op), VT: Op.getValueType(),
5427 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5428
5429 case Intrinsic::s390_vuphb:
5430 case Intrinsic::s390_vuphh:
5431 case Intrinsic::s390_vuphf:
5432 case Intrinsic::s390_vuphg:
5433 return DAG.getNode(Opcode: SystemZISD::UNPACK_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5434 Operand: Op.getOperand(i: 1));
5435
5436 case Intrinsic::s390_vuplhb:
5437 case Intrinsic::s390_vuplhh:
5438 case Intrinsic::s390_vuplhf:
5439 case Intrinsic::s390_vuplhg:
5440 return DAG.getNode(Opcode: SystemZISD::UNPACKL_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5441 Operand: Op.getOperand(i: 1));
5442
5443 case Intrinsic::s390_vuplb:
5444 case Intrinsic::s390_vuplhw:
5445 case Intrinsic::s390_vuplf:
5446 case Intrinsic::s390_vuplg:
5447 return DAG.getNode(Opcode: SystemZISD::UNPACK_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5448 Operand: Op.getOperand(i: 1));
5449
5450 case Intrinsic::s390_vupllb:
5451 case Intrinsic::s390_vupllh:
5452 case Intrinsic::s390_vupllf:
5453 case Intrinsic::s390_vupllg:
5454 return DAG.getNode(Opcode: SystemZISD::UNPACKL_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5455 Operand: Op.getOperand(i: 1));
5456
5457 case Intrinsic::s390_vsumb:
5458 case Intrinsic::s390_vsumh:
5459 case Intrinsic::s390_vsumgh:
5460 case Intrinsic::s390_vsumgf:
5461 case Intrinsic::s390_vsumqf:
5462 case Intrinsic::s390_vsumqg:
5463 return DAG.getNode(Opcode: SystemZISD::VSUM, DL: SDLoc(Op), VT: Op.getValueType(),
5464 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5465
5466 case Intrinsic::s390_vaq:
5467 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5468 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5469 case Intrinsic::s390_vaccb:
5470 case Intrinsic::s390_vacch:
5471 case Intrinsic::s390_vaccf:
5472 case Intrinsic::s390_vaccg:
5473 case Intrinsic::s390_vaccq:
5474 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(Op), VT: Op.getValueType(),
5475 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5476 case Intrinsic::s390_vacq:
5477 return DAG.getNode(Opcode: SystemZISD::VAC, DL: SDLoc(Op), VT: Op.getValueType(),
5478 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5479 case Intrinsic::s390_vacccq:
5480 return DAG.getNode(Opcode: SystemZISD::VACCC, DL: SDLoc(Op), VT: Op.getValueType(),
5481 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5482
5483 case Intrinsic::s390_vsq:
5484 return DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(Op), VT: Op.getValueType(),
5485 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5486 case Intrinsic::s390_vscbib:
5487 case Intrinsic::s390_vscbih:
5488 case Intrinsic::s390_vscbif:
5489 case Intrinsic::s390_vscbig:
5490 case Intrinsic::s390_vscbiq:
5491 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5492 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5493 case Intrinsic::s390_vsbiq:
5494 return DAG.getNode(Opcode: SystemZISD::VSBI, DL: SDLoc(Op), VT: Op.getValueType(),
5495 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5496 case Intrinsic::s390_vsbcbiq:
5497 return DAG.getNode(Opcode: SystemZISD::VSBCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5498 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5499
5500 case Intrinsic::s390_vmhb:
5501 case Intrinsic::s390_vmhh:
5502 case Intrinsic::s390_vmhf:
5503 case Intrinsic::s390_vmhg:
5504 case Intrinsic::s390_vmhq:
5505 return DAG.getNode(Opcode: ISD::MULHS, DL: SDLoc(Op), VT: Op.getValueType(),
5506 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5507 case Intrinsic::s390_vmlhb:
5508 case Intrinsic::s390_vmlhh:
5509 case Intrinsic::s390_vmlhf:
5510 case Intrinsic::s390_vmlhg:
5511 case Intrinsic::s390_vmlhq:
5512 return DAG.getNode(Opcode: ISD::MULHU, DL: SDLoc(Op), VT: Op.getValueType(),
5513 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5514
5515 case Intrinsic::s390_vmahb:
5516 case Intrinsic::s390_vmahh:
5517 case Intrinsic::s390_vmahf:
5518 case Intrinsic::s390_vmahg:
5519 case Intrinsic::s390_vmahq:
5520 return DAG.getNode(Opcode: SystemZISD::VMAH, DL: SDLoc(Op), VT: Op.getValueType(),
5521 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5522 case Intrinsic::s390_vmalhb:
5523 case Intrinsic::s390_vmalhh:
5524 case Intrinsic::s390_vmalhf:
5525 case Intrinsic::s390_vmalhg:
5526 case Intrinsic::s390_vmalhq:
5527 return DAG.getNode(Opcode: SystemZISD::VMALH, DL: SDLoc(Op), VT: Op.getValueType(),
5528 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5529
5530 case Intrinsic::s390_vmeb:
5531 case Intrinsic::s390_vmeh:
5532 case Intrinsic::s390_vmef:
5533 case Intrinsic::s390_vmeg:
5534 return DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5535 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5536 case Intrinsic::s390_vmleb:
5537 case Intrinsic::s390_vmleh:
5538 case Intrinsic::s390_vmlef:
5539 case Intrinsic::s390_vmleg:
5540 return DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5541 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5542 case Intrinsic::s390_vmob:
5543 case Intrinsic::s390_vmoh:
5544 case Intrinsic::s390_vmof:
5545 case Intrinsic::s390_vmog:
5546 return DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5547 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5548 case Intrinsic::s390_vmlob:
5549 case Intrinsic::s390_vmloh:
5550 case Intrinsic::s390_vmlof:
5551 case Intrinsic::s390_vmlog:
5552 return DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5553 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5554
5555 case Intrinsic::s390_vmaeb:
5556 case Intrinsic::s390_vmaeh:
5557 case Intrinsic::s390_vmaef:
5558 case Intrinsic::s390_vmaeg:
5559 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5560 N1: DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5561 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5562 N2: Op.getOperand(i: 3));
5563 case Intrinsic::s390_vmaleb:
5564 case Intrinsic::s390_vmaleh:
5565 case Intrinsic::s390_vmalef:
5566 case Intrinsic::s390_vmaleg:
5567 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5568 N1: DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5569 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5570 N2: Op.getOperand(i: 3));
5571 case Intrinsic::s390_vmaob:
5572 case Intrinsic::s390_vmaoh:
5573 case Intrinsic::s390_vmaof:
5574 case Intrinsic::s390_vmaog:
5575 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5576 N1: DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5577 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5578 N2: Op.getOperand(i: 3));
5579 case Intrinsic::s390_vmalob:
5580 case Intrinsic::s390_vmaloh:
5581 case Intrinsic::s390_vmalof:
5582 case Intrinsic::s390_vmalog:
5583 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5584 N1: DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5585 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5586 N2: Op.getOperand(i: 3));
5587 }
5588
5589 return SDValue();
5590}
5591
5592namespace {
5593// Says that SystemZISD operation Opcode can be used to perform the equivalent
5594// of a VPERM with permute vector Bytes. If Opcode takes three operands,
5595// Operand is the constant third operand, otherwise it is the number of
5596// bytes in each element of the result.
5597struct Permute {
5598 unsigned Opcode;
5599 unsigned Operand;
5600 unsigned char Bytes[SystemZ::VectorBytes];
5601};
5602}
5603
5604static const Permute PermuteForms[] = {
5605 // VMRHG
5606 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 8,
5607 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
5608 // VMRHF
5609 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 4,
5610 .Bytes: { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
5611 // VMRHH
5612 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 2,
5613 .Bytes: { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
5614 // VMRHB
5615 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 1,
5616 .Bytes: { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
5617 // VMRLG
5618 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 8,
5619 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
5620 // VMRLF
5621 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 4,
5622 .Bytes: { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
5623 // VMRLH
5624 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 2,
5625 .Bytes: { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
5626 // VMRLB
5627 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 1,
5628 .Bytes: { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
5629 // VPKG
5630 { .Opcode: SystemZISD::PACK, .Operand: 4,
5631 .Bytes: { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
5632 // VPKF
5633 { .Opcode: SystemZISD::PACK, .Operand: 2,
5634 .Bytes: { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
5635 // VPKH
5636 { .Opcode: SystemZISD::PACK, .Operand: 1,
5637 .Bytes: { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
5638 // VPDI V1, V2, 4 (low half of V1, high half of V2)
5639 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 4,
5640 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
5641 // VPDI V1, V2, 1 (high half of V1, low half of V2)
5642 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 1,
5643 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
5644};
5645
5646// Called after matching a vector shuffle against a particular pattern.
5647// Both the original shuffle and the pattern have two vector operands.
5648// OpNos[0] is the operand of the original shuffle that should be used for
5649// operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
5650// OpNos[1] is the same for operand 1 of the pattern. Resolve these -1s and
5651// set OpNo0 and OpNo1 to the shuffle operands that should actually be used
5652// for operands 0 and 1 of the pattern.
5653static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
5654 if (OpNos[0] < 0) {
5655 if (OpNos[1] < 0)
5656 return false;
5657 OpNo0 = OpNo1 = OpNos[1];
5658 } else if (OpNos[1] < 0) {
5659 OpNo0 = OpNo1 = OpNos[0];
5660 } else {
5661 OpNo0 = OpNos[0];
5662 OpNo1 = OpNos[1];
5663 }
5664 return true;
5665}
5666
5667// Bytes is a VPERM-like permute vector, except that -1 is used for
5668// undefined bytes. Return true if the VPERM can be implemented using P.
5669// When returning true set OpNo0 to the VPERM operand that should be
5670// used for operand 0 of P and likewise OpNo1 for operand 1 of P.
5671//
5672// For example, if swapping the VPERM operands allows P to match, OpNo0
5673// will be 1 and OpNo1 will be 0. If instead Bytes only refers to one
5674// operand, but rewriting it to use two duplicated operands allows it to
5675// match P, then OpNo0 and OpNo1 will be the same.
5676static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
5677 unsigned &OpNo0, unsigned &OpNo1) {
5678 int OpNos[] = { -1, -1 };
5679 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5680 int Elt = Bytes[I];
5681 if (Elt >= 0) {
5682 // Make sure that the two permute vectors use the same suboperand
5683 // byte number. Only the operand numbers (the high bits) are
5684 // allowed to differ.
5685 if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
5686 return false;
5687 int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
5688 int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
5689 // Make sure that the operand mappings are consistent with previous
5690 // elements.
5691 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5692 return false;
5693 OpNos[ModelOpNo] = RealOpNo;
5694 }
5695 }
5696 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5697}
5698
5699// As above, but search for a matching permute.
5700static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
5701 unsigned &OpNo0, unsigned &OpNo1) {
5702 for (auto &P : PermuteForms)
5703 if (matchPermute(Bytes, P, OpNo0, OpNo1))
5704 return &P;
5705 return nullptr;
5706}
5707
5708// Bytes is a VPERM-like permute vector, except that -1 is used for
5709// undefined bytes. This permute is an operand of an outer permute.
5710// See whether redistributing the -1 bytes gives a shuffle that can be
5711// implemented using P. If so, set Transform to a VPERM-like permute vector
5712// that, when applied to the result of P, gives the original permute in Bytes.
5713static bool matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5714 const Permute &P,
5715 SmallVectorImpl<int> &Transform) {
5716 unsigned To = 0;
5717 for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
5718 int Elt = Bytes[From];
5719 if (Elt < 0)
5720 // Byte number From of the result is undefined.
5721 Transform[From] = -1;
5722 else {
5723 while (P.Bytes[To] != Elt) {
5724 To += 1;
5725 if (To == SystemZ::VectorBytes)
5726 return false;
5727 }
5728 Transform[From] = To;
5729 }
5730 }
5731 return true;
5732}
5733
5734// As above, but search for a matching permute.
5735static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5736 SmallVectorImpl<int> &Transform) {
5737 for (auto &P : PermuteForms)
5738 if (matchDoublePermute(Bytes, P, Transform))
5739 return &P;
5740 return nullptr;
5741}
5742
5743// Convert the mask of the given shuffle op into a byte-level mask,
5744// as if it had type vNi8.
5745static bool getVPermMask(SDValue ShuffleOp,
5746 SmallVectorImpl<int> &Bytes) {
5747 EVT VT = ShuffleOp.getValueType();
5748 unsigned NumElements = VT.getVectorNumElements();
5749 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5750
5751 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: ShuffleOp)) {
5752 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5753 for (unsigned I = 0; I < NumElements; ++I) {
5754 int Index = VSN->getMaskElt(Idx: I);
5755 if (Index >= 0)
5756 for (unsigned J = 0; J < BytesPerElement; ++J)
5757 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5758 }
5759 return true;
5760 }
5761 if (SystemZISD::SPLAT == ShuffleOp.getOpcode() &&
5762 isa<ConstantSDNode>(Val: ShuffleOp.getOperand(i: 1))) {
5763 unsigned Index = ShuffleOp.getConstantOperandVal(i: 1);
5764 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5765 for (unsigned I = 0; I < NumElements; ++I)
5766 for (unsigned J = 0; J < BytesPerElement; ++J)
5767 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5768 return true;
5769 }
5770 return false;
5771}
5772
5773// Bytes is a VPERM-like permute vector, except that -1 is used for
5774// undefined bytes. See whether bytes [Start, Start + BytesPerElement) of
5775// the result come from a contiguous sequence of bytes from one input.
5776// Set Base to the selector for the first byte if so.
5777static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
5778 unsigned BytesPerElement, int &Base) {
5779 Base = -1;
5780 for (unsigned I = 0; I < BytesPerElement; ++I) {
5781 if (Bytes[Start + I] >= 0) {
5782 unsigned Elem = Bytes[Start + I];
5783 if (Base < 0) {
5784 Base = Elem - I;
5785 // Make sure the bytes would come from one input operand.
5786 if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
5787 return false;
5788 } else if (unsigned(Base) != Elem - I)
5789 return false;
5790 }
5791 }
5792 return true;
5793}
5794
5795// Bytes is a VPERM-like permute vector, except that -1 is used for
5796// undefined bytes. Return true if it can be performed using VSLDB.
5797// When returning true, set StartIndex to the shift amount and OpNo0
5798// and OpNo1 to the VPERM operands that should be used as the first
5799// and second shift operand respectively.
5800static bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes,
5801 unsigned &StartIndex, unsigned &OpNo0,
5802 unsigned &OpNo1) {
5803 int OpNos[] = { -1, -1 };
5804 int Shift = -1;
5805 for (unsigned I = 0; I < 16; ++I) {
5806 int Index = Bytes[I];
5807 if (Index >= 0) {
5808 int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
5809 int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
5810 int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
5811 if (Shift < 0)
5812 Shift = ExpectedShift;
5813 else if (Shift != ExpectedShift)
5814 return false;
5815 // Make sure that the operand mappings are consistent with previous
5816 // elements.
5817 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5818 return false;
5819 OpNos[ModelOpNo] = RealOpNo;
5820 }
5821 }
5822 StartIndex = Shift;
5823 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5824}
5825
5826// Create a node that performs P on operands Op0 and Op1, casting the
5827// operands to the appropriate type. The type of the result is determined by P.
5828static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5829 const Permute &P, SDValue Op0, SDValue Op1) {
5830 // VPDI (PERMUTE_DWORDS) always operates on v2i64s. The input
5831 // elements of a PACK are twice as wide as the outputs.
5832 unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
5833 P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
5834 P.Operand);
5835 // Cast both operands to the appropriate type.
5836 MVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBytes * 8),
5837 NumElements: SystemZ::VectorBytes / InBytes);
5838 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op0);
5839 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op1);
5840 SDValue Op;
5841 if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
5842 SDValue Op2 = DAG.getTargetConstant(Val: P.Operand, DL, VT: MVT::i32);
5843 Op = DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL, VT: InVT, N1: Op0, N2: Op1, N3: Op2);
5844 } else if (P.Opcode == SystemZISD::PACK) {
5845 MVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: P.Operand * 8),
5846 NumElements: SystemZ::VectorBytes / P.Operand);
5847 Op = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT: OutVT, N1: Op0, N2: Op1);
5848 } else {
5849 Op = DAG.getNode(Opcode: P.Opcode, DL, VT: InVT, N1: Op0, N2: Op1);
5850 }
5851 return Op;
5852}
5853
5854static bool isZeroVector(SDValue N) {
5855 if (N->getOpcode() == ISD::BITCAST)
5856 N = N->getOperand(Num: 0);
5857 if (N->getOpcode() == ISD::SPLAT_VECTOR)
5858 if (auto *Op = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0)))
5859 return Op->getZExtValue() == 0;
5860 return ISD::isBuildVectorAllZeros(N: N.getNode());
5861}
5862
5863// Return the index of the zero/undef vector, or UINT32_MAX if not found.
5864static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num) {
5865 for (unsigned I = 0; I < Num ; I++)
5866 if (isZeroVector(N: Ops[I]))
5867 return I;
5868 return UINT32_MAX;
5869}
5870
5871// Bytes is a VPERM-like permute vector, except that -1 is used for
5872// undefined bytes. Implement it on operands Ops[0] and Ops[1] using
5873// VSLDB or VPERM.
5874static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5875 SDValue *Ops,
5876 const SmallVectorImpl<int> &Bytes) {
5877 for (unsigned I = 0; I < 2; ++I)
5878 Ops[I] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Ops[I]);
5879
5880 // First see whether VSLDB can be used.
5881 unsigned StartIndex, OpNo0, OpNo1;
5882 if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
5883 return DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE, DL, VT: MVT::v16i8, N1: Ops[OpNo0],
5884 N2: Ops[OpNo1],
5885 N3: DAG.getTargetConstant(Val: StartIndex, DL, VT: MVT::i32));
5886
5887 // Fall back on VPERM. Construct an SDNode for the permute vector. Try to
5888 // eliminate a zero vector by reusing any zero index in the permute vector.
5889 unsigned ZeroVecIdx = findZeroVectorIdx(Ops: &Ops[0], Num: 2);
5890 if (ZeroVecIdx != UINT32_MAX) {
5891 bool MaskFirst = true;
5892 int ZeroIdx = -1;
5893 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5894 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5895 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
5896 if (OpNo == ZeroVecIdx && I == 0) {
5897 // If the first byte is zero, use mask as first operand.
5898 ZeroIdx = 0;
5899 break;
5900 }
5901 if (OpNo != ZeroVecIdx && Byte == 0) {
5902 // If mask contains a zero, use it by placing that vector first.
5903 ZeroIdx = I + SystemZ::VectorBytes;
5904 MaskFirst = false;
5905 break;
5906 }
5907 }
5908 if (ZeroIdx != -1) {
5909 SDValue IndexNodes[SystemZ::VectorBytes];
5910 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5911 if (Bytes[I] >= 0) {
5912 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5913 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
5914 if (OpNo == ZeroVecIdx)
5915 IndexNodes[I] = DAG.getConstant(Val: ZeroIdx, DL, VT: MVT::i32);
5916 else {
5917 unsigned BIdx = MaskFirst ? Byte + SystemZ::VectorBytes : Byte;
5918 IndexNodes[I] = DAG.getConstant(Val: BIdx, DL, VT: MVT::i32);
5919 }
5920 } else
5921 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
5922 }
5923 SDValue Mask = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
5924 SDValue Src = ZeroVecIdx == 0 ? Ops[1] : Ops[0];
5925 if (MaskFirst)
5926 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Mask, N2: Src,
5927 N3: Mask);
5928 else
5929 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Src, N2: Mask,
5930 N3: Mask);
5931 }
5932 }
5933
5934 SDValue IndexNodes[SystemZ::VectorBytes];
5935 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
5936 if (Bytes[I] >= 0)
5937 IndexNodes[I] = DAG.getConstant(Val: Bytes[I], DL, VT: MVT::i32);
5938 else
5939 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
5940 SDValue Op2 = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
5941 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Ops[0],
5942 N2: (!Ops[1].isUndef() ? Ops[1] : Ops[0]), N3: Op2);
5943}
5944
5945namespace {
5946// Describes a general N-operand vector shuffle.
5947struct GeneralShuffle {
5948 GeneralShuffle(EVT vt)
5949 : VT(vt), UnpackFromEltSize(UINT_MAX), UnpackLow(false) {}
5950 void addUndef();
5951 bool add(SDValue, unsigned);
5952 SDValue getNode(SelectionDAG &, const SDLoc &);
5953 void tryPrepareForUnpack();
5954 bool unpackWasPrepared() { return UnpackFromEltSize <= 4; }
5955 SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
5956
5957 // The operands of the shuffle.
5958 SmallVector<SDValue, SystemZ::VectorBytes> Ops;
5959
5960 // Index I is -1 if byte I of the result is undefined. Otherwise the
5961 // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
5962 // Bytes[I] / SystemZ::VectorBytes.
5963 SmallVector<int, SystemZ::VectorBytes> Bytes;
5964
5965 // The type of the shuffle result.
5966 EVT VT;
5967
5968 // Holds a value of 1, 2 or 4 if a final unpack has been prepared for.
5969 unsigned UnpackFromEltSize;
5970 // True if the final unpack uses the low half.
5971 bool UnpackLow;
5972};
5973} // namespace
5974
5975// Add an extra undefined element to the shuffle.
5976void GeneralShuffle::addUndef() {
5977 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5978 for (unsigned I = 0; I < BytesPerElement; ++I)
5979 Bytes.push_back(Elt: -1);
5980}
5981
5982// Add an extra element to the shuffle, taking it from element Elem of Op.
5983// A null Op indicates a vector input whose value will be calculated later;
5984// there is at most one such input per shuffle and it always has the same
5985// type as the result. Aborts and returns false if the source vector elements
5986// of an EXTRACT_VECTOR_ELT are smaller than the destination elements. Per
5987// LLVM they become implicitly extended, but this is rare and not optimized.
5988bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
5989 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5990
5991 // The source vector can have wider elements than the result,
5992 // either through an explicit TRUNCATE or because of type legalization.
5993 // We want the least significant part.
5994 EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
5995 unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
5996
5997 // Return false if the source elements are smaller than their destination
5998 // elements.
5999 if (FromBytesPerElement < BytesPerElement)
6000 return false;
6001
6002 unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
6003 (FromBytesPerElement - BytesPerElement));
6004
6005 // Look through things like shuffles and bitcasts.
6006 while (Op.getNode()) {
6007 if (Op.getOpcode() == ISD::BITCAST)
6008 Op = Op.getOperand(i: 0);
6009 else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
6010 // See whether the bytes we need come from a contiguous part of one
6011 // operand.
6012 SmallVector<int, SystemZ::VectorBytes> OpBytes;
6013 if (!getVPermMask(ShuffleOp: Op, Bytes&: OpBytes))
6014 break;
6015 int NewByte;
6016 if (!getShuffleInput(Bytes: OpBytes, Start: Byte, BytesPerElement, Base&: NewByte))
6017 break;
6018 if (NewByte < 0) {
6019 addUndef();
6020 return true;
6021 }
6022 Op = Op.getOperand(i: unsigned(NewByte) / SystemZ::VectorBytes);
6023 Byte = unsigned(NewByte) % SystemZ::VectorBytes;
6024 } else if (Op.isUndef()) {
6025 addUndef();
6026 return true;
6027 } else
6028 break;
6029 }
6030
6031 // Make sure that the source of the extraction is in Ops.
6032 unsigned OpNo = 0;
6033 for (; OpNo < Ops.size(); ++OpNo)
6034 if (Ops[OpNo] == Op)
6035 break;
6036 if (OpNo == Ops.size())
6037 Ops.push_back(Elt: Op);
6038
6039 // Add the element to Bytes.
6040 unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
6041 for (unsigned I = 0; I < BytesPerElement; ++I)
6042 Bytes.push_back(Elt: Base + I);
6043
6044 return true;
6045}
6046
6047// Return SDNodes for the completed shuffle.
6048SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
6049 assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
6050
6051 if (Ops.size() == 0)
6052 return DAG.getUNDEF(VT);
6053
6054 // Use a single unpack if possible as the last operation.
6055 tryPrepareForUnpack();
6056
6057 // Make sure that there are at least two shuffle operands.
6058 if (Ops.size() == 1)
6059 Ops.push_back(Elt: DAG.getUNDEF(VT: MVT::v16i8));
6060
6061 // Create a tree of shuffles, deferring root node until after the loop.
6062 // Try to redistribute the undefined elements of non-root nodes so that
6063 // the non-root shuffles match something like a pack or merge, then adjust
6064 // the parent node's permute vector to compensate for the new order.
6065 // Among other things, this copes with vectors like <2 x i16> that were
6066 // padded with undefined elements during type legalization.
6067 //
6068 // In the best case this redistribution will lead to the whole tree
6069 // using packs and merges. It should rarely be a loss in other cases.
6070 unsigned Stride = 1;
6071 for (; Stride * 2 < Ops.size(); Stride *= 2) {
6072 for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
6073 SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
6074
6075 // Create a mask for just these two operands.
6076 SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes);
6077 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6078 unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
6079 unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
6080 if (OpNo == I)
6081 NewBytes[J] = Byte;
6082 else if (OpNo == I + Stride)
6083 NewBytes[J] = SystemZ::VectorBytes + Byte;
6084 else
6085 NewBytes[J] = -1;
6086 }
6087 // See if it would be better to reorganize NewMask to avoid using VPERM.
6088 SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes);
6089 if (const Permute *P = matchDoublePermute(Bytes: NewBytes, Transform&: NewBytesMap)) {
6090 Ops[I] = getPermuteNode(DAG, DL, P: *P, Op0: SubOps[0], Op1: SubOps[1]);
6091 // Applying NewBytesMap to Ops[I] gets back to NewBytes.
6092 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6093 if (NewBytes[J] >= 0) {
6094 assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
6095 "Invalid double permute");
6096 Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
6097 } else
6098 assert(NewBytesMap[J] < 0 && "Invalid double permute");
6099 }
6100 } else {
6101 // Just use NewBytes on the operands.
6102 Ops[I] = getGeneralPermuteNode(DAG, DL, Ops: SubOps, Bytes: NewBytes);
6103 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
6104 if (NewBytes[J] >= 0)
6105 Bytes[J] = I * SystemZ::VectorBytes + J;
6106 }
6107 }
6108 }
6109
6110 // Now we just have 2 inputs. Put the second operand in Ops[1].
6111 if (Stride > 1) {
6112 Ops[1] = Ops[Stride];
6113 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6114 if (Bytes[I] >= int(SystemZ::VectorBytes))
6115 Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
6116 }
6117
6118 // Look for an instruction that can do the permute without resorting
6119 // to VPERM.
6120 unsigned OpNo0, OpNo1;
6121 SDValue Op;
6122 if (unpackWasPrepared() && Ops[1].isUndef())
6123 Op = Ops[0];
6124 else if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
6125 Op = getPermuteNode(DAG, DL, P: *P, Op0: Ops[OpNo0], Op1: Ops[OpNo1]);
6126 else
6127 Op = getGeneralPermuteNode(DAG, DL, Ops: &Ops[0], Bytes);
6128
6129 Op = insertUnpackIfPrepared(DAG, DL, Op);
6130
6131 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6132}
6133
6134#ifndef NDEBUG
6135static void dumpBytes(const SmallVectorImpl<int> &Bytes, std::string Msg) {
6136 dbgs() << Msg.c_str() << " { ";
6137 for (unsigned I = 0; I < Bytes.size(); I++)
6138 dbgs() << Bytes[I] << " ";
6139 dbgs() << "}\n";
6140}
6141#endif
6142
6143// If the Bytes vector matches an unpack operation, prepare to do the unpack
6144// after all else by removing the zero vector and the effect of the unpack on
6145// Bytes.
6146void GeneralShuffle::tryPrepareForUnpack() {
6147 uint32_t ZeroVecOpNo = findZeroVectorIdx(Ops: &Ops[0], Num: Ops.size());
6148 if (ZeroVecOpNo == UINT32_MAX || Ops.size() == 1)
6149 return;
6150
6151 // Only do this if removing the zero vector reduces the depth, otherwise
6152 // the critical path will increase with the final unpack.
6153 if (Ops.size() > 2 &&
6154 Log2_32_Ceil(Value: Ops.size()) == Log2_32_Ceil(Value: Ops.size() - 1))
6155 return;
6156
6157 // Find an unpack that would allow removing the zero vector from Ops.
6158 UnpackFromEltSize = 1;
6159 for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
6160 bool MatchUnpack = true;
6161 SmallVector<int, SystemZ::VectorBytes> SrcBytes;
6162 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes; Elt++) {
6163 unsigned ToEltSize = UnpackFromEltSize * 2;
6164 bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
6165 if (!IsZextByte)
6166 SrcBytes.push_back(Elt: Bytes[Elt]);
6167 if (Bytes[Elt] != -1) {
6168 unsigned OpNo = unsigned(Bytes[Elt]) / SystemZ::VectorBytes;
6169 if (IsZextByte != (OpNo == ZeroVecOpNo)) {
6170 MatchUnpack = false;
6171 break;
6172 }
6173 }
6174 }
6175 if (MatchUnpack) {
6176 if (Ops.size() == 2) {
6177 // Don't use unpack if a single source operand needs rearrangement.
6178 bool CanUseUnpackLow = true, CanUseUnpackHigh = true;
6179 for (unsigned i = 0; i < SystemZ::VectorBytes / 2; i++) {
6180 if (SrcBytes[i] == -1)
6181 continue;
6182 if (SrcBytes[i] % 16 != int(i))
6183 CanUseUnpackHigh = false;
6184 if (SrcBytes[i] % 16 != int(i + SystemZ::VectorBytes / 2))
6185 CanUseUnpackLow = false;
6186 if (!CanUseUnpackLow && !CanUseUnpackHigh) {
6187 UnpackFromEltSize = UINT_MAX;
6188 return;
6189 }
6190 }
6191 if (!CanUseUnpackHigh)
6192 UnpackLow = true;
6193 }
6194 break;
6195 }
6196 }
6197 if (UnpackFromEltSize > 4)
6198 return;
6199
6200 LLVM_DEBUG(dbgs() << "Preparing for final unpack of element size "
6201 << UnpackFromEltSize << ". Zero vector is Op#" << ZeroVecOpNo
6202 << ".\n";
6203 dumpBytes(Bytes, "Original Bytes vector:"););
6204
6205 // Apply the unpack in reverse to the Bytes array.
6206 unsigned B = 0;
6207 if (UnpackLow) {
6208 while (B < SystemZ::VectorBytes / 2)
6209 Bytes[B++] = -1;
6210 }
6211 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes;) {
6212 Elt += UnpackFromEltSize;
6213 for (unsigned i = 0; i < UnpackFromEltSize; i++, Elt++, B++)
6214 Bytes[B] = Bytes[Elt];
6215 }
6216 if (!UnpackLow) {
6217 while (B < SystemZ::VectorBytes)
6218 Bytes[B++] = -1;
6219 }
6220
6221 // Remove the zero vector from Ops
6222 Ops.erase(CI: &Ops[ZeroVecOpNo]);
6223 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6224 if (Bytes[I] >= 0) {
6225 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6226 if (OpNo > ZeroVecOpNo)
6227 Bytes[I] -= SystemZ::VectorBytes;
6228 }
6229
6230 LLVM_DEBUG(dumpBytes(Bytes, "Resulting Bytes vector, zero vector removed:");
6231 dbgs() << "\n";);
6232}
6233
6234SDValue GeneralShuffle::insertUnpackIfPrepared(SelectionDAG &DAG,
6235 const SDLoc &DL,
6236 SDValue Op) {
6237 if (!unpackWasPrepared())
6238 return Op;
6239 unsigned InBits = UnpackFromEltSize * 8;
6240 EVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBits),
6241 NumElements: SystemZ::VectorBits / InBits);
6242 SDValue PackedOp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op);
6243 unsigned OutBits = InBits * 2;
6244 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: OutBits),
6245 NumElements: SystemZ::VectorBits / OutBits);
6246 return DAG.getNode(Opcode: UnpackLow ? SystemZISD::UNPACKL_LOW
6247 : SystemZISD::UNPACKL_HIGH,
6248 DL, VT: OutVT, Operand: PackedOp);
6249}
6250
6251// Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
6252static bool isScalarToVector(SDValue Op) {
6253 for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
6254 if (!Op.getOperand(i: I).isUndef())
6255 return false;
6256 return true;
6257}
6258
6259// Return a vector of type VT that contains Value in the first element.
6260// The other elements don't matter.
6261static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6262 SDValue Value) {
6263 // If we have a constant, replicate it to all elements and let the
6264 // BUILD_VECTOR lowering take care of it.
6265 if (Value.getOpcode() == ISD::Constant ||
6266 Value.getOpcode() == ISD::ConstantFP) {
6267 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value);
6268 return DAG.getBuildVector(VT, DL, Ops);
6269 }
6270 if (Value.isUndef())
6271 return DAG.getUNDEF(VT);
6272 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Value);
6273}
6274
6275// Return a vector of type VT in which Op0 is in element 0 and Op1 is in
6276// element 1. Used for cases in which replication is cheap.
6277static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6278 SDValue Op0, SDValue Op1) {
6279 if (Op0.isUndef()) {
6280 if (Op1.isUndef())
6281 return DAG.getUNDEF(VT);
6282 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op1);
6283 }
6284 if (Op1.isUndef())
6285 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0);
6286 return DAG.getNode(Opcode: SystemZISD::MERGE_HIGH, DL, VT,
6287 N1: buildScalarToVector(DAG, DL, VT, Value: Op0),
6288 N2: buildScalarToVector(DAG, DL, VT, Value: Op1));
6289}
6290
6291// Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
6292// vector for them.
6293static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
6294 SDValue Op1) {
6295 if (Op0.isUndef() && Op1.isUndef())
6296 return DAG.getUNDEF(VT: MVT::v2i64);
6297 // If one of the two inputs is undefined then replicate the other one,
6298 // in order to avoid using another register unnecessarily.
6299 if (Op0.isUndef())
6300 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6301 else if (Op1.isUndef())
6302 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6303 else {
6304 Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6305 Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6306 }
6307 return DAG.getNode(Opcode: SystemZISD::JOIN_DWORDS, DL, VT: MVT::v2i64, N1: Op0, N2: Op1);
6308}
6309
6310// If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
6311// better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
6312// the non-EXTRACT_VECTOR_ELT elements. See if the given BUILD_VECTOR
6313// would benefit from this representation and return it if so.
6314static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
6315 BuildVectorSDNode *BVN) {
6316 EVT VT = BVN->getValueType(ResNo: 0);
6317 unsigned NumElements = VT.getVectorNumElements();
6318
6319 // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
6320 // on byte vectors. If there are non-EXTRACT_VECTOR_ELT elements that still
6321 // need a BUILD_VECTOR, add an additional placeholder operand for that
6322 // BUILD_VECTOR and store its operands in ResidueOps.
6323 GeneralShuffle GS(VT);
6324 SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
6325 bool FoundOne = false;
6326 for (unsigned I = 0; I < NumElements; ++I) {
6327 SDValue Op = BVN->getOperand(Num: I);
6328 if (Op.getOpcode() == ISD::TRUNCATE)
6329 Op = Op.getOperand(i: 0);
6330 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6331 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
6332 unsigned Elem = Op.getConstantOperandVal(i: 1);
6333 if (!GS.add(Op: Op.getOperand(i: 0), Elem))
6334 return SDValue();
6335 FoundOne = true;
6336 } else if (Op.isUndef()) {
6337 GS.addUndef();
6338 } else {
6339 if (!GS.add(Op: SDValue(), Elem: ResidueOps.size()))
6340 return SDValue();
6341 ResidueOps.push_back(Elt: BVN->getOperand(Num: I));
6342 }
6343 }
6344
6345 // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
6346 if (!FoundOne)
6347 return SDValue();
6348
6349 // Create the BUILD_VECTOR for the remaining elements, if any.
6350 if (!ResidueOps.empty()) {
6351 while (ResidueOps.size() < NumElements)
6352 ResidueOps.push_back(Elt: DAG.getUNDEF(VT: ResidueOps[0].getValueType()));
6353 for (auto &Op : GS.Ops) {
6354 if (!Op.getNode()) {
6355 Op = DAG.getBuildVector(VT, DL: SDLoc(BVN), Ops: ResidueOps);
6356 break;
6357 }
6358 }
6359 }
6360 return GS.getNode(DAG, DL: SDLoc(BVN));
6361}
6362
6363bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
6364 if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Val&: Op)->isUnindexed())
6365 return true;
6366 if (auto *AL = dyn_cast<AtomicSDNode>(Val&: Op))
6367 if (AL->getOpcode() == ISD::ATOMIC_LOAD)
6368 return true;
6369 if (Subtarget.hasVectorEnhancements2() && Op.getOpcode() == SystemZISD::LRV)
6370 return true;
6371 return false;
6372}
6373
6374static SDValue mergeHighParts(SelectionDAG &DAG, const SDLoc &DL,
6375 unsigned MergedBits, EVT VT, SDValue Op0,
6376 SDValue Op1) {
6377 MVT IntVecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: MergedBits),
6378 NumElements: SystemZ::VectorBits / MergedBits);
6379 assert(VT.getSizeInBits() == 128 && IntVecVT.getSizeInBits() == 128 &&
6380 "Handling full vectors only.");
6381 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0);
6382 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op1);
6383 SDValue Op = DAG.getNode(Opcode: SystemZISD::MERGE_HIGH, DL, VT: IntVecVT, N1: Op0, N2: Op1);
6384 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6385}
6386
6387static SDValue buildFPVecFromScalars4(SelectionDAG &DAG, const SDLoc &DL,
6388 EVT VT, SmallVectorImpl<SDValue> &Elems,
6389 unsigned Pos) {
6390 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Op0: Elems[Pos + 0], Op1: Elems[Pos + 1]);
6391 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Op0: Elems[Pos + 2], Op1: Elems[Pos + 3]);
6392 // Avoid unnecessary undefs by reusing the other operand.
6393 if (Op01.isUndef()) {
6394 if (Op23.isUndef())
6395 return Op01;
6396 Op01 = Op23;
6397 } else if (Op23.isUndef())
6398 Op23 = Op01;
6399 // Merging identical replications is a no-op.
6400 if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
6401 return Op01;
6402 unsigned MergedBits = VT.getSimpleVT().getScalarSizeInBits() * 2;
6403 return mergeHighParts(DAG, DL, MergedBits, VT, Op0: Op01, Op1: Op23);
6404}
6405
6406// Combine GPR scalar values Elems into a vector of type VT.
6407SDValue
6408SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6409 SmallVectorImpl<SDValue> &Elems) const {
6410 // See whether there is a single replicated value.
6411 SDValue Single;
6412 unsigned int NumElements = Elems.size();
6413 unsigned int Count = 0;
6414 for (auto Elem : Elems) {
6415 if (!Elem.isUndef()) {
6416 if (!Single.getNode())
6417 Single = Elem;
6418 else if (Elem != Single) {
6419 Single = SDValue();
6420 break;
6421 }
6422 Count += 1;
6423 }
6424 }
6425 // There are three cases here:
6426 //
6427 // - if the only defined element is a loaded one, the best sequence
6428 // is a replicating load.
6429 //
6430 // - otherwise, if the only defined element is an i64 value, we will
6431 // end up with the same VLVGP sequence regardless of whether we short-cut
6432 // for replication or fall through to the later code.
6433 //
6434 // - otherwise, if the only defined element is an i32 or smaller value,
6435 // we would need 2 instructions to replicate it: VLVGP followed by VREPx.
6436 // This is only a win if the single defined element is used more than once.
6437 // In other cases we're better off using a single VLVGx.
6438 if (Single.getNode() && (Count > 1 || isVectorElementLoad(Op: Single)))
6439 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Single);
6440
6441 // If all elements are loads, use VLREP/VLEs (below).
6442 bool AllLoads = true;
6443 for (auto Elem : Elems)
6444 if (!isVectorElementLoad(Op: Elem)) {
6445 AllLoads = false;
6446 break;
6447 }
6448
6449 // The best way of building a v2i64 from two i64s is to use VLVGP.
6450 if (VT == MVT::v2i64 && !AllLoads)
6451 return joinDwords(DAG, DL, Op0: Elems[0], Op1: Elems[1]);
6452
6453 // Use a 64-bit merge high to combine two doubles.
6454 if (VT == MVT::v2f64 && !AllLoads)
6455 return buildMergeScalars(DAG, DL, VT, Op0: Elems[0], Op1: Elems[1]);
6456
6457 // Build v4f32 values directly from the FPRs:
6458 //
6459 // <Axxx> <Bxxx> <Cxxxx> <Dxxx>
6460 // V V VMRHF
6461 // <ABxx> <CDxx>
6462 // V VMRHG
6463 // <ABCD>
6464 if (VT == MVT::v4f32 && !AllLoads)
6465 return buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 0);
6466
6467 // Same for v8f16.
6468 if (VT == MVT::v8f16 && !AllLoads) {
6469 SDValue Op0123 = buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 0);
6470 SDValue Op4567 = buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 4);
6471 // Avoid unnecessary undefs by reusing the other operand.
6472 if (Op0123.isUndef())
6473 Op0123 = Op4567;
6474 else if (Op4567.isUndef())
6475 Op4567 = Op0123;
6476 // Merging identical replications is a no-op.
6477 if (Op0123.getOpcode() == SystemZISD::REPLICATE && Op0123 == Op4567)
6478 return Op0123;
6479 return mergeHighParts(DAG, DL, MergedBits: 64, VT, Op0: Op0123, Op1: Op4567);
6480 }
6481
6482 // Collect the constant terms.
6483 SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
6484 SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false);
6485
6486 unsigned NumConstants = 0;
6487 for (unsigned I = 0; I < NumElements; ++I) {
6488 SDValue Elem = Elems[I];
6489 if (Elem.getOpcode() == ISD::Constant ||
6490 Elem.getOpcode() == ISD::ConstantFP) {
6491 NumConstants += 1;
6492 Constants[I] = Elem;
6493 Done[I] = true;
6494 }
6495 }
6496 // If there was at least one constant, fill in the other elements of
6497 // Constants with undefs to get a full vector constant and use that
6498 // as the starting point.
6499 SDValue Result;
6500 SDValue ReplicatedVal;
6501 if (NumConstants > 0) {
6502 for (unsigned I = 0; I < NumElements; ++I)
6503 if (!Constants[I].getNode())
6504 Constants[I] = DAG.getUNDEF(VT: Elems[I].getValueType());
6505 Result = DAG.getBuildVector(VT, DL, Ops: Constants);
6506 } else {
6507 // Otherwise try to use VLREP or VLVGP to start the sequence in order to
6508 // avoid a false dependency on any previous contents of the vector
6509 // register.
6510
6511 // Use a VLREP if at least one element is a load. Make sure to replicate
6512 // the load with the most elements having its value.
6513 std::map<const SDNode*, unsigned> UseCounts;
6514 SDNode *LoadMaxUses = nullptr;
6515 for (unsigned I = 0; I < NumElements; ++I)
6516 if (isVectorElementLoad(Op: Elems[I])) {
6517 SDNode *Ld = Elems[I].getNode();
6518 unsigned Count = ++UseCounts[Ld];
6519 if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < Count)
6520 LoadMaxUses = Ld;
6521 }
6522 if (LoadMaxUses != nullptr) {
6523 ReplicatedVal = SDValue(LoadMaxUses, 0);
6524 Result = DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: ReplicatedVal);
6525 } else {
6526 // Try to use VLVGP.
6527 unsigned I1 = NumElements / 2 - 1;
6528 unsigned I2 = NumElements - 1;
6529 bool Def1 = !Elems[I1].isUndef();
6530 bool Def2 = !Elems[I2].isUndef();
6531 if (Def1 || Def2) {
6532 SDValue Elem1 = Elems[Def1 ? I1 : I2];
6533 SDValue Elem2 = Elems[Def2 ? I2 : I1];
6534 Result = DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
6535 Operand: joinDwords(DAG, DL, Op0: Elem1, Op1: Elem2));
6536 Done[I1] = true;
6537 Done[I2] = true;
6538 } else
6539 Result = DAG.getUNDEF(VT);
6540 }
6541 }
6542
6543 // Use VLVGx to insert the other elements.
6544 for (unsigned I = 0; I < NumElements; ++I)
6545 if (!Done[I] && !Elems[I].isUndef() && Elems[I] != ReplicatedVal)
6546 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: Result, N2: Elems[I],
6547 N3: DAG.getConstant(Val: I, DL, VT: MVT::i32));
6548 return Result;
6549}
6550
6551SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
6552 SelectionDAG &DAG) const {
6553 auto *BVN = cast<BuildVectorSDNode>(Val: Op.getNode());
6554 SDLoc DL(Op);
6555 EVT VT = Op.getValueType();
6556
6557 if (BVN->isConstant()) {
6558 if (SystemZVectorConstantInfo(BVN).isVectorConstantLegal(Subtarget))
6559 return Op;
6560
6561 // Fall back to loading it from memory.
6562 return SDValue();
6563 }
6564
6565 // See if we should use shuffles to construct the vector from other vectors.
6566 if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
6567 return Res;
6568
6569 // Detect SCALAR_TO_VECTOR conversions.
6570 if (isOperationLegal(Op: ISD::SCALAR_TO_VECTOR, VT) && isScalarToVector(Op))
6571 return buildScalarToVector(DAG, DL, VT, Value: Op.getOperand(i: 0));
6572
6573 // Otherwise use buildVector to build the vector up from GPRs.
6574 unsigned NumElements = Op.getNumOperands();
6575 SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
6576 for (unsigned I = 0; I < NumElements; ++I)
6577 Ops[I] = Op.getOperand(i: I);
6578 return buildVector(DAG, DL, VT, Elems&: Ops);
6579}
6580
6581SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6582 SelectionDAG &DAG) const {
6583 auto *VSN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
6584 SDLoc DL(Op);
6585 EVT VT = Op.getValueType();
6586 unsigned NumElements = VT.getVectorNumElements();
6587
6588 if (VSN->isSplat()) {
6589 SDValue Op0 = Op.getOperand(i: 0);
6590 unsigned Index = VSN->getSplatIndex();
6591 assert(Index < VT.getVectorNumElements() &&
6592 "Splat index should be defined and in first operand");
6593 // See whether the value we're splatting is directly available as a scalar.
6594 if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6595 Op0.getOpcode() == ISD::BUILD_VECTOR)
6596 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0.getOperand(i: Index));
6597 // Otherwise keep it as a vector-to-vector operation.
6598 return DAG.getNode(Opcode: SystemZISD::SPLAT, DL, VT, N1: Op.getOperand(i: 0),
6599 N2: DAG.getTargetConstant(Val: Index, DL, VT: MVT::i32));
6600 }
6601
6602 GeneralShuffle GS(VT);
6603 for (unsigned I = 0; I < NumElements; ++I) {
6604 int Elt = VSN->getMaskElt(Idx: I);
6605 if (Elt < 0)
6606 GS.addUndef();
6607 else if (!GS.add(Op: Op.getOperand(i: unsigned(Elt) / NumElements),
6608 Elem: unsigned(Elt) % NumElements))
6609 return SDValue();
6610 }
6611 return GS.getNode(DAG, DL: SDLoc(VSN));
6612}
6613
6614SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
6615 SelectionDAG &DAG) const {
6616 SDLoc DL(Op);
6617 // Just insert the scalar into element 0 of an undefined vector.
6618 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL,
6619 VT: Op.getValueType(), N1: DAG.getUNDEF(VT: Op.getValueType()),
6620 N2: Op.getOperand(i: 0), N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
6621}
6622
6623// Shift the lower 2 bytes of Op to the left in order to insert into the
6624// upper 2 bytes of the FP register.
6625static SDValue convertToF16(SDValue Op, SelectionDAG &DAG) {
6626 assert(Op.getSimpleValueType() == MVT::i64 &&
6627 "Expexted to convert i64 to f16.");
6628 SDLoc DL(Op);
6629 SDValue Shft = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: Op,
6630 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i64));
6631 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Shft);
6632 SDValue F16Val =
6633 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f16, Operand: BCast);
6634 return F16Val;
6635}
6636
6637// Extract Op into GPR and shift the 2 f16 bytes to the right.
6638static SDValue convertFromF16(SDValue Op, SDLoc DL, SelectionDAG &DAG) {
6639 assert(Op.getSimpleValueType() == MVT::f16 &&
6640 "Expected to convert f16 to i64.");
6641 SDNode *U32 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
6642 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f64,
6643 Operand: SDValue(U32, 0), Subreg: Op);
6644 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
6645 SDValue Shft = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: BCast,
6646 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i32));
6647 return Shft;
6648}
6649
6650SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
6651 SelectionDAG &DAG) const {
6652 // Handle insertions of floating-point values.
6653 SDLoc DL(Op);
6654 SDValue Op0 = Op.getOperand(i: 0);
6655 SDValue Op1 = Op.getOperand(i: 1);
6656 SDValue Op2 = Op.getOperand(i: 2);
6657 EVT VT = Op.getValueType();
6658
6659 // Insertions into constant indices of a v2f64 can be done using VPDI.
6660 // However, if the inserted value is a bitcast or a constant then it's
6661 // better to use GPRs, as below.
6662 if (VT == MVT::v2f64 &&
6663 Op1.getOpcode() != ISD::BITCAST &&
6664 Op1.getOpcode() != ISD::ConstantFP &&
6665 Op2.getOpcode() == ISD::Constant) {
6666 uint64_t Index = Op2->getAsZExtVal();
6667 unsigned Mask = VT.getVectorNumElements() - 1;
6668 if (Index <= Mask)
6669 return Op;
6670 }
6671
6672 // Otherwise bitcast to the equivalent integer form and insert via a GPR.
6673 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits());
6674 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VT.getVectorNumElements());
6675 SDValue IntOp1 =
6676 VT == MVT::v8f16
6677 ? DAG.getZExtOrTrunc(Op: convertFromF16(Op: Op1, DL, DAG), DL, VT: MVT::i32)
6678 : DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Op1);
6679 SDValue Res =
6680 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: IntVecVT,
6681 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0), N2: IntOp1, N3: Op2);
6682 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
6683}
6684
6685SDValue
6686SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
6687 SelectionDAG &DAG) const {
6688 // Handle extractions of floating-point values.
6689 SDLoc DL(Op);
6690 SDValue Op0 = Op.getOperand(i: 0);
6691 SDValue Op1 = Op.getOperand(i: 1);
6692 EVT VT = Op.getValueType();
6693 EVT VecVT = Op0.getValueType();
6694
6695 // Extractions of constant indices can be done directly.
6696 if (auto *CIndexN = dyn_cast<ConstantSDNode>(Val&: Op1)) {
6697 uint64_t Index = CIndexN->getZExtValue();
6698 unsigned Mask = VecVT.getVectorNumElements() - 1;
6699 if (Index <= Mask)
6700 return Op;
6701 }
6702
6703 // Otherwise bitcast to the equivalent integer form and extract via a GPR.
6704 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getSizeInBits());
6705 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VecVT.getVectorNumElements());
6706 MVT ExtrVT = IntVT == MVT::i16 ? MVT::i32 : IntVT;
6707 SDValue Extr = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ExtrVT,
6708 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0), N2: Op1);
6709 if (VT == MVT::f16)
6710 return convertToF16(Op: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Extr), DAG);
6711 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Extr);
6712}
6713
6714SDValue SystemZTargetLowering::
6715lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6716 SDValue PackedOp = Op.getOperand(i: 0);
6717 EVT OutVT = Op.getValueType();
6718 EVT InVT = PackedOp.getValueType();
6719 unsigned ToBits = OutVT.getScalarSizeInBits();
6720 unsigned FromBits = InVT.getScalarSizeInBits();
6721 unsigned StartOffset = 0;
6722
6723 // If the input is a VECTOR_SHUFFLE, there are a number of important
6724 // cases where we can directly implement the sign-extension of the
6725 // original input lanes of the shuffle.
6726 if (PackedOp.getOpcode() == ISD::VECTOR_SHUFFLE) {
6727 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: PackedOp.getNode());
6728 ArrayRef<int> ShuffleMask = SVN->getMask();
6729 int OutNumElts = OutVT.getVectorNumElements();
6730
6731 // Recognize the special case where the sign-extension can be done
6732 // by the VSEG instruction. Handled via the default expander.
6733 if (ToBits == 64 && OutNumElts == 2) {
6734 int NumElem = ToBits / FromBits;
6735 if (ShuffleMask[0] == NumElem - 1 && ShuffleMask[1] == 2 * NumElem - 1)
6736 return SDValue();
6737 }
6738
6739 // Recognize the special case where we can fold the shuffle by
6740 // replacing some of the UNPACK_HIGH with UNPACK_LOW.
6741 int StartOffsetCandidate = -1;
6742 for (int Elt = 0; Elt < OutNumElts; Elt++) {
6743 if (ShuffleMask[Elt] == -1)
6744 continue;
6745 if (ShuffleMask[Elt] % OutNumElts == Elt) {
6746 if (StartOffsetCandidate == -1)
6747 StartOffsetCandidate = ShuffleMask[Elt] - Elt;
6748 if (StartOffsetCandidate == ShuffleMask[Elt] - Elt)
6749 continue;
6750 }
6751 StartOffsetCandidate = -1;
6752 break;
6753 }
6754 if (StartOffsetCandidate != -1) {
6755 StartOffset = StartOffsetCandidate;
6756 PackedOp = PackedOp.getOperand(i: 0);
6757 }
6758 }
6759
6760 do {
6761 FromBits *= 2;
6762 unsigned OutNumElts = SystemZ::VectorBits / FromBits;
6763 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: FromBits), NumElements: OutNumElts);
6764 unsigned Opcode = SystemZISD::UNPACK_HIGH;
6765 if (StartOffset >= OutNumElts) {
6766 Opcode = SystemZISD::UNPACK_LOW;
6767 StartOffset -= OutNumElts;
6768 }
6769 PackedOp = DAG.getNode(Opcode, DL: SDLoc(PackedOp), VT: OutVT, Operand: PackedOp);
6770 } while (FromBits != ToBits);
6771 return PackedOp;
6772}
6773
6774// Lower a ZERO_EXTEND_VECTOR_INREG to a vector shuffle with a zero vector.
6775SDValue SystemZTargetLowering::
6776lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6777 SDValue PackedOp = Op.getOperand(i: 0);
6778 SDLoc DL(Op);
6779 EVT OutVT = Op.getValueType();
6780 EVT InVT = PackedOp.getValueType();
6781 unsigned InNumElts = InVT.getVectorNumElements();
6782 unsigned OutNumElts = OutVT.getVectorNumElements();
6783 unsigned NumInPerOut = InNumElts / OutNumElts;
6784
6785 SDValue ZeroVec =
6786 DAG.getSplatVector(VT: InVT, DL, Op: DAG.getConstant(Val: 0, DL, VT: InVT.getScalarType()));
6787
6788 SmallVector<int, 16> Mask(InNumElts);
6789 unsigned ZeroVecElt = InNumElts;
6790 for (unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
6791 unsigned MaskElt = PackedElt * NumInPerOut;
6792 unsigned End = MaskElt + NumInPerOut - 1;
6793 for (; MaskElt < End; MaskElt++)
6794 Mask[MaskElt] = ZeroVecElt++;
6795 Mask[MaskElt] = PackedElt;
6796 }
6797 SDValue Shuf = DAG.getVectorShuffle(VT: InVT, dl: DL, N1: PackedOp, N2: ZeroVec, Mask);
6798 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: OutVT, Operand: Shuf);
6799}
6800
6801SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
6802 unsigned ByScalar) const {
6803 // Look for cases where a vector shift can use the *_BY_SCALAR form.
6804 SDValue Op0 = Op.getOperand(i: 0);
6805 SDValue Op1 = Op.getOperand(i: 1);
6806 SDLoc DL(Op);
6807 EVT VT = Op.getValueType();
6808 unsigned ElemBitSize = VT.getScalarSizeInBits();
6809
6810 // See whether the shift vector is a splat represented as BUILD_VECTOR.
6811 if (auto *BVN = dyn_cast<BuildVectorSDNode>(Val&: Op1)) {
6812 APInt SplatBits, SplatUndef;
6813 unsigned SplatBitSize;
6814 bool HasAnyUndefs;
6815 // Check for constant splats. Use ElemBitSize as the minimum element
6816 // width and reject splats that need wider elements.
6817 if (BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6818 MinSplatBits: ElemBitSize, isBigEndian: true) &&
6819 SplatBitSize == ElemBitSize) {
6820 SDValue Shift = DAG.getConstant(Val: SplatBits.getZExtValue() & 0xfff,
6821 DL, VT: MVT::i32);
6822 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6823 }
6824 // Check for variable splats.
6825 BitVector UndefElements;
6826 SDValue Splat = BVN->getSplatValue(UndefElements: &UndefElements);
6827 if (Splat) {
6828 // Since i32 is the smallest legal type, we either need a no-op
6829 // or a truncation.
6830 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Splat);
6831 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6832 }
6833 }
6834
6835 // See whether the shift vector is a splat represented as SHUFFLE_VECTOR,
6836 // and the shift amount is directly available in a GPR.
6837 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: Op1)) {
6838 if (VSN->isSplat()) {
6839 SDValue VSNOp0 = VSN->getOperand(Num: 0);
6840 unsigned Index = VSN->getSplatIndex();
6841 assert(Index < VT.getVectorNumElements() &&
6842 "Splat index should be defined and in first operand");
6843 if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6844 VSNOp0.getOpcode() == ISD::BUILD_VECTOR) {
6845 // Since i32 is the smallest legal type, we either need a no-op
6846 // or a truncation.
6847 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32,
6848 Operand: VSNOp0.getOperand(i: Index));
6849 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6850 }
6851 }
6852 }
6853
6854 // Otherwise just treat the current form as legal.
6855 return Op;
6856}
6857
6858SDValue SystemZTargetLowering::lowerFSHL(SDValue Op, SelectionDAG &DAG) const {
6859 SDLoc DL(Op);
6860
6861 // i128 FSHL with a constant amount that is a multiple of 8 can be
6862 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
6863 // facility, FSHL with a constant amount less than 8 can be implemented
6864 // via SHL_DOUBLE_BIT, and FSHL with other constant amounts by a
6865 // combination of the two.
6866 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
6867 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
6868 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
6869 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
6870 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
6871 if (ShiftAmt > 120) {
6872 // For N in 121..128, fshl N == fshr (128 - N), and for 1 <= N < 8
6873 // SHR_DOUBLE_BIT emits fewer instructions.
6874 SDValue Val =
6875 DAG.getNode(Opcode: SystemZISD::SHR_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Op0, N2: Op1,
6876 N3: DAG.getTargetConstant(Val: 128 - ShiftAmt, DL, VT: MVT::i32));
6877 return DAG.getBitcast(VT: MVT::i128, V: Val);
6878 }
6879 SmallVector<int, 16> Mask(16);
6880 for (unsigned Elt = 0; Elt < 16; Elt++)
6881 Mask[Elt] = (ShiftAmt >> 3) + Elt;
6882 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
6883 if ((ShiftAmt & 7) == 0)
6884 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
6885 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op1, N2: Op1, Mask);
6886 SDValue Val =
6887 DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf1, N2: Shuf2,
6888 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
6889 return DAG.getBitcast(VT: MVT::i128, V: Val);
6890 }
6891 }
6892
6893 return SDValue();
6894}
6895
6896SDValue SystemZTargetLowering::lowerFSHR(SDValue Op, SelectionDAG &DAG) const {
6897 SDLoc DL(Op);
6898
6899 // i128 FSHR with a constant amount that is a multiple of 8 can be
6900 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
6901 // facility, FSHR with a constant amount less than 8 can be implemented
6902 // via SHR_DOUBLE_BIT, and FSHR with other constant amounts by a
6903 // combination of the two.
6904 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
6905 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
6906 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
6907 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
6908 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
6909 if (ShiftAmt > 120) {
6910 // For N in 121..128, fshr N == fshl (128 - N), and for 1 <= N < 8
6911 // SHL_DOUBLE_BIT emits fewer instructions.
6912 SDValue Val =
6913 DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Op0, N2: Op1,
6914 N3: DAG.getTargetConstant(Val: 128 - ShiftAmt, DL, VT: MVT::i32));
6915 return DAG.getBitcast(VT: MVT::i128, V: Val);
6916 }
6917 SmallVector<int, 16> Mask(16);
6918 for (unsigned Elt = 0; Elt < 16; Elt++)
6919 Mask[Elt] = 16 - (ShiftAmt >> 3) + Elt;
6920 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
6921 if ((ShiftAmt & 7) == 0)
6922 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
6923 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op0, Mask);
6924 SDValue Val =
6925 DAG.getNode(Opcode: SystemZISD::SHR_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf2, N2: Shuf1,
6926 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
6927 return DAG.getBitcast(VT: MVT::i128, V: Val);
6928 }
6929 }
6930
6931 return SDValue();
6932}
6933
6934static SDValue lowerAddrSpaceCast(SDValue Op, SelectionDAG &DAG) {
6935 SDLoc DL(Op);
6936 SDValue Src = Op.getOperand(i: 0);
6937 MVT DstVT = Op.getSimpleValueType();
6938
6939 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Val: Op.getNode());
6940 unsigned SrcAS = N->getSrcAddressSpace();
6941
6942 assert(SrcAS != N->getDestAddressSpace() &&
6943 "addrspacecast must be between different address spaces");
6944
6945 // addrspacecast [0 <- 1] : Assinging a ptr32 value to a 64-bit pointer.
6946 // addrspacecast [1 <- 0] : Assigining a 64-bit pointer to a ptr32 value.
6947 if (SrcAS == SYSTEMZAS::PTR32 && DstVT == MVT::i64) {
6948 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Src,
6949 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
6950 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
6951 } else if (DstVT == MVT::i32) {
6952 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Src);
6953 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Op,
6954 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
6955 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
6956 } else {
6957 report_fatal_error(reason: "Bad address space in addrspacecast");
6958 }
6959 return Op;
6960}
6961
6962SDValue SystemZTargetLowering::lowerFP_EXTEND(SDValue Op,
6963 SelectionDAG &DAG) const {
6964 SDValue In = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0);
6965 if (In.getSimpleValueType() != MVT::f16)
6966 return Op; // Legal
6967 return SDValue(); // Let legalizer emit the libcall.
6968}
6969
6970SDValue SystemZTargetLowering::useLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
6971 MVT VT, SDValue Arg, SDLoc DL,
6972 SDValue Chain, bool IsStrict) const {
6973 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
6974 MakeLibCallOptions CallOptions;
6975 SDValue Result;
6976 std::tie(args&: Result, args&: Chain) =
6977 makeLibCall(DAG, LC, RetVT: VT, Ops: Arg, CallOptions, dl: DL, Chain);
6978 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: DL) : Result;
6979}
6980
6981SDValue SystemZTargetLowering::lower_FP_TO_INT(SDValue Op,
6982 SelectionDAG &DAG) const {
6983 bool IsSigned = (Op->getOpcode() == ISD::FP_TO_SINT ||
6984 Op->getOpcode() == ISD::STRICT_FP_TO_SINT);
6985 bool IsStrict = Op->isStrictFPOpcode();
6986 SDLoc DL(Op);
6987 MVT VT = Op.getSimpleValueType();
6988 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
6989 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
6990 EVT InVT = InOp.getValueType();
6991
6992 // FP to unsigned is not directly supported on z10. Promoting an i32
6993 // result to (signed) i64 doesn't generate an inexact condition (fp
6994 // exception) for values that are outside the i32 range but in the i64
6995 // range, so use the default expansion.
6996 if (!Subtarget.hasFPExtension() && !IsSigned)
6997 // Expand i32/i64. F16 values will be recognized to fit and extended.
6998 return SDValue();
6999
7000 // Conversion from f16 is done via f32.
7001 if (InOp.getSimpleValueType() == MVT::f16) {
7002 SmallVector<SDValue, 2> Results;
7003 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
7004 return DAG.getMergeValues(Ops: Results, dl: DL);
7005 }
7006
7007 if (VT == MVT::i128) {
7008 RTLIB::Libcall LC =
7009 IsSigned ? RTLIB::getFPTOSINT(OpVT: InVT, RetVT: VT) : RTLIB::getFPTOUINT(OpVT: InVT, RetVT: VT);
7010 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
7011 }
7012
7013 return Op; // Legal
7014}
7015
7016SDValue SystemZTargetLowering::lower_INT_TO_FP(SDValue Op,
7017 SelectionDAG &DAG) const {
7018 bool IsSigned = (Op->getOpcode() == ISD::SINT_TO_FP ||
7019 Op->getOpcode() == ISD::STRICT_SINT_TO_FP);
7020 bool IsStrict = Op->isStrictFPOpcode();
7021 SDLoc DL(Op);
7022 MVT VT = Op.getSimpleValueType();
7023 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
7024 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
7025 EVT InVT = InOp.getValueType();
7026
7027 // Conversion to f16 is done via f32.
7028 if (VT == MVT::f16) {
7029 SmallVector<SDValue, 2> Results;
7030 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
7031 return DAG.getMergeValues(Ops: Results, dl: DL);
7032 }
7033
7034 // Unsigned to fp is not directly supported on z10.
7035 if (!Subtarget.hasFPExtension() && !IsSigned)
7036 return SDValue(); // Expand i64.
7037
7038 if (InVT == MVT::i128) {
7039 RTLIB::Libcall LC =
7040 IsSigned ? RTLIB::getSINTTOFP(OpVT: InVT, RetVT: VT) : RTLIB::getUINTTOFP(OpVT: InVT, RetVT: VT);
7041 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
7042 }
7043
7044 return Op; // Legal
7045}
7046
7047// Lower an f16 LOAD in case of no vector support.
7048SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
7049 SelectionDAG &DAG) const {
7050 EVT RegVT = Op.getValueType();
7051 assert(RegVT == MVT::f16 && "Expected to lower an f16 load.");
7052 (void)RegVT;
7053
7054 // Load as integer.
7055 SDLoc DL(Op);
7056 SDValue NewLd;
7057 if (auto *AtomicLd = dyn_cast<AtomicSDNode>(Val: Op.getNode())) {
7058 assert(EVT(RegVT) == AtomicLd->getMemoryVT() && "Unhandled f16 load");
7059 NewLd = DAG.getAtomicLoad(ExtType: ISD::EXTLOAD, dl: DL, MemVT: MVT::i16, VT: MVT::i64,
7060 Chain: AtomicLd->getChain(), Ptr: AtomicLd->getBasePtr(),
7061 MMO: AtomicLd->getMemOperand());
7062 } else {
7063 LoadSDNode *Ld = cast<LoadSDNode>(Val: Op.getNode());
7064 assert(EVT(RegVT) == Ld->getMemoryVT() && "Unhandled f16 load");
7065 NewLd = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i64, Chain: Ld->getChain(),
7066 Ptr: Ld->getBasePtr(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i16,
7067 Alignment: Ld->getBaseAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
7068 }
7069 SDValue F16Val = convertToF16(Op: NewLd, DAG);
7070 return DAG.getMergeValues(Ops: {F16Val, NewLd.getValue(R: 1)}, dl: DL);
7071}
7072
7073// Lower an f16 STORE in case of no vector support.
7074SDValue SystemZTargetLowering::lowerStoreF16(SDValue Op,
7075 SelectionDAG &DAG) const {
7076 SDLoc DL(Op);
7077 SDValue Shft = convertFromF16(Op: Op->getOperand(Num: 1), DL, DAG);
7078
7079 if (auto *AtomicSt = dyn_cast<AtomicSDNode>(Val: Op.getNode()))
7080 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MVT::i16, Chain: AtomicSt->getChain(),
7081 Ptr: Shft, Val: AtomicSt->getBasePtr(),
7082 MMO: AtomicSt->getMemOperand());
7083
7084 StoreSDNode *St = cast<StoreSDNode>(Val: Op.getNode());
7085 return DAG.getTruncStore(Chain: St->getChain(), dl: DL, Val: Shft, Ptr: St->getBasePtr(), SVT: MVT::i16,
7086 MMO: St->getMemOperand());
7087}
7088
7089SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op,
7090 SelectionDAG &DAG) const {
7091 SDLoc DL(Op);
7092 MVT ResultVT = Op.getSimpleValueType();
7093 SDValue Arg = Op.getOperand(i: 0);
7094 unsigned Check = Op.getConstantOperandVal(i: 1);
7095
7096 unsigned TDCMask = 0;
7097 if (Check & fcSNan)
7098 TDCMask |= SystemZ::TDCMASK_SNAN_PLUS | SystemZ::TDCMASK_SNAN_MINUS;
7099 if (Check & fcQNan)
7100 TDCMask |= SystemZ::TDCMASK_QNAN_PLUS | SystemZ::TDCMASK_QNAN_MINUS;
7101 if (Check & fcPosInf)
7102 TDCMask |= SystemZ::TDCMASK_INFINITY_PLUS;
7103 if (Check & fcNegInf)
7104 TDCMask |= SystemZ::TDCMASK_INFINITY_MINUS;
7105 if (Check & fcPosNormal)
7106 TDCMask |= SystemZ::TDCMASK_NORMAL_PLUS;
7107 if (Check & fcNegNormal)
7108 TDCMask |= SystemZ::TDCMASK_NORMAL_MINUS;
7109 if (Check & fcPosSubnormal)
7110 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_PLUS;
7111 if (Check & fcNegSubnormal)
7112 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_MINUS;
7113 if (Check & fcPosZero)
7114 TDCMask |= SystemZ::TDCMASK_ZERO_PLUS;
7115 if (Check & fcNegZero)
7116 TDCMask |= SystemZ::TDCMASK_ZERO_MINUS;
7117 SDValue TDCMaskV = DAG.getConstant(Val: TDCMask, DL, VT: MVT::i64);
7118
7119 if (Arg.getSimpleValueType() == MVT::f16)
7120 Arg = DAG.getFPExtendOrRound(Op: Arg, DL: SDLoc(Arg), VT: MVT::f32);
7121 SDValue Intr = DAG.getNode(Opcode: SystemZISD::TDC, DL, VT: ResultVT, N1: Arg, N2: TDCMaskV);
7122 return getCCResult(DAG, CCReg: Intr);
7123}
7124
7125SDValue SystemZTargetLowering::lowerREADCYCLECOUNTER(SDValue Op,
7126 SelectionDAG &DAG) const {
7127 SDLoc DL(Op);
7128 SDValue Chain = Op.getOperand(i: 0);
7129
7130 // STCKF only supports a memory operand, so we have to use a temporary.
7131 SDValue StackPtr = DAG.CreateStackTemporary(VT: MVT::i64);
7132 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
7133 MachinePointerInfo MPI =
7134 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7135
7136 // Use STCFK to store the TOD clock into the temporary.
7137 SDValue StoreOps[] = {Chain, StackPtr};
7138 Chain = DAG.getMemIntrinsicNode(
7139 Opcode: SystemZISD::STCKF, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps, MemVT: MVT::i64,
7140 PtrInfo: MPI, Alignment: MaybeAlign(), Flags: MachineMemOperand::MOStore);
7141
7142 // And read it back from there.
7143 return DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: StackPtr, PtrInfo: MPI);
7144}
7145
7146SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
7147 SelectionDAG &DAG) const {
7148 switch (Op.getOpcode()) {
7149 case ISD::FRAMEADDR:
7150 return lowerFRAMEADDR(Op, DAG);
7151 case ISD::RETURNADDR:
7152 return lowerRETURNADDR(Op, DAG);
7153 case ISD::BR_CC:
7154 return lowerBR_CC(Op, DAG);
7155 case ISD::SELECT_CC:
7156 return lowerSELECT_CC(Op, DAG);
7157 case ISD::SETCC:
7158 return lowerSETCC(Op, DAG);
7159 case ISD::STRICT_FSETCC:
7160 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: false);
7161 case ISD::STRICT_FSETCCS:
7162 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: true);
7163 case ISD::GlobalAddress:
7164 return lowerGlobalAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7165 case ISD::GlobalTLSAddress:
7166 return lowerGlobalTLSAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7167 case ISD::BlockAddress:
7168 return lowerBlockAddress(Node: cast<BlockAddressSDNode>(Val&: Op), DAG);
7169 case ISD::JumpTable:
7170 return lowerJumpTable(JT: cast<JumpTableSDNode>(Val&: Op), DAG);
7171 case ISD::ConstantPool:
7172 return lowerConstantPool(CP: cast<ConstantPoolSDNode>(Val&: Op), DAG);
7173 case ISD::BITCAST:
7174 return lowerBITCAST(Op, DAG);
7175 case ISD::VASTART:
7176 return lowerVASTART(Op, DAG);
7177 case ISD::VACOPY:
7178 return lowerVACOPY(Op, DAG);
7179 case ISD::DYNAMIC_STACKALLOC:
7180 return lowerDYNAMIC_STACKALLOC(Op, DAG);
7181 case ISD::GET_DYNAMIC_AREA_OFFSET:
7182 return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
7183 case ISD::MULHS:
7184 return lowerMULH(Op, DAG, Opcode: SystemZISD::SMUL_LOHI);
7185 case ISD::MULHU:
7186 return lowerMULH(Op, DAG, Opcode: SystemZISD::UMUL_LOHI);
7187 case ISD::SMUL_LOHI:
7188 return lowerSMUL_LOHI(Op, DAG);
7189 case ISD::UMUL_LOHI:
7190 return lowerUMUL_LOHI(Op, DAG);
7191 case ISD::SDIVREM:
7192 return lowerSDIVREM(Op, DAG);
7193 case ISD::UDIVREM:
7194 return lowerUDIVREM(Op, DAG);
7195 case ISD::SADDO:
7196 case ISD::SSUBO:
7197 case ISD::UADDO:
7198 case ISD::USUBO:
7199 return lowerXALUO(Op, DAG);
7200 case ISD::UADDO_CARRY:
7201 case ISD::USUBO_CARRY:
7202 return lowerUADDSUBO_CARRY(Op, DAG);
7203 case ISD::OR:
7204 return lowerOR(Op, DAG);
7205 case ISD::CTPOP:
7206 return lowerCTPOP(Op, DAG);
7207 case ISD::VECREDUCE_ADD:
7208 return lowerVECREDUCE_ADD(Op, DAG);
7209 case ISD::ATOMIC_FENCE:
7210 return lowerATOMIC_FENCE(Op, DAG);
7211 case ISD::ATOMIC_SWAP:
7212 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_SWAPW);
7213 case ISD::ATOMIC_STORE:
7214 return lowerATOMIC_STORE(Op, DAG);
7215 case ISD::ATOMIC_LOAD:
7216 return lowerATOMIC_LOAD(Op, DAG);
7217 case ISD::ATOMIC_LOAD_ADD:
7218 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_ADD);
7219 case ISD::ATOMIC_LOAD_SUB:
7220 return lowerATOMIC_LOAD_SUB(Op, DAG);
7221 case ISD::ATOMIC_LOAD_AND:
7222 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_AND);
7223 case ISD::ATOMIC_LOAD_OR:
7224 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_OR);
7225 case ISD::ATOMIC_LOAD_XOR:
7226 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_XOR);
7227 case ISD::ATOMIC_LOAD_NAND:
7228 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_NAND);
7229 case ISD::ATOMIC_LOAD_MIN:
7230 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MIN);
7231 case ISD::ATOMIC_LOAD_MAX:
7232 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MAX);
7233 case ISD::ATOMIC_LOAD_UMIN:
7234 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMIN);
7235 case ISD::ATOMIC_LOAD_UMAX:
7236 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMAX);
7237 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
7238 return lowerATOMIC_CMP_SWAP(Op, DAG);
7239 case ISD::STACKSAVE:
7240 return lowerSTACKSAVE(Op, DAG);
7241 case ISD::STACKRESTORE:
7242 return lowerSTACKRESTORE(Op, DAG);
7243 case ISD::PREFETCH:
7244 return lowerPREFETCH(Op, DAG);
7245 case ISD::INTRINSIC_W_CHAIN:
7246 return lowerINTRINSIC_W_CHAIN(Op, DAG);
7247 case ISD::INTRINSIC_WO_CHAIN:
7248 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
7249 case ISD::BUILD_VECTOR:
7250 return lowerBUILD_VECTOR(Op, DAG);
7251 case ISD::VECTOR_SHUFFLE:
7252 return lowerVECTOR_SHUFFLE(Op, DAG);
7253 case ISD::SCALAR_TO_VECTOR:
7254 return lowerSCALAR_TO_VECTOR(Op, DAG);
7255 case ISD::INSERT_VECTOR_ELT:
7256 return lowerINSERT_VECTOR_ELT(Op, DAG);
7257 case ISD::EXTRACT_VECTOR_ELT:
7258 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7259 case ISD::SIGN_EXTEND_VECTOR_INREG:
7260 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
7261 case ISD::ZERO_EXTEND_VECTOR_INREG:
7262 return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
7263 case ISD::SHL:
7264 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSHL_BY_SCALAR);
7265 case ISD::SRL:
7266 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRL_BY_SCALAR);
7267 case ISD::SRA:
7268 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRA_BY_SCALAR);
7269 case ISD::ADDRSPACECAST:
7270 return lowerAddrSpaceCast(Op, DAG);
7271 case ISD::ROTL:
7272 return lowerShift(Op, DAG, ByScalar: SystemZISD::VROTL_BY_SCALAR);
7273 case ISD::FSHL:
7274 return lowerFSHL(Op, DAG);
7275 case ISD::FSHR:
7276 return lowerFSHR(Op, DAG);
7277 case ISD::FP_EXTEND:
7278 case ISD::STRICT_FP_EXTEND:
7279 return lowerFP_EXTEND(Op, DAG);
7280 case ISD::FP_TO_UINT:
7281 case ISD::FP_TO_SINT:
7282 case ISD::STRICT_FP_TO_UINT:
7283 case ISD::STRICT_FP_TO_SINT:
7284 return lower_FP_TO_INT(Op, DAG);
7285 case ISD::UINT_TO_FP:
7286 case ISD::SINT_TO_FP:
7287 case ISD::STRICT_UINT_TO_FP:
7288 case ISD::STRICT_SINT_TO_FP:
7289 return lower_INT_TO_FP(Op, DAG);
7290 case ISD::LOAD:
7291 return lowerLoadF16(Op, DAG);
7292 case ISD::STORE:
7293 return lowerStoreF16(Op, DAG);
7294 case ISD::IS_FPCLASS:
7295 return lowerIS_FPCLASS(Op, DAG);
7296 case ISD::GET_ROUNDING:
7297 return lowerGET_ROUNDING(Op, DAG);
7298 case ISD::READCYCLECOUNTER:
7299 return lowerREADCYCLECOUNTER(Op, DAG);
7300 case ISD::EH_SJLJ_SETJMP:
7301 case ISD::EH_SJLJ_LONGJMP:
7302 // These operations are legal on our platform, but we cannot actually
7303 // set the operation action to Legal as common code would treat this
7304 // as equivalent to Expand. Instead, we keep the operation action to
7305 // Custom and just leave them unchanged here.
7306 return Op;
7307
7308 default:
7309 llvm_unreachable("Unexpected node to lower");
7310 }
7311}
7312
7313static SDValue expandBitCastI128ToF128(SelectionDAG &DAG, SDValue Src,
7314 const SDLoc &SL) {
7315 // If i128 is legal, just use a normal bitcast.
7316 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7317 return DAG.getBitcast(VT: MVT::f128, V: Src);
7318
7319 // Otherwise, f128 must live in FP128, so do a partwise move.
7320 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7321 &SystemZ::FP128BitRegClass);
7322
7323 SDValue Hi, Lo;
7324 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Src, DL: SL, LoVT: MVT::i64, HiVT: MVT::i64);
7325
7326 Hi = DAG.getBitcast(VT: MVT::f64, V: Hi);
7327 Lo = DAG.getBitcast(VT: MVT::f64, V: Lo);
7328
7329 SDNode *Pair = DAG.getMachineNode(
7330 Opcode: SystemZ::REG_SEQUENCE, dl: SL, VT: MVT::f128,
7331 Ops: {DAG.getTargetConstant(Val: SystemZ::FP128BitRegClassID, DL: SL, VT: MVT::i32), Lo,
7332 DAG.getTargetConstant(Val: SystemZ::subreg_l64, DL: SL, VT: MVT::i32), Hi,
7333 DAG.getTargetConstant(Val: SystemZ::subreg_h64, DL: SL, VT: MVT::i32)});
7334 return SDValue(Pair, 0);
7335}
7336
7337static SDValue expandBitCastF128ToI128(SelectionDAG &DAG, SDValue Src,
7338 const SDLoc &SL) {
7339 // If i128 is legal, just use a normal bitcast.
7340 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7341 return DAG.getBitcast(VT: MVT::i128, V: Src);
7342
7343 // Otherwise, f128 must live in FP128, so do a partwise move.
7344 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7345 &SystemZ::FP128BitRegClass);
7346
7347 SDValue LoFP =
7348 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64, DL: SL, VT: MVT::f64, Operand: Src);
7349 SDValue HiFP =
7350 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64, DL: SL, VT: MVT::f64, Operand: Src);
7351 SDValue Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: LoFP);
7352 SDValue Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: HiFP);
7353
7354 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i128, N1: Lo, N2: Hi);
7355}
7356
7357// Lower operations with invalid operand or result types.
7358void
7359SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
7360 SmallVectorImpl<SDValue> &Results,
7361 SelectionDAG &DAG) const {
7362 switch (N->getOpcode()) {
7363 case ISD::ATOMIC_LOAD: {
7364 SDLoc DL(N);
7365 SDVTList Tys = DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::Other);
7366 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
7367 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7368 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_LOAD_128,
7369 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7370
7371 SDValue Lowered = lowerGR128ToI128(DAG, In: Res);
7372 if (N->getValueType(ResNo: 0) == MVT::f128)
7373 Lowered = expandBitCastI128ToF128(DAG, Src: Lowered, SL: DL);
7374 Results.push_back(Elt: Lowered);
7375 Results.push_back(Elt: Res.getValue(R: 1));
7376 break;
7377 }
7378 case ISD::ATOMIC_STORE: {
7379 SDLoc DL(N);
7380 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
7381 SDValue Val = N->getOperand(Num: 1);
7382 if (Val.getValueType() == MVT::f128)
7383 Val = expandBitCastF128ToI128(DAG, Src: Val, SL: DL);
7384 Val = lowerI128ToGR128(DAG, In: Val);
7385
7386 SDValue Ops[] = {N->getOperand(Num: 0), Val, N->getOperand(Num: 2)};
7387 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7388 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_STORE_128,
7389 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7390 // We have to enforce sequential consistency by performing a
7391 // serialization operation after the store.
7392 if (cast<AtomicSDNode>(Val: N)->getSuccessOrdering() ==
7393 AtomicOrdering::SequentiallyConsistent)
7394 Res = SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL,
7395 VT: MVT::Other, Op1: Res), 0);
7396 Results.push_back(Elt: Res);
7397 break;
7398 }
7399 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
7400 SDLoc DL(N);
7401 SDVTList Tys = DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::i32, VT3: MVT::Other);
7402 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1),
7403 lowerI128ToGR128(DAG, In: N->getOperand(Num: 2)),
7404 lowerI128ToGR128(DAG, In: N->getOperand(Num: 3)) };
7405 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7406 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP_128,
7407 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7408 SDValue Success = emitSETCC(DAG, DL, CCReg: Res.getValue(R: 1),
7409 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
7410 Success = DAG.getZExtOrTrunc(Op: Success, DL, VT: N->getValueType(ResNo: 1));
7411 Results.push_back(Elt: lowerGR128ToI128(DAG, In: Res));
7412 Results.push_back(Elt: Success);
7413 Results.push_back(Elt: Res.getValue(R: 2));
7414 break;
7415 }
7416 case ISD::BITCAST: {
7417 if (useSoftFloat())
7418 return;
7419 SDLoc DL(N);
7420 SDValue Src = N->getOperand(Num: 0);
7421 EVT SrcVT = Src.getValueType();
7422 EVT ResVT = N->getValueType(ResNo: 0);
7423 if (ResVT == MVT::i128 && SrcVT == MVT::f128)
7424 Results.push_back(Elt: expandBitCastF128ToI128(DAG, Src, SL: DL));
7425 else if (SrcVT == MVT::i16 && ResVT == MVT::f16) {
7426 if (Subtarget.hasVector()) {
7427 SDValue In32 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Src);
7428 Results.push_back(Elt: SDValue(
7429 DAG.getMachineNode(Opcode: SystemZ::LEFR_16, dl: DL, VT: MVT::f16, Op1: In32), 0));
7430 } else {
7431 SDValue In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Src);
7432 Results.push_back(Elt: convertToF16(Op: In64, DAG));
7433 }
7434 } else if (SrcVT == MVT::f16 && ResVT == MVT::i16) {
7435 SDValue ExtractedVal =
7436 Subtarget.hasVector()
7437 ? SDValue(DAG.getMachineNode(Opcode: SystemZ::LFER_16, dl: DL, VT: MVT::i32, Op1: Src),
7438 0)
7439 : convertFromF16(Op: Src, DL, DAG);
7440 Results.push_back(Elt: DAG.getZExtOrTrunc(Op: ExtractedVal, DL, VT: ResVT));
7441 }
7442 break;
7443 }
7444 case ISD::UINT_TO_FP:
7445 case ISD::SINT_TO_FP:
7446 case ISD::STRICT_UINT_TO_FP:
7447 case ISD::STRICT_SINT_TO_FP: {
7448 if (useSoftFloat())
7449 return;
7450 bool IsStrict = N->isStrictFPOpcode();
7451 SDLoc DL(N);
7452 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7453 EVT ResVT = N->getValueType(ResNo: 0);
7454 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7455 if (ResVT == MVT::f16) {
7456 if (!IsStrict) {
7457 SDValue OpF32 = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::f32, Operand: InOp);
7458 Results.push_back(Elt: DAG.getFPExtendOrRound(Op: OpF32, DL, VT: MVT::f16));
7459 } else {
7460 SDValue OpF32 =
7461 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
7462 Ops: {Chain, InOp});
7463 SDValue F16Res;
7464 std::tie(args&: F16Res, args&: Chain) = DAG.getStrictFPExtendOrRound(
7465 Op: OpF32, Chain: OpF32.getValue(R: 1), DL, VT: MVT::f16);
7466 Results.push_back(Elt: F16Res);
7467 Results.push_back(Elt: Chain);
7468 }
7469 }
7470 break;
7471 }
7472 case ISD::FP_TO_UINT:
7473 case ISD::FP_TO_SINT:
7474 case ISD::STRICT_FP_TO_UINT:
7475 case ISD::STRICT_FP_TO_SINT: {
7476 if (useSoftFloat())
7477 return;
7478 bool IsStrict = N->isStrictFPOpcode();
7479 SDLoc DL(N);
7480 EVT ResVT = N->getValueType(ResNo: 0);
7481 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7482 EVT InVT = InOp->getValueType(ResNo: 0);
7483 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7484 if (InVT == MVT::f16) {
7485 if (!IsStrict) {
7486 SDValue InF32 = DAG.getFPExtendOrRound(Op: InOp, DL, VT: MVT::f32);
7487 Results.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: InF32));
7488 } else {
7489 SDValue InF32;
7490 std::tie(args&: InF32, args&: Chain) =
7491 DAG.getStrictFPExtendOrRound(Op: InOp, Chain, DL, VT: MVT::f32);
7492 SDValue OpF32 =
7493 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: ResVT, VT2: MVT::Other),
7494 Ops: {Chain, InF32});
7495 Results.push_back(Elt: OpF32);
7496 Results.push_back(Elt: OpF32.getValue(R: 1));
7497 }
7498 }
7499 break;
7500 }
7501 default:
7502 llvm_unreachable("Unexpected node to lower");
7503 }
7504}
7505
7506void
7507SystemZTargetLowering::ReplaceNodeResults(SDNode *N,
7508 SmallVectorImpl<SDValue> &Results,
7509 SelectionDAG &DAG) const {
7510 return LowerOperationWrapper(N, Results, DAG);
7511}
7512
7513// Return true if VT is a vector whose elements are a whole number of bytes
7514// in width. Also check for presence of vector support.
7515bool SystemZTargetLowering::canTreatAsByteVector(EVT VT) const {
7516 if (!Subtarget.hasVector())
7517 return false;
7518
7519 return VT.isVector() && VT.getScalarSizeInBits() % 8 == 0 && VT.isSimple();
7520}
7521
7522// Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT
7523// producing a result of type ResVT. Op is a possibly bitcast version
7524// of the input vector and Index is the index (based on type VecVT) that
7525// should be extracted. Return the new extraction if a simplification
7526// was possible or if Force is true.
7527SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
7528 EVT VecVT, SDValue Op,
7529 unsigned Index,
7530 DAGCombinerInfo &DCI,
7531 bool Force) const {
7532 SelectionDAG &DAG = DCI.DAG;
7533
7534 // The number of bytes being extracted.
7535 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7536
7537 for (;;) {
7538 unsigned Opcode = Op.getOpcode();
7539 if (Opcode == ISD::BITCAST)
7540 // Look through bitcasts.
7541 Op = Op.getOperand(i: 0);
7542 else if ((Opcode == ISD::VECTOR_SHUFFLE || Opcode == SystemZISD::SPLAT) &&
7543 canTreatAsByteVector(VT: Op.getValueType())) {
7544 // Get a VPERM-like permute mask and see whether the bytes covered
7545 // by the extracted element are a contiguous sequence from one
7546 // source operand.
7547 SmallVector<int, SystemZ::VectorBytes> Bytes;
7548 if (!getVPermMask(ShuffleOp: Op, Bytes))
7549 break;
7550 int First;
7551 if (!getShuffleInput(Bytes, Start: Index * BytesPerElement,
7552 BytesPerElement, Base&: First))
7553 break;
7554 if (First < 0)
7555 return DAG.getUNDEF(VT: ResVT);
7556 // Make sure the contiguous sequence starts at a multiple of the
7557 // original element size.
7558 unsigned Byte = unsigned(First) % Bytes.size();
7559 if (Byte % BytesPerElement != 0)
7560 break;
7561 // We can get the extracted value directly from an input.
7562 Index = Byte / BytesPerElement;
7563 Op = Op.getOperand(i: unsigned(First) / Bytes.size());
7564 Force = true;
7565 } else if (Opcode == ISD::BUILD_VECTOR &&
7566 canTreatAsByteVector(VT: Op.getValueType())) {
7567 // We can only optimize this case if the BUILD_VECTOR elements are
7568 // at least as wide as the extracted value.
7569 EVT OpVT = Op.getValueType();
7570 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7571 if (OpBytesPerElement < BytesPerElement)
7572 break;
7573 // Make sure that the least-significant bit of the extracted value
7574 // is the least significant bit of an input.
7575 unsigned End = (Index + 1) * BytesPerElement;
7576 if (End % OpBytesPerElement != 0)
7577 break;
7578 // We're extracting the low part of one operand of the BUILD_VECTOR.
7579 Op = Op.getOperand(i: End / OpBytesPerElement - 1);
7580 if (!Op.getValueType().isInteger()) {
7581 EVT VT = MVT::getIntegerVT(BitWidth: Op.getValueSizeInBits());
7582 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
7583 DCI.AddToWorklist(N: Op.getNode());
7584 }
7585 EVT VT = MVT::getIntegerVT(BitWidth: ResVT.getSizeInBits());
7586 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
7587 if (VT != ResVT) {
7588 DCI.AddToWorklist(N: Op.getNode());
7589 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ResVT, Operand: Op);
7590 }
7591 return Op;
7592 } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
7593 Opcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
7594 Opcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
7595 canTreatAsByteVector(VT: Op.getValueType()) &&
7596 canTreatAsByteVector(VT: Op.getOperand(i: 0).getValueType())) {
7597 // Make sure that only the unextended bits are significant.
7598 EVT ExtVT = Op.getValueType();
7599 EVT OpVT = Op.getOperand(i: 0).getValueType();
7600 unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize();
7601 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7602 unsigned Byte = Index * BytesPerElement;
7603 unsigned SubByte = Byte % ExtBytesPerElement;
7604 unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
7605 if (SubByte < MinSubByte ||
7606 SubByte + BytesPerElement > ExtBytesPerElement)
7607 break;
7608 // Get the byte offset of the unextended element
7609 Byte = Byte / ExtBytesPerElement * OpBytesPerElement;
7610 // ...then add the byte offset relative to that element.
7611 Byte += SubByte - MinSubByte;
7612 if (Byte % BytesPerElement != 0)
7613 break;
7614 Op = Op.getOperand(i: 0);
7615 Index = Byte / BytesPerElement;
7616 Force = true;
7617 } else
7618 break;
7619 }
7620 if (Force) {
7621 if (Op.getValueType() != VecVT) {
7622 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VecVT, Operand: Op);
7623 DCI.AddToWorklist(N: Op.getNode());
7624 }
7625 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ResVT, N1: Op,
7626 N2: DAG.getConstant(Val: Index, DL, VT: MVT::i32));
7627 }
7628 return SDValue();
7629}
7630
7631// Optimize vector operations in scalar value Op on the basis that Op
7632// is truncated to TruncVT.
7633SDValue SystemZTargetLowering::combineTruncateExtract(
7634 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
7635 // If we have (trunc (extract_vector_elt X, Y)), try to turn it into
7636 // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements
7637 // of type TruncVT.
7638 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7639 TruncVT.getSizeInBits() % 8 == 0) {
7640 SDValue Vec = Op.getOperand(i: 0);
7641 EVT VecVT = Vec.getValueType();
7642 if (canTreatAsByteVector(VT: VecVT)) {
7643 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
7644 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7645 unsigned TruncBytes = TruncVT.getStoreSize();
7646 if (BytesPerElement % TruncBytes == 0) {
7647 // Calculate the value of Y' in the above description. We are
7648 // splitting the original elements into Scale equal-sized pieces
7649 // and for truncation purposes want the last (least-significant)
7650 // of these pieces for IndexN. This is easiest to do by calculating
7651 // the start index of the following element and then subtracting 1.
7652 unsigned Scale = BytesPerElement / TruncBytes;
7653 unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
7654
7655 // Defer the creation of the bitcast from X to combineExtract,
7656 // which might be able to optimize the extraction.
7657 VecVT = EVT::getVectorVT(Context&: *DCI.DAG.getContext(),
7658 VT: MVT::getIntegerVT(BitWidth: TruncBytes * 8),
7659 NumElements: VecVT.getStoreSize() / TruncBytes);
7660 EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
7661 return combineExtract(DL, ResVT, VecVT, Op: Vec, Index: NewIndex, DCI, Force: true);
7662 }
7663 }
7664 }
7665 }
7666 return SDValue();
7667}
7668
7669SDValue SystemZTargetLowering::combineZERO_EXTEND(
7670 SDNode *N, DAGCombinerInfo &DCI) const {
7671 // Convert (zext (select_ccmask C1, C2)) into (select_ccmask C1', C2')
7672 SelectionDAG &DAG = DCI.DAG;
7673 SDValue N0 = N->getOperand(Num: 0);
7674 EVT VT = N->getValueType(ResNo: 0);
7675 if (N0.getOpcode() == SystemZISD::SELECT_CCMASK) {
7676 auto *TrueOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0));
7677 auto *FalseOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7678 if (TrueOp && FalseOp) {
7679 SDLoc DL(N0);
7680 SDValue Ops[] = { DAG.getConstant(Val: TrueOp->getZExtValue(), DL, VT),
7681 DAG.getConstant(Val: FalseOp->getZExtValue(), DL, VT),
7682 N0.getOperand(i: 2), N0.getOperand(i: 3), N0.getOperand(i: 4) };
7683 SDValue NewSelect = DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT, Ops);
7684 // If N0 has multiple uses, change other uses as well.
7685 if (!N0.hasOneUse()) {
7686 SDValue TruncSelect =
7687 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N0.getValueType(), Operand: NewSelect);
7688 DCI.CombineTo(N: N0.getNode(), Res: TruncSelect);
7689 }
7690 return NewSelect;
7691 }
7692 }
7693 // Convert (zext (xor (trunc X), C)) into (xor (trunc X), C') if the size
7694 // of the result is smaller than the size of X and all the truncated bits
7695 // of X are already zero.
7696 if (N0.getOpcode() == ISD::XOR &&
7697 N0.hasOneUse() && N0.getOperand(i: 0).hasOneUse() &&
7698 N0.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
7699 N0.getOperand(i: 1).getOpcode() == ISD::Constant) {
7700 SDValue X = N0.getOperand(i: 0).getOperand(i: 0);
7701 if (VT.isScalarInteger() && VT.getSizeInBits() < X.getValueSizeInBits()) {
7702 KnownBits Known = DAG.computeKnownBits(Op: X);
7703 APInt TruncatedBits = APInt::getBitsSet(numBits: X.getValueSizeInBits(),
7704 loBit: N0.getValueSizeInBits(),
7705 hiBit: VT.getSizeInBits());
7706 if (TruncatedBits.isSubsetOf(RHS: Known.Zero)) {
7707 X = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(X), VT, Operand: X);
7708 APInt Mask = N0.getConstantOperandAPInt(i: 1).zext(width: VT.getSizeInBits());
7709 return DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N0), VT,
7710 N1: X, N2: DAG.getConstant(Val: Mask, DL: SDLoc(N0), VT));
7711 }
7712 }
7713 }
7714 // Recognize patterns for VECTOR SUBTRACT COMPUTE BORROW INDICATION
7715 // and VECTOR ADD COMPUTE CARRY for i128:
7716 // (zext (setcc_uge X Y)) --> (VSCBI X Y)
7717 // (zext (setcc_ule Y X)) --> (VSCBI X Y)
7718 // (zext (setcc_ult (add X Y) X/Y) -> (VACC X Y)
7719 // (zext (setcc_ugt X/Y (add X Y)) -> (VACC X Y)
7720 // For vector types, these patterns are recognized in the .td file.
7721 if (N0.getOpcode() == ISD::SETCC && isTypeLegal(VT) && VT == MVT::i128 &&
7722 N0.getOperand(i: 0).getValueType() == VT) {
7723 SDValue Op0 = N0.getOperand(i: 0);
7724 SDValue Op1 = N0.getOperand(i: 1);
7725 const ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
7726 switch (CC) {
7727 case ISD::SETULE:
7728 std::swap(a&: Op0, b&: Op1);
7729 [[fallthrough]];
7730 case ISD::SETUGE:
7731 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(N0), VT, N1: Op0, N2: Op1);
7732 case ISD::SETUGT:
7733 std::swap(a&: Op0, b&: Op1);
7734 [[fallthrough]];
7735 case ISD::SETULT:
7736 if (Op0->hasOneUse() && Op0->getOpcode() == ISD::ADD &&
7737 (Op0->getOperand(Num: 0) == Op1 || Op0->getOperand(Num: 1) == Op1))
7738 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(N0), VT, N1: Op0->getOperand(Num: 0),
7739 N2: Op0->getOperand(Num: 1));
7740 break;
7741 default:
7742 break;
7743 }
7744 }
7745
7746 return SDValue();
7747}
7748
7749SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
7750 SDNode *N, DAGCombinerInfo &DCI) const {
7751 // Convert (sext_in_reg (setcc LHS, RHS, COND), i1)
7752 // and (sext_in_reg (any_extend (setcc LHS, RHS, COND)), i1)
7753 // into (select_cc LHS, RHS, -1, 0, COND)
7754 SelectionDAG &DAG = DCI.DAG;
7755 SDValue N0 = N->getOperand(Num: 0);
7756 EVT VT = N->getValueType(ResNo: 0);
7757 EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
7758 if (N0.hasOneUse() && N0.getOpcode() == ISD::ANY_EXTEND)
7759 N0 = N0.getOperand(i: 0);
7760 if (EVT == MVT::i1 && N0.hasOneUse() && N0.getOpcode() == ISD::SETCC) {
7761 SDLoc DL(N0);
7762 SDValue Ops[] = { N0.getOperand(i: 0), N0.getOperand(i: 1),
7763 DAG.getAllOnesConstant(DL, VT),
7764 DAG.getConstant(Val: 0, DL, VT), N0.getOperand(i: 2) };
7765 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT, Ops);
7766 }
7767 return SDValue();
7768}
7769
7770SDValue SystemZTargetLowering::combineSIGN_EXTEND(
7771 SDNode *N, DAGCombinerInfo &DCI) const {
7772 // Convert (sext (ashr (shl X, C1), C2)) to
7773 // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as
7774 // cheap as narrower ones.
7775 SelectionDAG &DAG = DCI.DAG;
7776 SDValue N0 = N->getOperand(Num: 0);
7777 EVT VT = N->getValueType(ResNo: 0);
7778 if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) {
7779 auto *SraAmt = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7780 SDValue Inner = N0.getOperand(i: 0);
7781 if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) {
7782 if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Val: Inner.getOperand(i: 1))) {
7783 unsigned Extra = (VT.getSizeInBits() - N0.getValueSizeInBits());
7784 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
7785 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
7786 EVT ShiftVT = N0.getOperand(i: 1).getValueType();
7787 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(Inner), VT,
7788 Operand: Inner.getOperand(i: 0));
7789 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(Inner), VT, N1: Ext,
7790 N2: DAG.getConstant(Val: NewShlAmt, DL: SDLoc(Inner),
7791 VT: ShiftVT));
7792 return DAG.getNode(Opcode: ISD::SRA, DL: SDLoc(N0), VT, N1: Shl,
7793 N2: DAG.getConstant(Val: NewSraAmt, DL: SDLoc(N0), VT: ShiftVT));
7794 }
7795 }
7796 }
7797
7798 return SDValue();
7799}
7800
7801SDValue SystemZTargetLowering::combineMERGE(
7802 SDNode *N, DAGCombinerInfo &DCI) const {
7803 SelectionDAG &DAG = DCI.DAG;
7804 unsigned Opcode = N->getOpcode();
7805 SDValue Op0 = N->getOperand(Num: 0);
7806 SDValue Op1 = N->getOperand(Num: 1);
7807 if (Op0.getOpcode() == ISD::BITCAST)
7808 Op0 = Op0.getOperand(i: 0);
7809 if (ISD::isBuildVectorAllZeros(N: Op0.getNode())) {
7810 // (z_merge_* 0, 0) -> 0. This is mostly useful for using VLLEZF
7811 // for v4f32.
7812 if (Op1 == N->getOperand(Num: 0))
7813 return Op1;
7814 // (z_merge_? 0, X) -> (z_unpackl_? 0, X).
7815 EVT VT = Op1.getValueType();
7816 unsigned ElemBytes = VT.getVectorElementType().getStoreSize();
7817 if (ElemBytes <= 4) {
7818 Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
7819 SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
7820 EVT InVT = VT.changeVectorElementTypeToInteger();
7821 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ElemBytes * 16),
7822 NumElements: SystemZ::VectorBytes / ElemBytes / 2);
7823 if (VT != InVT) {
7824 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: InVT, Operand: Op1);
7825 DCI.AddToWorklist(N: Op1.getNode());
7826 }
7827 SDValue Op = DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: Op1);
7828 DCI.AddToWorklist(N: Op.getNode());
7829 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT, Operand: Op);
7830 }
7831 }
7832 return SDValue();
7833}
7834
7835static bool isI128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
7836 SDNode *&HiPart) {
7837 LoPart = HiPart = nullptr;
7838
7839 // Scan through all users.
7840 for (SDUse &Use : LD->uses()) {
7841 // Skip the uses of the chain.
7842 if (Use.getResNo() != 0)
7843 continue;
7844
7845 // Verify every user is a TRUNCATE to i64 of the low or high half.
7846 SDNode *User = Use.getUser();
7847 bool IsLoPart = true;
7848 if (User->getOpcode() == ISD::SRL &&
7849 User->getOperand(Num: 1).getOpcode() == ISD::Constant &&
7850 User->getConstantOperandVal(Num: 1) == 64 && User->hasOneUse()) {
7851 User = *User->user_begin();
7852 IsLoPart = false;
7853 }
7854 if (User->getOpcode() != ISD::TRUNCATE || User->getValueType(ResNo: 0) != MVT::i64)
7855 return false;
7856
7857 if (IsLoPart) {
7858 if (LoPart)
7859 return false;
7860 LoPart = User;
7861 } else {
7862 if (HiPart)
7863 return false;
7864 HiPart = User;
7865 }
7866 }
7867 return true;
7868}
7869
7870static bool isF128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
7871 SDNode *&HiPart) {
7872 LoPart = HiPart = nullptr;
7873
7874 // Scan through all users.
7875 for (SDUse &Use : LD->uses()) {
7876 // Skip the uses of the chain.
7877 if (Use.getResNo() != 0)
7878 continue;
7879
7880 // Verify every user is an EXTRACT_SUBREG of the low or high half.
7881 SDNode *User = Use.getUser();
7882 if (!User->hasOneUse() || !User->isMachineOpcode() ||
7883 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
7884 return false;
7885
7886 switch (User->getConstantOperandVal(Num: 1)) {
7887 case SystemZ::subreg_l64:
7888 if (LoPart)
7889 return false;
7890 LoPart = User;
7891 break;
7892 case SystemZ::subreg_h64:
7893 if (HiPart)
7894 return false;
7895 HiPart = User;
7896 break;
7897 default:
7898 return false;
7899 }
7900 }
7901 return true;
7902}
7903
7904SDValue SystemZTargetLowering::combineLOAD(
7905 SDNode *N, DAGCombinerInfo &DCI) const {
7906 SelectionDAG &DAG = DCI.DAG;
7907 EVT LdVT = N->getValueType(ResNo: 0);
7908 if (auto *LN = dyn_cast<LoadSDNode>(Val: N)) {
7909 if (LN->getAddressSpace() == SYSTEMZAS::PTR32) {
7910 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7911 MVT LoadNodeVT = LN->getBasePtr().getSimpleValueType();
7912 if (PtrVT != LoadNodeVT) {
7913 SDLoc DL(LN);
7914 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(
7915 dl: DL, VT: PtrVT, Ptr: LN->getBasePtr(), SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
7916 return DAG.getExtLoad(ExtType: LN->getExtensionType(), dl: DL, VT: LN->getValueType(ResNo: 0),
7917 Chain: LN->getChain(), Ptr: AddrSpaceCast, MemVT: LN->getMemoryVT(),
7918 MMO: LN->getMemOperand());
7919 }
7920 }
7921 }
7922 SDLoc DL(N);
7923
7924 // Replace a 128-bit load that is used solely to move its value into GPRs
7925 // by separate loads of both halves.
7926 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
7927 if (LD->isSimple() && ISD::isNormalLoad(N: LD)) {
7928 SDNode *LoPart, *HiPart;
7929 if ((LdVT == MVT::i128 && isI128MovedToParts(LD, LoPart, HiPart)) ||
7930 (LdVT == MVT::f128 && isF128MovedToParts(LD, LoPart, HiPart))) {
7931 // Rewrite each extraction as an independent load.
7932 SmallVector<SDValue, 2> ArgChains;
7933 if (HiPart) {
7934 SDValue EltLoad = DAG.getLoad(
7935 VT: HiPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(), Ptr: LD->getBasePtr(),
7936 PtrInfo: LD->getPointerInfo(), Alignment: LD->getBaseAlign(),
7937 MMOFlags: LD->getMemOperand()->getFlags(), AAInfo: LD->getAAInfo());
7938
7939 DCI.CombineTo(N: HiPart, Res: EltLoad, AddTo: true);
7940 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
7941 }
7942 if (LoPart) {
7943 SDValue EltLoad = DAG.getLoad(
7944 VT: LoPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(),
7945 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: LD->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
7946 PtrInfo: LD->getPointerInfo().getWithOffset(O: 8), Alignment: LD->getBaseAlign(),
7947 MMOFlags: LD->getMemOperand()->getFlags(), AAInfo: LD->getAAInfo());
7948
7949 DCI.CombineTo(N: LoPart, Res: EltLoad, AddTo: true);
7950 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
7951 }
7952
7953 // Collect all chains via TokenFactor.
7954 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ArgChains);
7955 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: Chain);
7956 DCI.AddToWorklist(N: Chain.getNode());
7957 return SDValue(N, 0);
7958 }
7959 }
7960
7961 if (LdVT.isVector() || LdVT.isInteger())
7962 return SDValue();
7963 // Transform a scalar load that is REPLICATEd as well as having other
7964 // use(s) to the form where the other use(s) use the first element of the
7965 // REPLICATE instead of the load. Otherwise instruction selection will not
7966 // produce a VLREP. Avoid extracting to a GPR, so only do this for floating
7967 // point loads.
7968
7969 SDValue Replicate;
7970 SmallVector<SDNode*, 8> OtherUses;
7971 for (SDUse &Use : N->uses()) {
7972 if (Use.getUser()->getOpcode() == SystemZISD::REPLICATE) {
7973 if (Replicate)
7974 return SDValue(); // Should never happen
7975 Replicate = SDValue(Use.getUser(), 0);
7976 } else if (Use.getResNo() == 0)
7977 OtherUses.push_back(Elt: Use.getUser());
7978 }
7979 if (!Replicate || OtherUses.empty())
7980 return SDValue();
7981
7982 SDValue Extract0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: LdVT,
7983 N1: Replicate, N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7984 // Update uses of the loaded Value while preserving old chains.
7985 for (SDNode *U : OtherUses) {
7986 SmallVector<SDValue, 8> Ops;
7987 for (SDValue Op : U->ops())
7988 Ops.push_back(Elt: (Op.getNode() == N && Op.getResNo() == 0) ? Extract0 : Op);
7989 DAG.UpdateNodeOperands(N: U, Ops);
7990 }
7991 return SDValue(N, 0);
7992}
7993
7994bool SystemZTargetLowering::canLoadStoreByteSwapped(EVT VT) const {
7995 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64)
7996 return true;
7997 if (Subtarget.hasVectorEnhancements2())
7998 if (VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v2i64 || VT == MVT::i128)
7999 return true;
8000 return false;
8001}
8002
8003static bool isVectorElementSwap(ArrayRef<int> M, EVT VT) {
8004 if (!VT.isVector() || !VT.isSimple() ||
8005 VT.getSizeInBits() != 128 ||
8006 VT.getScalarSizeInBits() % 8 != 0)
8007 return false;
8008
8009 unsigned NumElts = VT.getVectorNumElements();
8010 for (unsigned i = 0; i < NumElts; ++i) {
8011 if (M[i] < 0) continue; // ignore UNDEF indices
8012 if ((unsigned) M[i] != NumElts - 1 - i)
8013 return false;
8014 }
8015
8016 return true;
8017}
8018
8019static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG) {
8020 for (auto *U : StoredVal->users()) {
8021 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: U)) {
8022 EVT CurrMemVT = ST->getMemoryVT().getScalarType();
8023 if (CurrMemVT.isRound() && CurrMemVT.getStoreSize() <= 16)
8024 continue;
8025 } else if (isa<BuildVectorSDNode>(Val: U)) {
8026 SDValue BuildVector = SDValue(U, 0);
8027 if (DAG.isSplatValue(V: BuildVector, AllowUndefs: true/*AllowUndefs*/) &&
8028 isOnlyUsedByStores(StoredVal: BuildVector, DAG))
8029 continue;
8030 }
8031 return false;
8032 }
8033 return true;
8034}
8035
8036static bool isI128MovedFromParts(SDValue Val, SDValue &LoPart,
8037 SDValue &HiPart) {
8038 if (Val.getOpcode() != ISD::OR || !Val.getNode()->hasOneUse())
8039 return false;
8040
8041 SDValue Op0 = Val.getOperand(i: 0);
8042 SDValue Op1 = Val.getOperand(i: 1);
8043
8044 if (Op0.getOpcode() == ISD::SHL)
8045 std::swap(a&: Op0, b&: Op1);
8046 if (Op1.getOpcode() != ISD::SHL || !Op1.getNode()->hasOneUse() ||
8047 Op1.getOperand(i: 1).getOpcode() != ISD::Constant ||
8048 Op1.getConstantOperandVal(i: 1) != 64)
8049 return false;
8050 Op1 = Op1.getOperand(i: 0);
8051
8052 if (Op0.getOpcode() != ISD::ZERO_EXTEND || !Op0.getNode()->hasOneUse() ||
8053 Op0.getOperand(i: 0).getValueType() != MVT::i64)
8054 return false;
8055 if (Op1.getOpcode() != ISD::ANY_EXTEND || !Op1.getNode()->hasOneUse() ||
8056 Op1.getOperand(i: 0).getValueType() != MVT::i64)
8057 return false;
8058
8059 LoPart = Op0.getOperand(i: 0);
8060 HiPart = Op1.getOperand(i: 0);
8061 return true;
8062}
8063
8064static bool isF128MovedFromParts(SDValue Val, SDValue &LoPart,
8065 SDValue &HiPart) {
8066 if (!Val.getNode()->hasOneUse() || !Val.isMachineOpcode() ||
8067 Val.getMachineOpcode() != TargetOpcode::REG_SEQUENCE)
8068 return false;
8069
8070 if (Val->getNumOperands() != 5 ||
8071 Val->getOperand(Num: 0)->getAsZExtVal() != SystemZ::FP128BitRegClassID ||
8072 Val->getOperand(Num: 2)->getAsZExtVal() != SystemZ::subreg_l64 ||
8073 Val->getOperand(Num: 4)->getAsZExtVal() != SystemZ::subreg_h64)
8074 return false;
8075
8076 LoPart = Val->getOperand(Num: 1);
8077 HiPart = Val->getOperand(Num: 3);
8078 return true;
8079}
8080
8081SDValue SystemZTargetLowering::combineSTORE(
8082 SDNode *N, DAGCombinerInfo &DCI) const {
8083 SelectionDAG &DAG = DCI.DAG;
8084 auto *SN = cast<StoreSDNode>(Val: N);
8085 auto &Op1 = N->getOperand(Num: 1);
8086 EVT MemVT = SN->getMemoryVT();
8087
8088 if (SN->getAddressSpace() == SYSTEMZAS::PTR32) {
8089 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8090 MVT StoreNodeVT = SN->getBasePtr().getSimpleValueType();
8091 if (PtrVT != StoreNodeVT) {
8092 SDLoc DL(SN);
8093 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(dl: DL, VT: PtrVT, Ptr: SN->getBasePtr(),
8094 SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
8095 return DAG.getStore(Chain: SN->getChain(), dl: DL, Val: SN->getValue(), Ptr: AddrSpaceCast,
8096 PtrInfo: SN->getPointerInfo(), Alignment: SN->getBaseAlign(),
8097 MMOFlags: SN->getMemOperand()->getFlags(), AAInfo: SN->getAAInfo());
8098 }
8099 }
8100
8101 // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better
8102 // for the extraction to be done on a vMiN value, so that we can use VSTE.
8103 // If X has wider elements then convert it to:
8104 // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z).
8105 if (MemVT.isInteger() && SN->isTruncatingStore()) {
8106 if (SDValue Value =
8107 combineTruncateExtract(DL: SDLoc(N), TruncVT: MemVT, Op: SN->getValue(), DCI)) {
8108 DCI.AddToWorklist(N: Value.getNode());
8109
8110 // Rewrite the store with the new form of stored value.
8111 return DAG.getTruncStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: Value,
8112 Ptr: SN->getBasePtr(), SVT: SN->getMemoryVT(),
8113 MMO: SN->getMemOperand());
8114 }
8115 }
8116 // Combine STORE (BSWAP) into STRVH/STRV/STRVG/VSTBR
8117 if (!SN->isTruncatingStore() &&
8118 Op1.getOpcode() == ISD::BSWAP &&
8119 Op1.getNode()->hasOneUse() &&
8120 canLoadStoreByteSwapped(VT: Op1.getValueType())) {
8121
8122 SDValue BSwapOp = Op1.getOperand(i: 0);
8123
8124 if (BSwapOp.getValueType() == MVT::i16)
8125 BSwapOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: MVT::i32, Operand: BSwapOp);
8126
8127 SDValue Ops[] = {
8128 N->getOperand(Num: 0), BSwapOp, N->getOperand(Num: 2)
8129 };
8130
8131 return
8132 DAG.getMemIntrinsicNode(Opcode: SystemZISD::STRV, dl: SDLoc(N), VTList: DAG.getVTList(VT: MVT::Other),
8133 Ops, MemVT, MMO: SN->getMemOperand());
8134 }
8135 // Combine STORE (element-swap) into VSTER
8136 if (!SN->isTruncatingStore() &&
8137 Op1.getOpcode() == ISD::VECTOR_SHUFFLE &&
8138 Op1.getNode()->hasOneUse() &&
8139 Subtarget.hasVectorEnhancements2()) {
8140 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op1.getNode());
8141 ArrayRef<int> ShuffleMask = SVN->getMask();
8142 if (isVectorElementSwap(M: ShuffleMask, VT: Op1.getValueType())) {
8143 SDValue Ops[] = {
8144 N->getOperand(Num: 0), Op1.getOperand(i: 0), N->getOperand(Num: 2)
8145 };
8146
8147 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::VSTER, dl: SDLoc(N),
8148 VTList: DAG.getVTList(VT: MVT::Other),
8149 Ops, MemVT, MMO: SN->getMemOperand());
8150 }
8151 }
8152
8153 // Combine STORE (READCYCLECOUNTER) into STCKF.
8154 if (!SN->isTruncatingStore() &&
8155 Op1.getOpcode() == ISD::READCYCLECOUNTER &&
8156 Op1.hasOneUse() &&
8157 N->getOperand(Num: 0).reachesChainWithoutSideEffects(Dest: SDValue(Op1.getNode(), 1))) {
8158 SDValue Ops[] = { Op1.getOperand(i: 0), N->getOperand(Num: 2) };
8159 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::STCKF, dl: SDLoc(N),
8160 VTList: DAG.getVTList(VT: MVT::Other),
8161 Ops, MemVT, MMO: SN->getMemOperand());
8162 }
8163
8164 // Transform a store of a 128-bit value moved from parts into two stores.
8165 if (SN->isSimple() && ISD::isNormalStore(N: SN)) {
8166 SDValue LoPart, HiPart;
8167 if ((MemVT == MVT::i128 && isI128MovedFromParts(Val: Op1, LoPart, HiPart)) ||
8168 (MemVT == MVT::f128 && isF128MovedFromParts(Val: Op1, LoPart, HiPart))) {
8169 SDLoc DL(SN);
8170 SDValue Chain0 = DAG.getStore(
8171 Chain: SN->getChain(), dl: DL, Val: HiPart, Ptr: SN->getBasePtr(), PtrInfo: SN->getPointerInfo(),
8172 Alignment: SN->getBaseAlign(), MMOFlags: SN->getMemOperand()->getFlags(), AAInfo: SN->getAAInfo());
8173 SDValue Chain1 = DAG.getStore(
8174 Chain: SN->getChain(), dl: DL, Val: LoPart,
8175 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: SN->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
8176 PtrInfo: SN->getPointerInfo().getWithOffset(O: 8), Alignment: SN->getBaseAlign(),
8177 MMOFlags: SN->getMemOperand()->getFlags(), AAInfo: SN->getAAInfo());
8178
8179 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Chain0, N2: Chain1);
8180 }
8181 }
8182
8183 // Replicate a reg or immediate with VREP instead of scalar multiply or
8184 // immediate load. It seems best to do this during the first DAGCombine as
8185 // it is straight-forward to handle the zero-extend node in the initial
8186 // DAG, and also not worry about the keeping the new MemVT legal (e.g. when
8187 // extracting an i16 element from a v16i8 vector).
8188 if (Subtarget.hasVector() && DCI.Level == BeforeLegalizeTypes &&
8189 isOnlyUsedByStores(StoredVal: Op1, DAG)) {
8190 SDValue Word = SDValue();
8191 EVT WordVT;
8192
8193 // Find a replicated immediate and return it if found in Word and its
8194 // type in WordVT.
8195 auto FindReplicatedImm = [&](ConstantSDNode *C, unsigned TotBytes) {
8196 // Some constants are better handled with a scalar store.
8197 if (C->getAPIntValue().getBitWidth() > 64 || C->isAllOnes() ||
8198 isInt<16>(x: C->getSExtValue()) || MemVT.getStoreSize() <= 2)
8199 return;
8200
8201 APInt Val = C->getAPIntValue();
8202 // Truncate Val in case of a truncating store.
8203 if (!llvm::isUIntN(N: TotBytes * 8, x: Val.getZExtValue())) {
8204 assert(SN->isTruncatingStore() &&
8205 "Non-truncating store and immediate value does not fit?");
8206 Val = Val.trunc(width: TotBytes * 8);
8207 }
8208
8209 SystemZVectorConstantInfo VCI(APInt(TotBytes * 8, Val.getZExtValue()));
8210 if (VCI.isVectorConstantLegal(Subtarget) &&
8211 VCI.Opcode == SystemZISD::REPLICATE) {
8212 Word = DAG.getConstant(Val: VCI.OpVals[0], DL: SDLoc(SN), VT: MVT::i32);
8213 WordVT = VCI.VecVT.getScalarType();
8214 }
8215 };
8216
8217 // Find a replicated register and return it if found in Word and its type
8218 // in WordVT.
8219 auto FindReplicatedReg = [&](SDValue MulOp) {
8220 EVT MulVT = MulOp.getValueType();
8221 if (MulOp->getOpcode() == ISD::MUL &&
8222 (MulVT == MVT::i16 || MulVT == MVT::i32 || MulVT == MVT::i64)) {
8223 // Find a zero extended value and its type.
8224 SDValue LHS = MulOp->getOperand(Num: 0);
8225 if (LHS->getOpcode() == ISD::ZERO_EXTEND)
8226 WordVT = LHS->getOperand(Num: 0).getValueType();
8227 else if (LHS->getOpcode() == ISD::AssertZext)
8228 WordVT = cast<VTSDNode>(Val: LHS->getOperand(Num: 1))->getVT();
8229 else
8230 return;
8231 // Find a replicating constant, e.g. 0x00010001.
8232 if (auto *C = dyn_cast<ConstantSDNode>(Val: MulOp->getOperand(Num: 1))) {
8233 SystemZVectorConstantInfo VCI(
8234 APInt(MulVT.getSizeInBits(), C->getZExtValue()));
8235 if (VCI.isVectorConstantLegal(Subtarget) &&
8236 VCI.Opcode == SystemZISD::REPLICATE && VCI.OpVals[0] == 1 &&
8237 WordVT == VCI.VecVT.getScalarType())
8238 Word = DAG.getZExtOrTrunc(Op: LHS->getOperand(Num: 0), DL: SDLoc(SN), VT: WordVT);
8239 }
8240 }
8241 };
8242
8243 if (isa<BuildVectorSDNode>(Val: Op1) &&
8244 DAG.isSplatValue(V: Op1, AllowUndefs: true/*AllowUndefs*/)) {
8245 SDValue SplatVal = Op1->getOperand(Num: 0);
8246 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatVal))
8247 FindReplicatedImm(C, SplatVal.getValueType().getStoreSize());
8248 else
8249 FindReplicatedReg(SplatVal);
8250 } else {
8251 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op1))
8252 FindReplicatedImm(C, MemVT.getStoreSize());
8253 else
8254 FindReplicatedReg(Op1);
8255 }
8256
8257 if (Word != SDValue()) {
8258 assert(MemVT.getSizeInBits() % WordVT.getSizeInBits() == 0 &&
8259 "Bad type handling");
8260 unsigned NumElts = MemVT.getSizeInBits() / WordVT.getSizeInBits();
8261 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WordVT, NumElements: NumElts);
8262 SDValue SplatVal = DAG.getSplatVector(VT: SplatVT, DL: SDLoc(SN), Op: Word);
8263 return DAG.getStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: SplatVal,
8264 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
8265 }
8266 }
8267
8268 return SDValue();
8269}
8270
8271SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
8272 SDNode *N, DAGCombinerInfo &DCI) const {
8273 SelectionDAG &DAG = DCI.DAG;
8274 // Combine element-swap (LOAD) into VLER
8275 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8276 N->getOperand(Num: 0).hasOneUse() &&
8277 Subtarget.hasVectorEnhancements2()) {
8278 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
8279 ArrayRef<int> ShuffleMask = SVN->getMask();
8280 if (isVectorElementSwap(M: ShuffleMask, VT: N->getValueType(ResNo: 0))) {
8281 SDValue Load = N->getOperand(Num: 0);
8282 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8283
8284 // Create the element-swapping load.
8285 SDValue Ops[] = {
8286 LD->getChain(), // Chain
8287 LD->getBasePtr() // Ptr
8288 };
8289 SDValue ESLoad =
8290 DAG.getMemIntrinsicNode(Opcode: SystemZISD::VLER, dl: SDLoc(N),
8291 VTList: DAG.getVTList(VT1: LD->getValueType(ResNo: 0), VT2: MVT::Other),
8292 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8293
8294 // First, combine the VECTOR_SHUFFLE away. This makes the value produced
8295 // by the load dead.
8296 DCI.CombineTo(N, Res: ESLoad);
8297
8298 // Next, combine the load away, we give it a bogus result value but a real
8299 // chain result. The result value is dead because the shuffle is dead.
8300 DCI.CombineTo(N: Load.getNode(), Res0: ESLoad, Res1: ESLoad.getValue(R: 1));
8301
8302 // Return N so it doesn't get rechecked!
8303 return SDValue(N, 0);
8304 }
8305 }
8306
8307 return SDValue();
8308}
8309
8310SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
8311 SDNode *N, DAGCombinerInfo &DCI) const {
8312 SelectionDAG &DAG = DCI.DAG;
8313
8314 if (!Subtarget.hasVector())
8315 return SDValue();
8316
8317 // Look through bitcasts that retain the number of vector elements.
8318 SDValue Op = N->getOperand(Num: 0);
8319 if (Op.getOpcode() == ISD::BITCAST &&
8320 Op.getValueType().isVector() &&
8321 Op.getOperand(i: 0).getValueType().isVector() &&
8322 Op.getValueType().getVectorNumElements() ==
8323 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8324 Op = Op.getOperand(i: 0);
8325
8326 // Pull BSWAP out of a vector extraction.
8327 if (Op.getOpcode() == ISD::BSWAP && Op.hasOneUse()) {
8328 EVT VecVT = Op.getValueType();
8329 EVT EltVT = VecVT.getVectorElementType();
8330 Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), VT: EltVT,
8331 N1: Op.getOperand(i: 0), N2: N->getOperand(Num: 1));
8332 DCI.AddToWorklist(N: Op.getNode());
8333 Op = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Op);
8334 if (EltVT != N->getValueType(ResNo: 0)) {
8335 DCI.AddToWorklist(N: Op.getNode());
8336 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
8337 }
8338 return Op;
8339 }
8340
8341 // Try to simplify a vector extraction.
8342 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
8343 SDValue Op0 = N->getOperand(Num: 0);
8344 EVT VecVT = Op0.getValueType();
8345 if (canTreatAsByteVector(VT: VecVT))
8346 return combineExtract(DL: SDLoc(N), ResVT: N->getValueType(ResNo: 0), VecVT, Op: Op0,
8347 Index: IndexN->getZExtValue(), DCI, Force: false);
8348 }
8349 return SDValue();
8350}
8351
8352SDValue SystemZTargetLowering::combineJOIN_DWORDS(
8353 SDNode *N, DAGCombinerInfo &DCI) const {
8354 SelectionDAG &DAG = DCI.DAG;
8355 // (join_dwords X, X) == (replicate X)
8356 if (N->getOperand(Num: 0) == N->getOperand(Num: 1))
8357 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8358 Operand: N->getOperand(Num: 0));
8359 return SDValue();
8360}
8361
8362static SDValue MergeInputChains(SDNode *N1, SDNode *N2) {
8363 SDValue Chain1 = N1->getOperand(Num: 0);
8364 SDValue Chain2 = N2->getOperand(Num: 0);
8365
8366 // Trivial case: both nodes take the same chain.
8367 if (Chain1 == Chain2)
8368 return Chain1;
8369
8370 // FIXME - we could handle more complex cases via TokenFactor,
8371 // assuming we can verify that this would not create a cycle.
8372 return SDValue();
8373}
8374
8375SDValue SystemZTargetLowering::combineFP_ROUND(
8376 SDNode *N, DAGCombinerInfo &DCI) const {
8377
8378 if (!Subtarget.hasVector())
8379 return SDValue();
8380
8381 // (fpround (extract_vector_elt X 0))
8382 // (fpround (extract_vector_elt X 1)) ->
8383 // (extract_vector_elt (VROUND X) 0)
8384 // (extract_vector_elt (VROUND X) 2)
8385 //
8386 // This is a special case since the target doesn't really support v2f32s.
8387 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8388 SelectionDAG &DAG = DCI.DAG;
8389 SDValue Op0 = N->getOperand(Num: OpNo);
8390 if (N->getValueType(ResNo: 0) == MVT::f32 && Op0.hasOneUse() &&
8391 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8392 Op0.getOperand(i: 0).getValueType() == MVT::v2f64 &&
8393 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8394 Op0.getConstantOperandVal(i: 1) == 0) {
8395 SDValue Vec = Op0.getOperand(i: 0);
8396 for (auto *U : Vec->users()) {
8397 if (U != Op0.getNode() && U->hasOneUse() &&
8398 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8399 U->getOperand(Num: 0) == Vec &&
8400 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8401 U->getConstantOperandVal(Num: 1) == 1) {
8402 SDValue OtherRound = SDValue(*U->user_begin(), 0);
8403 if (OtherRound.getOpcode() == N->getOpcode() &&
8404 OtherRound.getOperand(i: OpNo) == SDValue(U, 0) &&
8405 OtherRound.getValueType() == MVT::f32) {
8406 SDValue VRound, Chain;
8407 if (N->isStrictFPOpcode()) {
8408 Chain = MergeInputChains(N1: N, N2: OtherRound.getNode());
8409 if (!Chain)
8410 continue;
8411 VRound = DAG.getNode(Opcode: SystemZISD::STRICT_VROUND, DL: SDLoc(N),
8412 ResultTys: {MVT::v4f32, MVT::Other}, Ops: {Chain, Vec});
8413 Chain = VRound.getValue(R: 1);
8414 } else
8415 VRound = DAG.getNode(Opcode: SystemZISD::VROUND, DL: SDLoc(N),
8416 VT: MVT::v4f32, Operand: Vec);
8417 DCI.AddToWorklist(N: VRound.getNode());
8418 SDValue Extract1 =
8419 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f32,
8420 N1: VRound, N2: DAG.getConstant(Val: 2, DL: SDLoc(U), VT: MVT::i32));
8421 DCI.AddToWorklist(N: Extract1.getNode());
8422 DAG.ReplaceAllUsesOfValueWith(From: OtherRound, To: Extract1);
8423 if (Chain)
8424 DAG.ReplaceAllUsesOfValueWith(From: OtherRound.getValue(R: 1), To: Chain);
8425 SDValue Extract0 =
8426 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f32,
8427 N1: VRound, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8428 if (Chain)
8429 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8430 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8431 return Extract0;
8432 }
8433 }
8434 }
8435 }
8436 return SDValue();
8437}
8438
8439SDValue SystemZTargetLowering::combineFP_EXTEND(
8440 SDNode *N, DAGCombinerInfo &DCI) const {
8441
8442 if (!Subtarget.hasVector())
8443 return SDValue();
8444
8445 // (fpextend (extract_vector_elt X 0))
8446 // (fpextend (extract_vector_elt X 2)) ->
8447 // (extract_vector_elt (VEXTEND X) 0)
8448 // (extract_vector_elt (VEXTEND X) 1)
8449 //
8450 // This is a special case since the target doesn't really support v2f32s.
8451 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8452 SelectionDAG &DAG = DCI.DAG;
8453 SDValue Op0 = N->getOperand(Num: OpNo);
8454 if (N->getValueType(ResNo: 0) == MVT::f64 && Op0.hasOneUse() &&
8455 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8456 Op0.getOperand(i: 0).getValueType() == MVT::v4f32 &&
8457 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8458 Op0.getConstantOperandVal(i: 1) == 0) {
8459 SDValue Vec = Op0.getOperand(i: 0);
8460 for (auto *U : Vec->users()) {
8461 if (U != Op0.getNode() && U->hasOneUse() &&
8462 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8463 U->getOperand(Num: 0) == Vec &&
8464 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8465 U->getConstantOperandVal(Num: 1) == 2) {
8466 SDValue OtherExtend = SDValue(*U->user_begin(), 0);
8467 if (OtherExtend.getOpcode() == N->getOpcode() &&
8468 OtherExtend.getOperand(i: OpNo) == SDValue(U, 0) &&
8469 OtherExtend.getValueType() == MVT::f64) {
8470 SDValue VExtend, Chain;
8471 if (N->isStrictFPOpcode()) {
8472 Chain = MergeInputChains(N1: N, N2: OtherExtend.getNode());
8473 if (!Chain)
8474 continue;
8475 VExtend = DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL: SDLoc(N),
8476 ResultTys: {MVT::v2f64, MVT::Other}, Ops: {Chain, Vec});
8477 Chain = VExtend.getValue(R: 1);
8478 } else
8479 VExtend = DAG.getNode(Opcode: SystemZISD::VEXTEND, DL: SDLoc(N),
8480 VT: MVT::v2f64, Operand: Vec);
8481 DCI.AddToWorklist(N: VExtend.getNode());
8482 SDValue Extract1 =
8483 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f64,
8484 N1: VExtend, N2: DAG.getConstant(Val: 1, DL: SDLoc(U), VT: MVT::i32));
8485 DCI.AddToWorklist(N: Extract1.getNode());
8486 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend, To: Extract1);
8487 if (Chain)
8488 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend.getValue(R: 1), To: Chain);
8489 SDValue Extract0 =
8490 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f64,
8491 N1: VExtend, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8492 if (Chain)
8493 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8494 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8495 return Extract0;
8496 }
8497 }
8498 }
8499 }
8500 return SDValue();
8501}
8502
8503SDValue SystemZTargetLowering::combineINT_TO_FP(
8504 SDNode *N, DAGCombinerInfo &DCI) const {
8505 if (DCI.Level != BeforeLegalizeTypes)
8506 return SDValue();
8507 SelectionDAG &DAG = DCI.DAG;
8508 LLVMContext &Ctx = *DAG.getContext();
8509 unsigned Opcode = N->getOpcode();
8510 EVT OutVT = N->getValueType(ResNo: 0);
8511 Type *OutLLVMTy = OutVT.getTypeForEVT(Context&: Ctx);
8512 SDValue Op = N->getOperand(Num: 0);
8513 unsigned OutScalarBits = OutLLVMTy->getScalarSizeInBits();
8514 unsigned InScalarBits = Op->getValueType(ResNo: 0).getScalarSizeInBits();
8515
8516 // Insert an extension before type-legalization to avoid scalarization, e.g.:
8517 // v2f64 = uint_to_fp v2i16
8518 // =>
8519 // v2f64 = uint_to_fp (v2i64 zero_extend v2i16)
8520 if (OutLLVMTy->isVectorTy() && OutScalarBits > InScalarBits &&
8521 OutScalarBits <= 64) {
8522 unsigned NumElts = cast<FixedVectorType>(Val: OutLLVMTy)->getNumElements();
8523 EVT ExtVT = EVT::getVectorVT(
8524 Context&: Ctx, VT: EVT::getIntegerVT(Context&: Ctx, BitWidth: OutLLVMTy->getScalarSizeInBits()), NumElements: NumElts);
8525 unsigned ExtOpcode =
8526 (Opcode == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND);
8527 SDValue ExtOp = DAG.getNode(Opcode: ExtOpcode, DL: SDLoc(N), VT: ExtVT, Operand: Op);
8528 return DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: ExtOp);
8529 }
8530 return SDValue();
8531}
8532
8533SDValue SystemZTargetLowering::combineFCOPYSIGN(
8534 SDNode *N, DAGCombinerInfo &DCI) const {
8535 SelectionDAG &DAG = DCI.DAG;
8536 EVT VT = N->getValueType(ResNo: 0);
8537 SDValue ValOp = N->getOperand(Num: 0);
8538 SDValue SignOp = N->getOperand(Num: 1);
8539
8540 // Remove the rounding which is not needed.
8541 if (SignOp.getOpcode() == ISD::FP_ROUND) {
8542 SDValue WideOp = SignOp.getOperand(i: 0);
8543 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N), VT, N1: ValOp, N2: WideOp);
8544 }
8545
8546 return SDValue();
8547}
8548
8549SDValue SystemZTargetLowering::combineBSWAP(
8550 SDNode *N, DAGCombinerInfo &DCI) const {
8551 SelectionDAG &DAG = DCI.DAG;
8552 // Combine BSWAP (LOAD) into LRVH/LRV/LRVG/VLBR
8553 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8554 N->getOperand(Num: 0).hasOneUse() &&
8555 canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0))) {
8556 SDValue Load = N->getOperand(Num: 0);
8557 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8558
8559 // Create the byte-swapping load.
8560 SDValue Ops[] = {
8561 LD->getChain(), // Chain
8562 LD->getBasePtr() // Ptr
8563 };
8564 EVT LoadVT = N->getValueType(ResNo: 0);
8565 if (LoadVT == MVT::i16)
8566 LoadVT = MVT::i32;
8567 SDValue BSLoad =
8568 DAG.getMemIntrinsicNode(Opcode: SystemZISD::LRV, dl: SDLoc(N),
8569 VTList: DAG.getVTList(VT1: LoadVT, VT2: MVT::Other),
8570 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8571
8572 // If this is an i16 load, insert the truncate.
8573 SDValue ResVal = BSLoad;
8574 if (N->getValueType(ResNo: 0) == MVT::i16)
8575 ResVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: MVT::i16, Operand: BSLoad);
8576
8577 // First, combine the bswap away. This makes the value produced by the
8578 // load dead.
8579 DCI.CombineTo(N, Res: ResVal);
8580
8581 // Next, combine the load away, we give it a bogus result value but a real
8582 // chain result. The result value is dead because the bswap is dead.
8583 DCI.CombineTo(N: Load.getNode(), Res0: ResVal, Res1: BSLoad.getValue(R: 1));
8584
8585 // Return N so it doesn't get rechecked!
8586 return SDValue(N, 0);
8587 }
8588
8589 // Look through bitcasts that retain the number of vector elements.
8590 SDValue Op = N->getOperand(Num: 0);
8591 if (Op.getOpcode() == ISD::BITCAST &&
8592 Op.getValueType().isVector() &&
8593 Op.getOperand(i: 0).getValueType().isVector() &&
8594 Op.getValueType().getVectorNumElements() ==
8595 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8596 Op = Op.getOperand(i: 0);
8597
8598 // Push BSWAP into a vector insertion if at least one side then simplifies.
8599 if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT && Op.hasOneUse()) {
8600 SDValue Vec = Op.getOperand(i: 0);
8601 SDValue Elt = Op.getOperand(i: 1);
8602 SDValue Idx = Op.getOperand(i: 2);
8603
8604 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Vec) ||
8605 Vec.getOpcode() == ISD::BSWAP || Vec.isUndef() ||
8606 DAG.isConstantIntBuildVectorOrConstantInt(N: Elt) ||
8607 Elt.getOpcode() == ISD::BSWAP || Elt.isUndef() ||
8608 (canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0)) &&
8609 ISD::isNON_EXTLoad(N: Elt.getNode()) && Elt.hasOneUse())) {
8610 EVT VecVT = N->getValueType(ResNo: 0);
8611 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
8612 if (VecVT != Vec.getValueType()) {
8613 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8614 DCI.AddToWorklist(N: Vec.getNode());
8615 }
8616 if (EltVT != Elt.getValueType()) {
8617 Elt = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8618 DCI.AddToWorklist(N: Elt.getNode());
8619 }
8620 Vec = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8621 DCI.AddToWorklist(N: Vec.getNode());
8622 Elt = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8623 DCI.AddToWorklist(N: Elt.getNode());
8624 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: VecVT,
8625 N1: Vec, N2: Elt, N3: Idx);
8626 }
8627 }
8628
8629 // Push BSWAP into a vector shuffle if at least one side then simplifies.
8630 ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(Val&: Op);
8631 if (SV && Op.hasOneUse()) {
8632 SDValue Op0 = Op.getOperand(i: 0);
8633 SDValue Op1 = Op.getOperand(i: 1);
8634
8635 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Op0) ||
8636 Op0.getOpcode() == ISD::BSWAP || Op0.isUndef() ||
8637 DAG.isConstantIntBuildVectorOrConstantInt(N: Op1) ||
8638 Op1.getOpcode() == ISD::BSWAP || Op1.isUndef()) {
8639 EVT VecVT = N->getValueType(ResNo: 0);
8640 if (VecVT != Op0.getValueType()) {
8641 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8642 DCI.AddToWorklist(N: Op0.getNode());
8643 }
8644 if (VecVT != Op1.getValueType()) {
8645 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8646 DCI.AddToWorklist(N: Op1.getNode());
8647 }
8648 Op0 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8649 DCI.AddToWorklist(N: Op0.getNode());
8650 Op1 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8651 DCI.AddToWorklist(N: Op1.getNode());
8652 return DAG.getVectorShuffle(VT: VecVT, dl: SDLoc(N), N1: Op0, N2: Op1, Mask: SV->getMask());
8653 }
8654 }
8655
8656 return SDValue();
8657}
8658
8659SDValue SystemZTargetLowering::combineSETCC(
8660 SDNode *N, DAGCombinerInfo &DCI) const {
8661 SelectionDAG &DAG = DCI.DAG;
8662 const ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
8663 const SDValue LHS = N->getOperand(Num: 0);
8664 const SDValue RHS = N->getOperand(Num: 1);
8665 bool CmpNull = isNullConstant(V: RHS);
8666 bool CmpAllOnes = isAllOnesConstant(V: RHS);
8667 EVT VT = N->getValueType(ResNo: 0);
8668 SDLoc DL(N);
8669
8670 // Match icmp_eq/ne(bitcast(icmp(X,Y)),0/-1) reduction patterns, and
8671 // change the outer compare to a i128 compare. This will normally
8672 // allow the reduction to be recognized in adjustICmp128, and even if
8673 // not, the i128 compare will still generate better code.
8674 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && (CmpNull || CmpAllOnes)) {
8675 SDValue Src = peekThroughBitcasts(V: LHS);
8676 if (Src.getOpcode() == ISD::SETCC &&
8677 Src.getValueType().isFixedLengthVector() &&
8678 Src.getValueType().getScalarType() == MVT::i1) {
8679 EVT CmpVT = Src.getOperand(i: 0).getValueType();
8680 if (CmpVT.getSizeInBits() == 128) {
8681 EVT IntVT = CmpVT.changeVectorElementTypeToInteger();
8682 SDValue LHS =
8683 DAG.getBitcast(VT: MVT::i128, V: DAG.getSExtOrTrunc(Op: Src, DL, VT: IntVT));
8684 SDValue RHS = CmpNull ? DAG.getConstant(Val: 0, DL, VT: MVT::i128)
8685 : DAG.getAllOnesConstant(DL, VT: MVT::i128);
8686 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: LHS, N2: RHS, N3: N->getOperand(Num: 2),
8687 Flags: N->getFlags());
8688 }
8689 }
8690 }
8691
8692 return SDValue();
8693}
8694
8695static std::pair<SDValue, int> findCCUse(const SDValue &Val,
8696 unsigned Depth = 0) {
8697 // Limit depth of potentially exponential walk.
8698 if (Depth > 5)
8699 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8700
8701 switch (Val.getOpcode()) {
8702 default:
8703 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8704 case SystemZISD::IPM:
8705 if (Val.getOperand(i: 0).getOpcode() == SystemZISD::CLC ||
8706 Val.getOperand(i: 0).getOpcode() == SystemZISD::STRCMP)
8707 return std::make_pair(x: Val.getOperand(i: 0), y: SystemZ::CCMASK_ICMP);
8708 return std::make_pair(x: Val.getOperand(i: 0), y: SystemZ::CCMASK_ANY);
8709 case SystemZISD::SELECT_CCMASK: {
8710 SDValue Op4CCReg = Val.getOperand(i: 4);
8711 if (Op4CCReg.getOpcode() == SystemZISD::ICMP ||
8712 Op4CCReg.getOpcode() == SystemZISD::TM) {
8713 auto [OpCC, OpCCValid] = findCCUse(Val: Op4CCReg.getOperand(i: 0), Depth: Depth + 1);
8714 if (OpCC != SDValue())
8715 return std::make_pair(x&: OpCC, y&: OpCCValid);
8716 }
8717 auto *CCValid = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 2));
8718 if (!CCValid)
8719 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8720 int CCValidVal = CCValid->getZExtValue();
8721 return std::make_pair(x&: Op4CCReg, y&: CCValidVal);
8722 }
8723 case ISD::ADD:
8724 case ISD::AND:
8725 case ISD::OR:
8726 case ISD::XOR:
8727 case ISD::SHL:
8728 case ISD::SRA:
8729 case ISD::SRL:
8730 auto [Op0CC, Op0CCValid] = findCCUse(Val: Val.getOperand(i: 0), Depth: Depth + 1);
8731 if (Op0CC != SDValue())
8732 return std::make_pair(x&: Op0CC, y&: Op0CCValid);
8733 return findCCUse(Val: Val.getOperand(i: 1), Depth: Depth + 1);
8734 }
8735}
8736
8737static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
8738 SelectionDAG &DAG);
8739
8740SmallVector<SDValue, 4> static simplifyAssumingCCVal(SDValue &Val, SDValue &CC,
8741 SelectionDAG &DAG) {
8742 SDLoc DL(Val);
8743 auto Opcode = Val.getOpcode();
8744 switch (Opcode) {
8745 default:
8746 return {};
8747 case ISD::Constant:
8748 return {Val, Val, Val, Val};
8749 case SystemZISD::IPM: {
8750 SDValue IPMOp0 = Val.getOperand(i: 0);
8751 if (IPMOp0 != CC)
8752 return {};
8753 SmallVector<SDValue, 4> ShiftedCCVals;
8754 for (auto CC : {0, 1, 2, 3})
8755 ShiftedCCVals.emplace_back(
8756 Args: DAG.getConstant(Val: (CC << SystemZ::IPM_CC), DL, VT: MVT::i32));
8757 return ShiftedCCVals;
8758 }
8759 case SystemZISD::SELECT_CCMASK: {
8760 SDValue TrueVal = Val.getOperand(i: 0), FalseVal = Val.getOperand(i: 1);
8761 auto *CCValid = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 2));
8762 auto *CCMask = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 3));
8763 if (!CCValid || !CCMask)
8764 return {};
8765
8766 int CCValidVal = CCValid->getZExtValue();
8767 int CCMaskVal = CCMask->getZExtValue();
8768 // Pruning search tree early - Moving CC test and combineCCMask ahead of
8769 // recursive call to simplifyAssumingCCVal.
8770 SDValue Op4CCReg = Val.getOperand(i: 4);
8771 if (Op4CCReg != CC)
8772 combineCCMask(CCReg&: Op4CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG);
8773 if (Op4CCReg != CC)
8774 return {};
8775 const auto &&TrueSDVals = simplifyAssumingCCVal(Val&: TrueVal, CC, DAG);
8776 const auto &&FalseSDVals = simplifyAssumingCCVal(Val&: FalseVal, CC, DAG);
8777 if (TrueSDVals.empty() || FalseSDVals.empty())
8778 return {};
8779 SmallVector<SDValue, 4> MergedSDVals;
8780 for (auto &CCVal : {0, 1, 2, 3})
8781 MergedSDVals.emplace_back(Args: ((CCMaskVal & (1 << (3 - CCVal))) != 0)
8782 ? TrueSDVals[CCVal]
8783 : FalseSDVals[CCVal]);
8784 return MergedSDVals;
8785 }
8786 case ISD::ADD:
8787 case ISD::AND:
8788 case ISD::OR:
8789 case ISD::XOR:
8790 case ISD::SRA:
8791 // Avoid introducing CC spills (because ADD/AND/OR/XOR/SRA
8792 // would clobber CC).
8793 if (!Val.hasOneUse())
8794 return {};
8795 [[fallthrough]];
8796 case ISD::SHL:
8797 case ISD::SRL:
8798 SDValue Op0 = Val.getOperand(i: 0), Op1 = Val.getOperand(i: 1);
8799 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: Op0, CC, DAG);
8800 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: Op1, CC, DAG);
8801 if (Op0SDVals.empty() || Op1SDVals.empty())
8802 return {};
8803 SmallVector<SDValue, 4> BinaryOpSDVals;
8804 for (auto CCVal : {0, 1, 2, 3})
8805 BinaryOpSDVals.emplace_back(Args: DAG.getNode(
8806 Opcode, DL, VT: Val.getValueType(), N1: Op0SDVals[CCVal], N2: Op1SDVals[CCVal]));
8807 return BinaryOpSDVals;
8808 }
8809}
8810
8811static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
8812 SelectionDAG &DAG) {
8813 // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
8814 // set by the CCReg instruction using the CCValid / CCMask masks,
8815 // If the CCReg instruction is itself a ICMP / TM testing the condition
8816 // code set by some other instruction, see whether we can directly
8817 // use that condition code.
8818 auto *CCNode = CCReg.getNode();
8819 if (!CCNode)
8820 return false;
8821
8822 if (CCNode->getOpcode() == SystemZISD::TM) {
8823 if (CCValid != SystemZ::CCMASK_TM)
8824 return false;
8825 auto emulateTMCCMask = [](const SDValue &Op0Val, const SDValue &Op1Val) {
8826 auto *Op0Node = dyn_cast<ConstantSDNode>(Val: Op0Val.getNode());
8827 auto *Op1Node = dyn_cast<ConstantSDNode>(Val: Op1Val.getNode());
8828 if (!Op0Node || !Op1Node)
8829 return -1;
8830 auto Op0APVal = Op0Node->getAPIntValue();
8831 auto Op1APVal = Op1Node->getAPIntValue();
8832 auto Result = Op0APVal & Op1APVal;
8833 bool AllOnes = Result == Op1APVal;
8834 bool AllZeros = Result == 0;
8835 bool IsLeftMostBitSet = Result[Op1APVal.getActiveBits()] != 0;
8836 return AllZeros ? 0 : AllOnes ? 3 : IsLeftMostBitSet ? 2 : 1;
8837 };
8838 SDValue Op0 = CCNode->getOperand(Num: 0);
8839 SDValue Op1 = CCNode->getOperand(Num: 1);
8840 auto [Op0CC, Op0CCValid] = findCCUse(Val: Op0);
8841 if (Op0CC == SDValue())
8842 return false;
8843 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: Op0, CC&: Op0CC, DAG);
8844 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: Op1, CC&: Op0CC, DAG);
8845 if (Op0SDVals.empty() || Op1SDVals.empty())
8846 return false;
8847 int NewCCMask = 0;
8848 for (auto CC : {0, 1, 2, 3}) {
8849 auto CCVal = emulateTMCCMask(Op0SDVals[CC], Op1SDVals[CC]);
8850 if (CCVal < 0)
8851 return false;
8852 NewCCMask <<= 1;
8853 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
8854 }
8855 NewCCMask &= Op0CCValid;
8856 CCReg = Op0CC;
8857 CCMask = NewCCMask;
8858 CCValid = Op0CCValid;
8859 return true;
8860 }
8861 if (CCNode->getOpcode() != SystemZISD::ICMP ||
8862 CCValid != SystemZ::CCMASK_ICMP)
8863 return false;
8864
8865 SDValue CmpOp0 = CCNode->getOperand(Num: 0);
8866 SDValue CmpOp1 = CCNode->getOperand(Num: 1);
8867 SDValue CmpOp2 = CCNode->getOperand(Num: 2);
8868 auto [Op0CC, Op0CCValid] = findCCUse(Val: CmpOp0);
8869 if (Op0CC != SDValue()) {
8870 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: CmpOp0, CC&: Op0CC, DAG);
8871 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: CmpOp1, CC&: Op0CC, DAG);
8872 if (Op0SDVals.empty() || Op1SDVals.empty())
8873 return false;
8874
8875 auto *CmpType = dyn_cast<ConstantSDNode>(Val&: CmpOp2);
8876 auto CmpTypeVal = CmpType->getZExtValue();
8877 const auto compareCCSigned = [&CmpTypeVal](const SDValue &Op0Val,
8878 const SDValue &Op1Val) {
8879 auto *Op0Node = dyn_cast<ConstantSDNode>(Val: Op0Val.getNode());
8880 auto *Op1Node = dyn_cast<ConstantSDNode>(Val: Op1Val.getNode());
8881 if (!Op0Node || !Op1Node)
8882 return -1;
8883 auto Op0APVal = Op0Node->getAPIntValue();
8884 auto Op1APVal = Op1Node->getAPIntValue();
8885 if (CmpTypeVal == SystemZICMP::SignedOnly)
8886 return Op0APVal == Op1APVal ? 0 : Op0APVal.slt(RHS: Op1APVal) ? 1 : 2;
8887 return Op0APVal == Op1APVal ? 0 : Op0APVal.ult(RHS: Op1APVal) ? 1 : 2;
8888 };
8889 int NewCCMask = 0;
8890 for (auto CC : {0, 1, 2, 3}) {
8891 auto CCVal = compareCCSigned(Op0SDVals[CC], Op1SDVals[CC]);
8892 if (CCVal < 0)
8893 return false;
8894 NewCCMask <<= 1;
8895 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
8896 }
8897 NewCCMask &= Op0CCValid;
8898 CCMask = NewCCMask;
8899 CCReg = Op0CC;
8900 CCValid = Op0CCValid;
8901 return true;
8902 }
8903
8904 return false;
8905}
8906
8907// Merging versus split in multiple branches cost.
8908TargetLoweringBase::CondMergingParams
8909SystemZTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
8910 const Value *Lhs,
8911 const Value *Rhs) const {
8912 const auto isFlagOutOpCC = [](const Value *V) {
8913 using namespace llvm::PatternMatch;
8914 const Value *RHSVal;
8915 const APInt *RHSC;
8916 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
8917 // PatternMatch.h provides concise tree-based pattern match of llvm IR.
8918 if (match(V: I->getOperand(i: 0), P: m_And(L: m_Value(V&: RHSVal), R: m_APInt(Res&: RHSC))) ||
8919 match(V: I, P: m_Cmp(L: m_Value(V&: RHSVal), R: m_APInt(Res&: RHSC)))) {
8920 if (const auto *CB = dyn_cast<CallBase>(Val: RHSVal)) {
8921 if (CB->isInlineAsm()) {
8922 const InlineAsm *IA = cast<InlineAsm>(Val: CB->getCalledOperand());
8923 return IA && IA->getConstraintString().contains(Other: "{@cc}");
8924 }
8925 }
8926 }
8927 }
8928 return false;
8929 };
8930 // Pattern (ICmp %asm) or (ICmp (And %asm)).
8931 // Cost of longest dependency chain (ICmp, And) is 2. CostThreshold or
8932 // BaseCost can be set >=2. If cost of instruction <= CostThreshold
8933 // conditionals will be merged or else conditionals will be split.
8934 if (isFlagOutOpCC(Lhs) && isFlagOutOpCC(Rhs))
8935 return {.BaseCost: 3, .LikelyBias: 0, .UnlikelyBias: -1};
8936 // Default.
8937 return {.BaseCost: -1, .LikelyBias: -1, .UnlikelyBias: -1};
8938}
8939
8940SDValue SystemZTargetLowering::combineBR_CCMASK(SDNode *N,
8941 DAGCombinerInfo &DCI) const {
8942 SelectionDAG &DAG = DCI.DAG;
8943
8944 // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK.
8945 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
8946 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
8947 if (!CCValid || !CCMask)
8948 return SDValue();
8949
8950 int CCValidVal = CCValid->getZExtValue();
8951 int CCMaskVal = CCMask->getZExtValue();
8952 SDValue Chain = N->getOperand(Num: 0);
8953 SDValue CCReg = N->getOperand(Num: 4);
8954 // If combineCMask was able to merge or simplify ccvalid or ccmask, re-emit
8955 // the modified BR_CCMASK with the new values.
8956 // In order to avoid conditional branches with full or empty cc masks, do not
8957 // do this if ccmask is 0 or equal to ccvalid.
8958 if (combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG) && CCMaskVal != 0 &&
8959 CCMaskVal != CCValidVal)
8960 return DAG.getNode(Opcode: SystemZISD::BR_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8961 N1: Chain,
8962 N2: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
8963 N3: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32),
8964 N4: N->getOperand(Num: 3), N5: CCReg);
8965 return SDValue();
8966}
8967
8968SDValue SystemZTargetLowering::combineSELECT_CCMASK(
8969 SDNode *N, DAGCombinerInfo &DCI) const {
8970 SelectionDAG &DAG = DCI.DAG;
8971
8972 // Combine SELECT_CCMASK (ICMP (SELECT_CCMASK)) into a single SELECT_CCMASK.
8973 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
8974 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3));
8975 if (!CCValid || !CCMask)
8976 return SDValue();
8977
8978 int CCValidVal = CCValid->getZExtValue();
8979 int CCMaskVal = CCMask->getZExtValue();
8980 SDValue CCReg = N->getOperand(Num: 4);
8981
8982 bool IsCombinedCCReg = combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG);
8983
8984 // Populate SDVals vector for each condition code ccval for given Val, which
8985 // can again be another nested select_ccmask with the same CC.
8986 const auto constructCCSDValsFromSELECT = [&CCReg](SDValue &Val) {
8987 if (Val.getOpcode() == SystemZISD::SELECT_CCMASK) {
8988 SmallVector<SDValue, 4> Res;
8989 if (Val.getOperand(i: 4) != CCReg)
8990 return SmallVector<SDValue, 4>{};
8991 SDValue TrueVal = Val.getOperand(i: 0), FalseVal = Val.getOperand(i: 1);
8992 auto *CCMask = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 3));
8993 if (!CCMask)
8994 return SmallVector<SDValue, 4>{};
8995
8996 int CCMaskVal = CCMask->getZExtValue();
8997 for (auto &CC : {0, 1, 2, 3})
8998 Res.emplace_back(Args&: ((CCMaskVal & (1 << (3 - CC))) != 0) ? TrueVal
8999 : FalseVal);
9000 return Res;
9001 }
9002 return SmallVector<SDValue, 4>{Val, Val, Val, Val};
9003 };
9004 // Attempting to optimize TrueVal/FalseVal in outermost select_ccmask either
9005 // with CCReg found by combineCCMask or original CCReg.
9006 SDValue TrueVal = N->getOperand(Num: 0);
9007 SDValue FalseVal = N->getOperand(Num: 1);
9008 auto &&TrueSDVals = simplifyAssumingCCVal(Val&: TrueVal, CC&: CCReg, DAG);
9009 auto &&FalseSDVals = simplifyAssumingCCVal(Val&: FalseVal, CC&: CCReg, DAG);
9010 // TrueSDVals/FalseSDVals might be empty in case of non-constant
9011 // TrueVal/FalseVal for select_ccmask, which can not be optimized further.
9012 if (TrueSDVals.empty())
9013 TrueSDVals = constructCCSDValsFromSELECT(TrueVal);
9014 if (FalseSDVals.empty())
9015 FalseSDVals = constructCCSDValsFromSELECT(FalseVal);
9016 if (!TrueSDVals.empty() && !FalseSDVals.empty()) {
9017 SmallSet<SDValue, 4> MergedSDValsSet;
9018 // Ignoring CC values outside CCValiid.
9019 for (auto CC : {0, 1, 2, 3}) {
9020 if ((CCValidVal & ((1 << (3 - CC)))) != 0)
9021 MergedSDValsSet.insert(V: ((CCMaskVal & (1 << (3 - CC))) != 0)
9022 ? TrueSDVals[CC]
9023 : FalseSDVals[CC]);
9024 }
9025 if (MergedSDValsSet.size() == 1)
9026 return *MergedSDValsSet.begin();
9027 if (MergedSDValsSet.size() == 2) {
9028 auto BeginIt = MergedSDValsSet.begin();
9029 SDValue NewTrueVal = *BeginIt, NewFalseVal = *next(x: BeginIt);
9030 if (NewTrueVal == FalseVal || NewFalseVal == TrueVal)
9031 std::swap(a&: NewTrueVal, b&: NewFalseVal);
9032 int NewCCMask = 0;
9033 for (auto CC : {0, 1, 2, 3}) {
9034 NewCCMask <<= 1;
9035 NewCCMask |= ((CCMaskVal & (1 << (3 - CC))) != 0)
9036 ? (TrueSDVals[CC] == NewTrueVal)
9037 : (FalseSDVals[CC] == NewTrueVal);
9038 }
9039 CCMaskVal = NewCCMask;
9040 CCMaskVal &= CCValidVal;
9041 TrueVal = NewTrueVal;
9042 FalseVal = NewFalseVal;
9043 IsCombinedCCReg = true;
9044 }
9045 }
9046 // If the condition is trivially false or trivially true after
9047 // combineCCMask, just collapse this SELECT_CCMASK to the indicated value
9048 // (possibly modified by constructCCSDValsFromSELECT).
9049 if (CCMaskVal == 0)
9050 return FalseVal;
9051 if (CCMaskVal == CCValidVal)
9052 return TrueVal;
9053
9054 if (IsCombinedCCReg)
9055 return DAG.getNode(
9056 Opcode: SystemZISD::SELECT_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: TrueVal,
9057 N2: FalseVal, N3: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
9058 N4: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32), N5: CCReg);
9059
9060 return SDValue();
9061}
9062
9063SDValue SystemZTargetLowering::combineGET_CCMASK(
9064 SDNode *N, DAGCombinerInfo &DCI) const {
9065
9066 // Optimize away GET_CCMASK (SELECT_CCMASK) if the CC masks are compatible
9067 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
9068 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9069 if (!CCValid || !CCMask)
9070 return SDValue();
9071 int CCValidVal = CCValid->getZExtValue();
9072 int CCMaskVal = CCMask->getZExtValue();
9073
9074 SDValue Select = N->getOperand(Num: 0);
9075 if (Select->getOpcode() == ISD::TRUNCATE)
9076 Select = Select->getOperand(Num: 0);
9077 if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
9078 return SDValue();
9079
9080 auto *SelectCCValid = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 2));
9081 auto *SelectCCMask = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 3));
9082 if (!SelectCCValid || !SelectCCMask)
9083 return SDValue();
9084 int SelectCCValidVal = SelectCCValid->getZExtValue();
9085 int SelectCCMaskVal = SelectCCMask->getZExtValue();
9086
9087 auto *TrueVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 0));
9088 auto *FalseVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 1));
9089 if (!TrueVal || !FalseVal)
9090 return SDValue();
9091 if (TrueVal->getZExtValue() == 1 && FalseVal->getZExtValue() == 0)
9092 ;
9093 else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() == 1)
9094 SelectCCMaskVal ^= SelectCCValidVal;
9095 else
9096 return SDValue();
9097
9098 if (SelectCCValidVal & ~CCValidVal)
9099 return SDValue();
9100 if (SelectCCMaskVal != (CCMaskVal & SelectCCValidVal))
9101 return SDValue();
9102
9103 return Select->getOperand(Num: 4);
9104}
9105
9106SDValue SystemZTargetLowering::combineIntDIVREM(
9107 SDNode *N, DAGCombinerInfo &DCI) const {
9108 SelectionDAG &DAG = DCI.DAG;
9109 EVT VT = N->getValueType(ResNo: 0);
9110 // In the case where the divisor is a vector of constants a cheaper
9111 // sequence of instructions can replace the divide. BuildSDIV is called to
9112 // do this during DAG combining, but it only succeeds when it can build a
9113 // multiplication node. The only option for SystemZ is ISD::SMUL_LOHI, and
9114 // since it is not Legal but Custom it can only happen before
9115 // legalization. Therefore we must scalarize this early before Combine
9116 // 1. For widened vectors, this is already the result of type legalization.
9117 if (DCI.Level == BeforeLegalizeTypes && VT.isVector() && isTypeLegal(VT) &&
9118 DAG.isConstantIntBuildVectorOrConstantInt(N: N->getOperand(Num: 1)))
9119 return DAG.UnrollVectorOp(N);
9120 return SDValue();
9121}
9122
9123
9124// Transform a right shift of a multiply-and-add into a multiply-and-add-high.
9125// This is closely modeled after the common-code combineShiftToMULH.
9126SDValue SystemZTargetLowering::combineShiftToMulAddHigh(
9127 SDNode *N, DAGCombinerInfo &DCI) const {
9128 SelectionDAG &DAG = DCI.DAG;
9129 SDLoc DL(N);
9130
9131 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
9132 "SRL or SRA node is required here!");
9133
9134 if (!Subtarget.hasVector())
9135 return SDValue();
9136
9137 // Check the shift amount. Proceed with the transformation if the shift
9138 // amount is constant.
9139 ConstantSDNode *ShiftAmtSrc = isConstOrConstSplat(N: N->getOperand(Num: 1));
9140 if (!ShiftAmtSrc)
9141 return SDValue();
9142
9143 // The operation feeding into the shift must be an add.
9144 SDValue ShiftOperand = N->getOperand(Num: 0);
9145 if (ShiftOperand.getOpcode() != ISD::ADD)
9146 return SDValue();
9147
9148 // One operand of the add must be a multiply.
9149 SDValue MulOp = ShiftOperand.getOperand(i: 0);
9150 SDValue AddOp = ShiftOperand.getOperand(i: 1);
9151 if (MulOp.getOpcode() != ISD::MUL) {
9152 if (AddOp.getOpcode() != ISD::MUL)
9153 return SDValue();
9154 std::swap(a&: MulOp, b&: AddOp);
9155 }
9156
9157 // All operands must be equivalent extend nodes.
9158 SDValue LeftOp = MulOp.getOperand(i: 0);
9159 SDValue RightOp = MulOp.getOperand(i: 1);
9160
9161 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND;
9162 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND;
9163
9164 if (!IsSignExt && !IsZeroExt)
9165 return SDValue();
9166
9167 EVT NarrowVT = LeftOp.getOperand(i: 0).getValueType();
9168 unsigned NarrowVTSize = NarrowVT.getScalarSizeInBits();
9169
9170 SDValue MulhRightOp;
9171 if (ConstantSDNode *Constant = isConstOrConstSplat(N: RightOp)) {
9172 unsigned ActiveBits = IsSignExt
9173 ? Constant->getAPIntValue().getSignificantBits()
9174 : Constant->getAPIntValue().getActiveBits();
9175 if (ActiveBits > NarrowVTSize)
9176 return SDValue();
9177 MulhRightOp = DAG.getConstant(
9178 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
9179 VT: NarrowVT);
9180 } else {
9181 if (LeftOp.getOpcode() != RightOp.getOpcode())
9182 return SDValue();
9183 // Check that the two extend nodes are the same type.
9184 if (NarrowVT != RightOp.getOperand(i: 0).getValueType())
9185 return SDValue();
9186 MulhRightOp = RightOp.getOperand(i: 0);
9187 }
9188
9189 SDValue MulhAddOp;
9190 if (ConstantSDNode *Constant = isConstOrConstSplat(N: AddOp)) {
9191 unsigned ActiveBits = IsSignExt
9192 ? Constant->getAPIntValue().getSignificantBits()
9193 : Constant->getAPIntValue().getActiveBits();
9194 if (ActiveBits > NarrowVTSize)
9195 return SDValue();
9196 MulhAddOp = DAG.getConstant(
9197 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
9198 VT: NarrowVT);
9199 } else {
9200 if (LeftOp.getOpcode() != AddOp.getOpcode())
9201 return SDValue();
9202 // Check that the two extend nodes are the same type.
9203 if (NarrowVT != AddOp.getOperand(i: 0).getValueType())
9204 return SDValue();
9205 MulhAddOp = AddOp.getOperand(i: 0);
9206 }
9207
9208 EVT WideVT = LeftOp.getValueType();
9209 // Proceed with the transformation if the wide types match.
9210 assert((WideVT == RightOp.getValueType()) &&
9211 "Cannot have a multiply node with two different operand types.");
9212 assert((WideVT == AddOp.getValueType()) &&
9213 "Cannot have an add node with two different operand types.");
9214
9215 // Proceed with the transformation if the wide type is twice as large
9216 // as the narrow type.
9217 if (WideVT.getScalarSizeInBits() != 2 * NarrowVTSize)
9218 return SDValue();
9219
9220 // Check the shift amount with the narrow type size.
9221 // Proceed with the transformation if the shift amount is the width
9222 // of the narrow type.
9223 unsigned ShiftAmt = ShiftAmtSrc->getZExtValue();
9224 if (ShiftAmt != NarrowVTSize)
9225 return SDValue();
9226
9227 // Proceed if we support the multiply-and-add-high operation.
9228 if (!(NarrowVT == MVT::v16i8 || NarrowVT == MVT::v8i16 ||
9229 NarrowVT == MVT::v4i32 ||
9230 (Subtarget.hasVectorEnhancements3() &&
9231 (NarrowVT == MVT::v2i64 || NarrowVT == MVT::i128))))
9232 return SDValue();
9233
9234 // Emit the VMAH (signed) or VMALH (unsigned) operation.
9235 SDValue Result = DAG.getNode(Opcode: IsSignExt ? SystemZISD::VMAH : SystemZISD::VMALH,
9236 DL, VT: NarrowVT, N1: LeftOp.getOperand(i: 0),
9237 N2: MulhRightOp, N3: MulhAddOp);
9238 bool IsSigned = N->getOpcode() == ISD::SRA;
9239 return DAG.getExtOrTrunc(IsSigned, Op: Result, DL, VT: WideVT);
9240}
9241
9242// Op is an operand of a multiplication. Check whether this can be folded
9243// into an even/odd widening operation; if so, return the opcode to be used
9244// and update Op to the appropriate sub-operand. Note that the caller must
9245// verify that *both* operands of the multiplication support the operation.
9246static unsigned detectEvenOddMultiplyOperand(const SelectionDAG &DAG,
9247 const SystemZSubtarget &Subtarget,
9248 SDValue &Op) {
9249 EVT VT = Op.getValueType();
9250
9251 // Check for (sign/zero_extend_vector_inreg (vector_shuffle)) corresponding
9252 // to selecting the even or odd vector elements.
9253 if (VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
9254 (Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
9255 Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG)) {
9256 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
9257 unsigned NumElts = VT.getVectorNumElements();
9258 Op = Op.getOperand(i: 0);
9259 if (Op.getValueType().getVectorNumElements() == 2 * NumElts &&
9260 Op.getOpcode() == ISD::VECTOR_SHUFFLE) {
9261 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
9262 ArrayRef<int> ShuffleMask = SVN->getMask();
9263 bool CanUseEven = true, CanUseOdd = true;
9264 for (unsigned Elt = 0; Elt < NumElts; Elt++) {
9265 if (ShuffleMask[Elt] == -1)
9266 continue;
9267 if (unsigned(ShuffleMask[Elt]) != 2 * Elt)
9268 CanUseEven = false;
9269 if (unsigned(ShuffleMask[Elt]) != 2 * Elt + 1)
9270 CanUseOdd = false;
9271 }
9272 Op = Op.getOperand(i: 0);
9273 if (CanUseEven)
9274 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9275 if (CanUseOdd)
9276 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9277 }
9278 }
9279
9280 // For z17, we can also support the v2i64->i128 case, which looks like
9281 // (sign/zero_extend (extract_vector_elt X 0/1))
9282 if (VT == MVT::i128 && Subtarget.hasVectorEnhancements3() &&
9283 (Op.getOpcode() == ISD::SIGN_EXTEND ||
9284 Op.getOpcode() == ISD::ZERO_EXTEND)) {
9285 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND;
9286 Op = Op.getOperand(i: 0);
9287 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
9288 Op.getOperand(i: 0).getValueType() == MVT::v2i64 &&
9289 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
9290 unsigned Elem = Op.getConstantOperandVal(i: 1);
9291 Op = Op.getOperand(i: 0);
9292 if (Elem == 0)
9293 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9294 if (Elem == 1)
9295 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9296 }
9297 }
9298
9299 return 0;
9300}
9301
9302SDValue SystemZTargetLowering::combineMUL(
9303 SDNode *N, DAGCombinerInfo &DCI) const {
9304 SelectionDAG &DAG = DCI.DAG;
9305
9306 // Detect even/odd widening multiplication.
9307 SDValue Op0 = N->getOperand(Num: 0);
9308 SDValue Op1 = N->getOperand(Num: 1);
9309 unsigned OpcodeCand0 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op0);
9310 unsigned OpcodeCand1 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op1);
9311 if (OpcodeCand0 && OpcodeCand0 == OpcodeCand1)
9312 return DAG.getNode(Opcode: OpcodeCand0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1);
9313
9314 return SDValue();
9315}
9316
9317SDValue SystemZTargetLowering::combineINTRINSIC(
9318 SDNode *N, DAGCombinerInfo &DCI) const {
9319 SelectionDAG &DAG = DCI.DAG;
9320
9321 unsigned Id = N->getConstantOperandVal(Num: 1);
9322 switch (Id) {
9323 // VECTOR LOAD (RIGHTMOST) WITH LENGTH with a length operand of 15
9324 // or larger is simply a vector load.
9325 case Intrinsic::s390_vll:
9326 case Intrinsic::s390_vlrl:
9327 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2)))
9328 if (C->getZExtValue() >= 15)
9329 return DAG.getLoad(VT: N->getValueType(ResNo: 0), dl: SDLoc(N), Chain: N->getOperand(Num: 0),
9330 Ptr: N->getOperand(Num: 3), PtrInfo: MachinePointerInfo());
9331 break;
9332 // Likewise for VECTOR STORE (RIGHTMOST) WITH LENGTH.
9333 case Intrinsic::s390_vstl:
9334 case Intrinsic::s390_vstrl:
9335 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3)))
9336 if (C->getZExtValue() >= 15)
9337 return DAG.getStore(Chain: N->getOperand(Num: 0), dl: SDLoc(N), Val: N->getOperand(Num: 2),
9338 Ptr: N->getOperand(Num: 4), PtrInfo: MachinePointerInfo());
9339 break;
9340 }
9341
9342 return SDValue();
9343}
9344
9345SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
9346 if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
9347 return N->getOperand(Num: 0);
9348 return N;
9349}
9350
9351SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
9352 DAGCombinerInfo &DCI) const {
9353 switch(N->getOpcode()) {
9354 default: break;
9355 case ISD::ZERO_EXTEND: return combineZERO_EXTEND(N, DCI);
9356 case ISD::SIGN_EXTEND: return combineSIGN_EXTEND(N, DCI);
9357 case ISD::SIGN_EXTEND_INREG: return combineSIGN_EXTEND_INREG(N, DCI);
9358 case SystemZISD::MERGE_HIGH:
9359 case SystemZISD::MERGE_LOW: return combineMERGE(N, DCI);
9360 case ISD::LOAD: return combineLOAD(N, DCI);
9361 case ISD::STORE: return combineSTORE(N, DCI);
9362 case ISD::VECTOR_SHUFFLE: return combineVECTOR_SHUFFLE(N, DCI);
9363 case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI);
9364 case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI);
9365 case ISD::STRICT_FP_ROUND:
9366 case ISD::FP_ROUND: return combineFP_ROUND(N, DCI);
9367 case ISD::STRICT_FP_EXTEND:
9368 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DCI);
9369 case ISD::SINT_TO_FP:
9370 case ISD::UINT_TO_FP: return combineINT_TO_FP(N, DCI);
9371 case ISD::FCOPYSIGN: return combineFCOPYSIGN(N, DCI);
9372 case ISD::BSWAP: return combineBSWAP(N, DCI);
9373 case ISD::SETCC: return combineSETCC(N, DCI);
9374 case SystemZISD::BR_CCMASK: return combineBR_CCMASK(N, DCI);
9375 case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI);
9376 case SystemZISD::GET_CCMASK: return combineGET_CCMASK(N, DCI);
9377 case ISD::SRL:
9378 case ISD::SRA: return combineShiftToMulAddHigh(N, DCI);
9379 case ISD::MUL: return combineMUL(N, DCI);
9380 case ISD::SDIV:
9381 case ISD::UDIV:
9382 case ISD::SREM:
9383 case ISD::UREM: return combineIntDIVREM(N, DCI);
9384 case ISD::INTRINSIC_W_CHAIN:
9385 case ISD::INTRINSIC_VOID: return combineINTRINSIC(N, DCI);
9386 }
9387
9388 return SDValue();
9389}
9390
9391// Return the demanded elements for the OpNo source operand of Op. DemandedElts
9392// are for Op.
9393static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts,
9394 unsigned OpNo) {
9395 EVT VT = Op.getValueType();
9396 unsigned NumElts = (VT.isVector() ? VT.getVectorNumElements() : 1);
9397 APInt SrcDemE;
9398 unsigned Opcode = Op.getOpcode();
9399 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9400 unsigned Id = Op.getConstantOperandVal(i: 0);
9401 switch (Id) {
9402 case Intrinsic::s390_vpksh: // PACKS
9403 case Intrinsic::s390_vpksf:
9404 case Intrinsic::s390_vpksg:
9405 case Intrinsic::s390_vpkshs: // PACKS_CC
9406 case Intrinsic::s390_vpksfs:
9407 case Intrinsic::s390_vpksgs:
9408 case Intrinsic::s390_vpklsh: // PACKLS
9409 case Intrinsic::s390_vpklsf:
9410 case Intrinsic::s390_vpklsg:
9411 case Intrinsic::s390_vpklshs: // PACKLS_CC
9412 case Intrinsic::s390_vpklsfs:
9413 case Intrinsic::s390_vpklsgs:
9414 // VECTOR PACK truncates the elements of two source vectors into one.
9415 SrcDemE = DemandedElts;
9416 if (OpNo == 2)
9417 SrcDemE.lshrInPlace(ShiftAmt: NumElts / 2);
9418 SrcDemE = SrcDemE.trunc(width: NumElts / 2);
9419 break;
9420 // VECTOR UNPACK extends half the elements of the source vector.
9421 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9422 case Intrinsic::s390_vuphh:
9423 case Intrinsic::s390_vuphf:
9424 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9425 case Intrinsic::s390_vuplhh:
9426 case Intrinsic::s390_vuplhf:
9427 SrcDemE = APInt(NumElts * 2, 0);
9428 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: 0);
9429 break;
9430 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9431 case Intrinsic::s390_vuplhw:
9432 case Intrinsic::s390_vuplf:
9433 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9434 case Intrinsic::s390_vupllh:
9435 case Intrinsic::s390_vupllf:
9436 SrcDemE = APInt(NumElts * 2, 0);
9437 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: NumElts);
9438 break;
9439 case Intrinsic::s390_vpdi: {
9440 // VECTOR PERMUTE DWORD IMMEDIATE selects one element from each source.
9441 SrcDemE = APInt(NumElts, 0);
9442 if (!DemandedElts[OpNo - 1])
9443 break;
9444 unsigned Mask = Op.getConstantOperandVal(i: 3);
9445 unsigned MaskBit = ((OpNo - 1) ? 1 : 4);
9446 // Demand input element 0 or 1, given by the mask bit value.
9447 SrcDemE.setBit((Mask & MaskBit)? 1 : 0);
9448 break;
9449 }
9450 case Intrinsic::s390_vsldb: {
9451 // VECTOR SHIFT LEFT DOUBLE BY BYTE
9452 assert(VT == MVT::v16i8 && "Unexpected type.");
9453 unsigned FirstIdx = Op.getConstantOperandVal(i: 3);
9454 assert (FirstIdx > 0 && FirstIdx < 16 && "Unused operand.");
9455 unsigned NumSrc0Els = 16 - FirstIdx;
9456 SrcDemE = APInt(NumElts, 0);
9457 if (OpNo == 1) {
9458 APInt DemEls = DemandedElts.trunc(width: NumSrc0Els);
9459 SrcDemE.insertBits(SubBits: DemEls, bitPosition: FirstIdx);
9460 } else {
9461 APInt DemEls = DemandedElts.lshr(shiftAmt: NumSrc0Els);
9462 SrcDemE.insertBits(SubBits: DemEls, bitPosition: 0);
9463 }
9464 break;
9465 }
9466 case Intrinsic::s390_vperm:
9467 SrcDemE = APInt::getAllOnes(numBits: NumElts);
9468 break;
9469 default:
9470 llvm_unreachable("Unhandled intrinsic.");
9471 break;
9472 }
9473 } else {
9474 switch (Opcode) {
9475 case SystemZISD::JOIN_DWORDS:
9476 // Scalar operand.
9477 SrcDemE = APInt(1, 1);
9478 break;
9479 case SystemZISD::SELECT_CCMASK:
9480 SrcDemE = DemandedElts;
9481 break;
9482 default:
9483 llvm_unreachable("Unhandled opcode.");
9484 break;
9485 }
9486 }
9487 return SrcDemE;
9488}
9489
9490static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known,
9491 const APInt &DemandedElts,
9492 const SelectionDAG &DAG, unsigned Depth,
9493 unsigned OpNo) {
9494 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9495 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9496 KnownBits LHSKnown =
9497 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9498 KnownBits RHSKnown =
9499 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9500 Known = LHSKnown.intersectWith(RHS: RHSKnown);
9501}
9502
9503void
9504SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
9505 KnownBits &Known,
9506 const APInt &DemandedElts,
9507 const SelectionDAG &DAG,
9508 unsigned Depth) const {
9509 Known.resetAll();
9510
9511 // Intrinsic CC result is returned in the two low bits.
9512 unsigned Tmp0, Tmp1; // not used
9513 if (Op.getResNo() == 1 && isIntrinsicWithCC(Op, Opcode&: Tmp0, CCValid&: Tmp1)) {
9514 Known.Zero.setBitsFrom(2);
9515 return;
9516 }
9517 EVT VT = Op.getValueType();
9518 if (Op.getResNo() != 0 || VT == MVT::Untyped)
9519 return;
9520 assert (Known.getBitWidth() == VT.getScalarSizeInBits() &&
9521 "KnownBits does not match VT in bitwidth");
9522 assert ((!VT.isVector() ||
9523 (DemandedElts.getBitWidth() == VT.getVectorNumElements())) &&
9524 "DemandedElts does not match VT number of elements");
9525 unsigned BitWidth = Known.getBitWidth();
9526 unsigned Opcode = Op.getOpcode();
9527 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9528 bool IsLogical = false;
9529 unsigned Id = Op.getConstantOperandVal(i: 0);
9530 switch (Id) {
9531 case Intrinsic::s390_vpksh: // PACKS
9532 case Intrinsic::s390_vpksf:
9533 case Intrinsic::s390_vpksg:
9534 case Intrinsic::s390_vpkshs: // PACKS_CC
9535 case Intrinsic::s390_vpksfs:
9536 case Intrinsic::s390_vpksgs:
9537 case Intrinsic::s390_vpklsh: // PACKLS
9538 case Intrinsic::s390_vpklsf:
9539 case Intrinsic::s390_vpklsg:
9540 case Intrinsic::s390_vpklshs: // PACKLS_CC
9541 case Intrinsic::s390_vpklsfs:
9542 case Intrinsic::s390_vpklsgs:
9543 case Intrinsic::s390_vpdi:
9544 case Intrinsic::s390_vsldb:
9545 case Intrinsic::s390_vperm:
9546 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 1);
9547 break;
9548 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9549 case Intrinsic::s390_vuplhh:
9550 case Intrinsic::s390_vuplhf:
9551 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9552 case Intrinsic::s390_vupllh:
9553 case Intrinsic::s390_vupllf:
9554 IsLogical = true;
9555 [[fallthrough]];
9556 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9557 case Intrinsic::s390_vuphh:
9558 case Intrinsic::s390_vuphf:
9559 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9560 case Intrinsic::s390_vuplhw:
9561 case Intrinsic::s390_vuplf: {
9562 SDValue SrcOp = Op.getOperand(i: 1);
9563 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 0);
9564 Known = DAG.computeKnownBits(Op: SrcOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9565 if (IsLogical) {
9566 Known = Known.zext(BitWidth);
9567 } else
9568 Known = Known.sext(BitWidth);
9569 break;
9570 }
9571 default:
9572 break;
9573 }
9574 } else {
9575 switch (Opcode) {
9576 case SystemZISD::JOIN_DWORDS:
9577 case SystemZISD::SELECT_CCMASK:
9578 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 0);
9579 break;
9580 case SystemZISD::REPLICATE: {
9581 SDValue SrcOp = Op.getOperand(i: 0);
9582 Known = DAG.computeKnownBits(Op: SrcOp, Depth: Depth + 1);
9583 if (Known.getBitWidth() < BitWidth && isa<ConstantSDNode>(Val: SrcOp))
9584 Known = Known.sext(BitWidth); // VREPI sign extends the immedate.
9585 break;
9586 }
9587 default:
9588 break;
9589 }
9590 }
9591
9592 // Known has the width of the source operand(s). Adjust if needed to match
9593 // the passed bitwidth.
9594 if (Known.getBitWidth() != BitWidth)
9595 Known = Known.anyextOrTrunc(BitWidth);
9596}
9597
9598static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts,
9599 const SelectionDAG &DAG, unsigned Depth,
9600 unsigned OpNo) {
9601 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9602 unsigned LHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9603 if (LHS == 1) return 1; // Early out.
9604 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9605 unsigned RHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9606 if (RHS == 1) return 1; // Early out.
9607 unsigned Common = std::min(a: LHS, b: RHS);
9608 unsigned SrcBitWidth = Op.getOperand(i: OpNo).getScalarValueSizeInBits();
9609 EVT VT = Op.getValueType();
9610 unsigned VTBits = VT.getScalarSizeInBits();
9611 if (SrcBitWidth > VTBits) { // PACK
9612 unsigned SrcExtraBits = SrcBitWidth - VTBits;
9613 if (Common > SrcExtraBits)
9614 return (Common - SrcExtraBits);
9615 return 1;
9616 }
9617 assert (SrcBitWidth == VTBits && "Expected operands of same bitwidth.");
9618 return Common;
9619}
9620
9621unsigned
9622SystemZTargetLowering::ComputeNumSignBitsForTargetNode(
9623 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9624 unsigned Depth) const {
9625 if (Op.getResNo() != 0)
9626 return 1;
9627 unsigned Opcode = Op.getOpcode();
9628 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9629 unsigned Id = Op.getConstantOperandVal(i: 0);
9630 switch (Id) {
9631 case Intrinsic::s390_vpksh: // PACKS
9632 case Intrinsic::s390_vpksf:
9633 case Intrinsic::s390_vpksg:
9634 case Intrinsic::s390_vpkshs: // PACKS_CC
9635 case Intrinsic::s390_vpksfs:
9636 case Intrinsic::s390_vpksgs:
9637 case Intrinsic::s390_vpklsh: // PACKLS
9638 case Intrinsic::s390_vpklsf:
9639 case Intrinsic::s390_vpklsg:
9640 case Intrinsic::s390_vpklshs: // PACKLS_CC
9641 case Intrinsic::s390_vpklsfs:
9642 case Intrinsic::s390_vpklsgs:
9643 case Intrinsic::s390_vpdi:
9644 case Intrinsic::s390_vsldb:
9645 case Intrinsic::s390_vperm:
9646 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 1);
9647 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9648 case Intrinsic::s390_vuphh:
9649 case Intrinsic::s390_vuphf:
9650 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9651 case Intrinsic::s390_vuplhw:
9652 case Intrinsic::s390_vuplf: {
9653 SDValue PackedOp = Op.getOperand(i: 1);
9654 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 1);
9655 unsigned Tmp = DAG.ComputeNumSignBits(Op: PackedOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9656 EVT VT = Op.getValueType();
9657 unsigned VTBits = VT.getScalarSizeInBits();
9658 Tmp += VTBits - PackedOp.getScalarValueSizeInBits();
9659 return Tmp;
9660 }
9661 default:
9662 break;
9663 }
9664 } else {
9665 switch (Opcode) {
9666 case SystemZISD::SELECT_CCMASK:
9667 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 0);
9668 default:
9669 break;
9670 }
9671 }
9672
9673 return 1;
9674}
9675
9676bool SystemZTargetLowering::
9677isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op,
9678 const APInt &DemandedElts, const SelectionDAG &DAG,
9679 bool PoisonOnly, unsigned Depth) const {
9680 switch (Op->getOpcode()) {
9681 case SystemZISD::PCREL_WRAPPER:
9682 case SystemZISD::PCREL_OFFSET:
9683 return true;
9684 }
9685 return false;
9686}
9687
9688unsigned
9689SystemZTargetLowering::getStackProbeSize(const MachineFunction &MF) const {
9690 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
9691 unsigned StackAlign = TFI->getStackAlignment();
9692 assert(StackAlign >=1 && isPowerOf2_32(StackAlign) &&
9693 "Unexpected stack alignment");
9694 // The default stack probe size is 4096 if the function has no
9695 // stack-probe-size attribute.
9696 unsigned StackProbeSize =
9697 MF.getFunction().getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
9698 // Round down to the stack alignment.
9699 StackProbeSize &= ~(StackAlign - 1);
9700 return StackProbeSize ? StackProbeSize : StackAlign;
9701}
9702
9703//===----------------------------------------------------------------------===//
9704// Custom insertion
9705//===----------------------------------------------------------------------===//
9706
9707// Force base value Base into a register before MI. Return the register.
9708static Register forceReg(MachineInstr &MI, MachineOperand &Base,
9709 const SystemZInstrInfo *TII) {
9710 MachineBasicBlock *MBB = MI.getParent();
9711 MachineFunction &MF = *MBB->getParent();
9712 MachineRegisterInfo &MRI = MF.getRegInfo();
9713
9714 if (Base.isReg()) {
9715 // Copy Base into a new virtual register to help register coalescing in
9716 // cases with multiple uses.
9717 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9718 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::COPY), DestReg: Reg)
9719 .add(MO: Base);
9720 return Reg;
9721 }
9722
9723 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9724 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::LA), DestReg: Reg)
9725 .add(MO: Base)
9726 .addImm(Val: 0)
9727 .addReg(RegNo: 0);
9728 return Reg;
9729}
9730
9731// The CC operand of MI might be missing a kill marker because there
9732// were multiple uses of CC, and ISel didn't know which to mark.
9733// Figure out whether MI should have had a kill marker.
9734static bool checkCCKill(MachineInstr &MI, MachineBasicBlock *MBB) {
9735 // Scan forward through BB for a use/def of CC.
9736 MachineBasicBlock::iterator miI(std::next(x: MachineBasicBlock::iterator(MI)));
9737 for (MachineBasicBlock::iterator miE = MBB->end(); miI != miE; ++miI) {
9738 const MachineInstr &MI = *miI;
9739 if (MI.readsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9740 return false;
9741 if (MI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9742 break; // Should have kill-flag - update below.
9743 }
9744
9745 // If we hit the end of the block, check whether CC is live into a
9746 // successor.
9747 if (miI == MBB->end()) {
9748 for (const MachineBasicBlock *Succ : MBB->successors())
9749 if (Succ->isLiveIn(Reg: SystemZ::CC))
9750 return false;
9751 }
9752
9753 return true;
9754}
9755
9756// Return true if it is OK for this Select pseudo-opcode to be cascaded
9757// together with other Select pseudo-opcodes into a single basic-block with
9758// a conditional jump around it.
9759static bool isSelectPseudo(MachineInstr &MI) {
9760 switch (MI.getOpcode()) {
9761 case SystemZ::Select32:
9762 case SystemZ::Select64:
9763 case SystemZ::Select128:
9764 case SystemZ::SelectF32:
9765 case SystemZ::SelectF64:
9766 case SystemZ::SelectF128:
9767 case SystemZ::SelectVR32:
9768 case SystemZ::SelectVR64:
9769 case SystemZ::SelectVR128:
9770 return true;
9771
9772 default:
9773 return false;
9774 }
9775}
9776
9777// Helper function, which inserts PHI functions into SinkMBB:
9778// %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
9779// where %FalseValue(i) and %TrueValue(i) are taken from Selects.
9780static void createPHIsForSelects(SmallVector<MachineInstr*, 8> &Selects,
9781 MachineBasicBlock *TrueMBB,
9782 MachineBasicBlock *FalseMBB,
9783 MachineBasicBlock *SinkMBB) {
9784 MachineFunction *MF = TrueMBB->getParent();
9785 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
9786
9787 MachineInstr *FirstMI = Selects.front();
9788 unsigned CCValid = FirstMI->getOperand(i: 3).getImm();
9789 unsigned CCMask = FirstMI->getOperand(i: 4).getImm();
9790
9791 MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
9792
9793 // As we are creating the PHIs, we have to be careful if there is more than
9794 // one. Later Selects may reference the results of earlier Selects, but later
9795 // PHIs have to reference the individual true/false inputs from earlier PHIs.
9796 // That also means that PHI construction must work forward from earlier to
9797 // later, and that the code must maintain a mapping from earlier PHI's
9798 // destination registers, and the registers that went into the PHI.
9799 DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
9800
9801 for (auto *MI : Selects) {
9802 Register DestReg = MI->getOperand(i: 0).getReg();
9803 Register TrueReg = MI->getOperand(i: 1).getReg();
9804 Register FalseReg = MI->getOperand(i: 2).getReg();
9805
9806 // If this Select we are generating is the opposite condition from
9807 // the jump we generated, then we have to swap the operands for the
9808 // PHI that is going to be generated.
9809 if (MI->getOperand(i: 4).getImm() == (CCValid ^ CCMask))
9810 std::swap(a&: TrueReg, b&: FalseReg);
9811
9812 if (auto It = RegRewriteTable.find(Val: TrueReg); It != RegRewriteTable.end())
9813 TrueReg = It->second.first;
9814
9815 if (auto It = RegRewriteTable.find(Val: FalseReg); It != RegRewriteTable.end())
9816 FalseReg = It->second.second;
9817
9818 DebugLoc DL = MI->getDebugLoc();
9819 BuildMI(BB&: *SinkMBB, I: SinkInsertionPoint, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg)
9820 .addReg(RegNo: TrueReg).addMBB(MBB: TrueMBB)
9821 .addReg(RegNo: FalseReg).addMBB(MBB: FalseMBB);
9822
9823 // Add this PHI to the rewrite table.
9824 RegRewriteTable[DestReg] = std::make_pair(x&: TrueReg, y&: FalseReg);
9825 }
9826
9827 MF->getProperties().resetNoPHIs();
9828}
9829
9830MachineBasicBlock *
9831SystemZTargetLowering::emitAdjCallStack(MachineInstr &MI,
9832 MachineBasicBlock *BB) const {
9833 MachineFunction &MF = *BB->getParent();
9834 MachineFrameInfo &MFI = MF.getFrameInfo();
9835 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
9836 assert(TFL->hasReservedCallFrame(MF) &&
9837 "ADJSTACKDOWN and ADJSTACKUP should be no-ops");
9838 (void)TFL;
9839 // Get the MaxCallFrameSize value and erase MI since it serves no further
9840 // purpose as the call frame is statically reserved in the prolog. Set
9841 // AdjustsStack as MI is *not* mapped as a frame instruction.
9842 uint32_t NumBytes = MI.getOperand(i: 0).getImm();
9843 if (NumBytes > MFI.getMaxCallFrameSize())
9844 MFI.setMaxCallFrameSize(NumBytes);
9845 MFI.setAdjustsStack(true);
9846
9847 MI.eraseFromParent();
9848 return BB;
9849}
9850
9851// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI.
9852MachineBasicBlock *
9853SystemZTargetLowering::emitSelect(MachineInstr &MI,
9854 MachineBasicBlock *MBB) const {
9855 assert(isSelectPseudo(MI) && "Bad call to emitSelect()");
9856 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9857
9858 unsigned CCValid = MI.getOperand(i: 3).getImm();
9859 unsigned CCMask = MI.getOperand(i: 4).getImm();
9860
9861 // If we have a sequence of Select* pseudo instructions using the
9862 // same condition code value, we want to expand all of them into
9863 // a single pair of basic blocks using the same condition.
9864 SmallVector<MachineInstr*, 8> Selects;
9865 SmallVector<MachineInstr*, 8> DbgValues;
9866 Selects.push_back(Elt: &MI);
9867 unsigned Count = 0;
9868 for (MachineInstr &NextMI : llvm::make_range(
9869 x: std::next(x: MachineBasicBlock::iterator(MI)), y: MBB->end())) {
9870 if (isSelectPseudo(MI&: NextMI)) {
9871 assert(NextMI.getOperand(3).getImm() == CCValid &&
9872 "Bad CCValid operands since CC was not redefined.");
9873 if (NextMI.getOperand(i: 4).getImm() == CCMask ||
9874 NextMI.getOperand(i: 4).getImm() == (CCValid ^ CCMask)) {
9875 Selects.push_back(Elt: &NextMI);
9876 continue;
9877 }
9878 break;
9879 }
9880 if (NextMI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
9881 NextMI.usesCustomInsertionHook())
9882 break;
9883 bool User = false;
9884 for (auto *SelMI : Selects)
9885 if (NextMI.readsVirtualRegister(Reg: SelMI->getOperand(i: 0).getReg())) {
9886 User = true;
9887 break;
9888 }
9889 if (NextMI.isDebugInstr()) {
9890 if (User) {
9891 assert(NextMI.isDebugValue() && "Unhandled debug opcode.");
9892 DbgValues.push_back(Elt: &NextMI);
9893 }
9894 } else if (User || ++Count > 20)
9895 break;
9896 }
9897
9898 MachineInstr *LastMI = Selects.back();
9899 bool CCKilled = (LastMI->killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
9900 checkCCKill(MI&: *LastMI, MBB));
9901 MachineBasicBlock *StartMBB = MBB;
9902 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI: LastMI, MBB);
9903 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
9904
9905 // Unless CC was killed in the last Select instruction, mark it as
9906 // live-in to both FalseMBB and JoinMBB.
9907 if (!CCKilled) {
9908 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
9909 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
9910 }
9911
9912 // StartMBB:
9913 // BRC CCMask, JoinMBB
9914 // # fallthrough to FalseMBB
9915 MBB = StartMBB;
9916 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
9917 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
9918 MBB->addSuccessor(Succ: JoinMBB);
9919 MBB->addSuccessor(Succ: FalseMBB);
9920
9921 // FalseMBB:
9922 // # fallthrough to JoinMBB
9923 MBB = FalseMBB;
9924 MBB->addSuccessor(Succ: JoinMBB);
9925
9926 // JoinMBB:
9927 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ]
9928 // ...
9929 MBB = JoinMBB;
9930 createPHIsForSelects(Selects, TrueMBB: StartMBB, FalseMBB, SinkMBB: MBB);
9931 for (auto *SelMI : Selects)
9932 SelMI->eraseFromParent();
9933
9934 MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();
9935 for (auto *DbgMI : DbgValues)
9936 MBB->splice(Where: InsertPos, Other: StartMBB, From: DbgMI);
9937
9938 return JoinMBB;
9939}
9940
9941// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI.
9942// StoreOpcode is the store to use and Invert says whether the store should
9943// happen when the condition is false rather than true. If a STORE ON
9944// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0.
9945MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI,
9946 MachineBasicBlock *MBB,
9947 unsigned StoreOpcode,
9948 unsigned STOCOpcode,
9949 bool Invert) const {
9950 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9951
9952 Register SrcReg = MI.getOperand(i: 0).getReg();
9953 MachineOperand Base = MI.getOperand(i: 1);
9954 int64_t Disp = MI.getOperand(i: 2).getImm();
9955 Register IndexReg = MI.getOperand(i: 3).getReg();
9956 unsigned CCValid = MI.getOperand(i: 4).getImm();
9957 unsigned CCMask = MI.getOperand(i: 5).getImm();
9958 DebugLoc DL = MI.getDebugLoc();
9959
9960 StoreOpcode = TII->getOpcodeForOffset(Opcode: StoreOpcode, Offset: Disp);
9961
9962 // ISel pattern matching also adds a load memory operand of the same
9963 // address, so take special care to find the storing memory operand.
9964 MachineMemOperand *MMO = nullptr;
9965 for (auto *I : MI.memoperands())
9966 if (I->isStore()) {
9967 MMO = I;
9968 break;
9969 }
9970
9971 // Use STOCOpcode if possible. We could use different store patterns in
9972 // order to avoid matching the index register, but the performance trade-offs
9973 // might be more complicated in that case.
9974 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
9975 if (Invert)
9976 CCMask ^= CCValid;
9977
9978 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: STOCOpcode))
9979 .addReg(RegNo: SrcReg)
9980 .add(MO: Base)
9981 .addImm(Val: Disp)
9982 .addImm(Val: CCValid)
9983 .addImm(Val: CCMask)
9984 .addMemOperand(MMO);
9985
9986 MI.eraseFromParent();
9987 return MBB;
9988 }
9989
9990 // Get the condition needed to branch around the store.
9991 if (!Invert)
9992 CCMask ^= CCValid;
9993
9994 MachineBasicBlock *StartMBB = MBB;
9995 MachineBasicBlock *JoinMBB = SystemZ::splitBlockBefore(MI, MBB);
9996 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
9997
9998 // Unless CC was killed in the CondStore instruction, mark it as
9999 // live-in to both FalseMBB and JoinMBB.
10000 if (!MI.killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) &&
10001 !checkCCKill(MI, MBB: JoinMBB)) {
10002 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
10003 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10004 }
10005
10006 // StartMBB:
10007 // BRC CCMask, JoinMBB
10008 // # fallthrough to FalseMBB
10009 MBB = StartMBB;
10010 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10011 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
10012 MBB->addSuccessor(Succ: JoinMBB);
10013 MBB->addSuccessor(Succ: FalseMBB);
10014
10015 // FalseMBB:
10016 // store %SrcReg, %Disp(%Index,%Base)
10017 // # fallthrough to JoinMBB
10018 MBB = FalseMBB;
10019 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: StoreOpcode))
10020 .addReg(RegNo: SrcReg)
10021 .add(MO: Base)
10022 .addImm(Val: Disp)
10023 .addReg(RegNo: IndexReg)
10024 .addMemOperand(MMO);
10025 MBB->addSuccessor(Succ: JoinMBB);
10026
10027 MI.eraseFromParent();
10028 return JoinMBB;
10029}
10030
10031// Implement EmitInstrWithCustomInserter for pseudo [SU]Cmp128Hi instruction MI.
10032MachineBasicBlock *
10033SystemZTargetLowering::emitICmp128Hi(MachineInstr &MI,
10034 MachineBasicBlock *MBB,
10035 bool Unsigned) const {
10036 MachineFunction &MF = *MBB->getParent();
10037 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10038 MachineRegisterInfo &MRI = MF.getRegInfo();
10039
10040 // Synthetic instruction to compare 128-bit values.
10041 // Sets CC 1 if Op0 > Op1, sets a different CC otherwise.
10042 Register Op0 = MI.getOperand(i: 0).getReg();
10043 Register Op1 = MI.getOperand(i: 1).getReg();
10044
10045 MachineBasicBlock *StartMBB = MBB;
10046 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI, MBB);
10047 MachineBasicBlock *HiEqMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10048
10049 // StartMBB:
10050 //
10051 // Use VECTOR ELEMENT COMPARE [LOGICAL] to compare the high parts.
10052 // Swap the inputs to get:
10053 // CC 1 if high(Op0) > high(Op1)
10054 // CC 2 if high(Op0) < high(Op1)
10055 // CC 0 if high(Op0) == high(Op1)
10056 //
10057 // If CC != 0, we'd done, so jump over the next instruction.
10058 //
10059 // VEC[L]G Op1, Op0
10060 // JNE JoinMBB
10061 // # fallthrough to HiEqMBB
10062 MBB = StartMBB;
10063 int HiOpcode = Unsigned? SystemZ::VECLG : SystemZ::VECG;
10064 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: HiOpcode))
10065 .addReg(RegNo: Op1).addReg(RegNo: Op0);
10066 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
10067 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: JoinMBB);
10068 MBB->addSuccessor(Succ: JoinMBB);
10069 MBB->addSuccessor(Succ: HiEqMBB);
10070
10071 // HiEqMBB:
10072 //
10073 // Otherwise, use VECTOR COMPARE HIGH LOGICAL.
10074 // Since we already know the high parts are equal, the CC
10075 // result will only depend on the low parts:
10076 // CC 1 if low(Op0) > low(Op1)
10077 // CC 3 if low(Op0) <= low(Op1)
10078 //
10079 // VCHLGS Tmp, Op0, Op1
10080 // # fallthrough to JoinMBB
10081 MBB = HiEqMBB;
10082 Register Temp = MRI.createVirtualRegister(RegClass: &SystemZ::VR128BitRegClass);
10083 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::VCHLGS), DestReg: Temp)
10084 .addReg(RegNo: Op0).addReg(RegNo: Op1);
10085 MBB->addSuccessor(Succ: JoinMBB);
10086
10087 // Mark CC as live-in to JoinMBB.
10088 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10089
10090 MI.eraseFromParent();
10091 return JoinMBB;
10092}
10093
10094// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_LOADW_* or
10095// ATOMIC_SWAPW instruction MI. BinOpcode is the instruction that performs
10096// the binary operation elided by "*", or 0 for ATOMIC_SWAPW. Invert says
10097// whether the field should be inverted after performing BinOpcode (e.g. for
10098// NAND).
10099MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
10100 MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode,
10101 bool Invert) const {
10102 MachineFunction &MF = *MBB->getParent();
10103 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10104 MachineRegisterInfo &MRI = MF.getRegInfo();
10105
10106 // Extract the operands. Base can be a register or a frame index.
10107 // Src2 can be a register or immediate.
10108 Register Dest = MI.getOperand(i: 0).getReg();
10109 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10110 int64_t Disp = MI.getOperand(i: 2).getImm();
10111 MachineOperand Src2 = earlyUseOperand(Op: MI.getOperand(i: 3));
10112 Register BitShift = MI.getOperand(i: 4).getReg();
10113 Register NegBitShift = MI.getOperand(i: 5).getReg();
10114 unsigned BitSize = MI.getOperand(i: 6).getImm();
10115 DebugLoc DL = MI.getDebugLoc();
10116
10117 // Get the right opcodes for the displacement.
10118 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10119 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10120 assert(LOpcode && CSOpcode && "Displacement out of range");
10121
10122 // Create virtual registers for temporary results.
10123 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10124 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10125 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10126 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10127 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10128
10129 // Insert a basic block for the main loop.
10130 MachineBasicBlock *StartMBB = MBB;
10131 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10132 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10133
10134 // StartMBB:
10135 // ...
10136 // %OrigVal = L Disp(%Base)
10137 // # fall through to LoopMBB
10138 MBB = StartMBB;
10139 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10140 MBB->addSuccessor(Succ: LoopMBB);
10141
10142 // LoopMBB:
10143 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ]
10144 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10145 // %RotatedNewVal = OP %RotatedOldVal, %Src2
10146 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10147 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10148 // JNE LoopMBB
10149 // # fall through to DoneMBB
10150 MBB = LoopMBB;
10151 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10152 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
10153 .addReg(RegNo: Dest).addMBB(MBB: LoopMBB);
10154 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
10155 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
10156 if (Invert) {
10157 // Perform the operation normally and then invert every bit of the field.
10158 Register Tmp = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10159 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: Tmp).addReg(RegNo: RotatedOldVal).add(MO: Src2);
10160 // XILF with the upper BitSize bits set.
10161 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::XILF), DestReg: RotatedNewVal)
10162 .addReg(RegNo: Tmp).addImm(Val: -1U << (32 - BitSize));
10163 } else if (BinOpcode)
10164 // A simply binary operation.
10165 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: RotatedNewVal)
10166 .addReg(RegNo: RotatedOldVal)
10167 .add(MO: Src2);
10168 else
10169 // Use RISBG to rotate Src2 into position and use it to replace the
10170 // field in RotatedOldVal.
10171 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedNewVal)
10172 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2.getReg())
10173 .addImm(Val: 32).addImm(Val: 31 + BitSize).addImm(Val: 32 - BitSize);
10174 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
10175 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
10176 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
10177 .addReg(RegNo: OldVal)
10178 .addReg(RegNo: NewVal)
10179 .add(MO: Base)
10180 .addImm(Val: Disp);
10181 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10182 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10183 MBB->addSuccessor(Succ: LoopMBB);
10184 MBB->addSuccessor(Succ: DoneMBB);
10185
10186 MI.eraseFromParent();
10187 return DoneMBB;
10188}
10189
10190// Implement EmitInstrWithCustomInserter for subword pseudo
10191// ATOMIC_LOADW_{,U}{MIN,MAX} instruction MI. CompareOpcode is the
10192// instruction that should be used to compare the current field with the
10193// minimum or maximum value. KeepOldMask is the BRC condition-code mask
10194// for when the current field should be kept.
10195MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
10196 MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode,
10197 unsigned KeepOldMask) const {
10198 MachineFunction &MF = *MBB->getParent();
10199 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10200 MachineRegisterInfo &MRI = MF.getRegInfo();
10201
10202 // Extract the operands. Base can be a register or a frame index.
10203 Register Dest = MI.getOperand(i: 0).getReg();
10204 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10205 int64_t Disp = MI.getOperand(i: 2).getImm();
10206 Register Src2 = MI.getOperand(i: 3).getReg();
10207 Register BitShift = MI.getOperand(i: 4).getReg();
10208 Register NegBitShift = MI.getOperand(i: 5).getReg();
10209 unsigned BitSize = MI.getOperand(i: 6).getImm();
10210 DebugLoc DL = MI.getDebugLoc();
10211
10212 // Get the right opcodes for the displacement.
10213 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10214 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10215 assert(LOpcode && CSOpcode && "Displacement out of range");
10216
10217 // Create virtual registers for temporary results.
10218 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10219 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10220 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10221 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10222 Register RotatedAltVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10223 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10224
10225 // Insert 3 basic blocks for the loop.
10226 MachineBasicBlock *StartMBB = MBB;
10227 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10228 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10229 MachineBasicBlock *UseAltMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10230 MachineBasicBlock *UpdateMBB = SystemZ::emitBlockAfter(MBB: UseAltMBB);
10231
10232 // StartMBB:
10233 // ...
10234 // %OrigVal = L Disp(%Base)
10235 // # fall through to LoopMBB
10236 MBB = StartMBB;
10237 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10238 MBB->addSuccessor(Succ: LoopMBB);
10239
10240 // LoopMBB:
10241 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ]
10242 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10243 // CompareOpcode %RotatedOldVal, %Src2
10244 // BRC KeepOldMask, UpdateMBB
10245 MBB = LoopMBB;
10246 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10247 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
10248 .addReg(RegNo: Dest).addMBB(MBB: UpdateMBB);
10249 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
10250 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
10251 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CompareOpcode))
10252 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2);
10253 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10254 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: KeepOldMask).addMBB(MBB: UpdateMBB);
10255 MBB->addSuccessor(Succ: UpdateMBB);
10256 MBB->addSuccessor(Succ: UseAltMBB);
10257
10258 // UseAltMBB:
10259 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0
10260 // # fall through to UpdateMBB
10261 MBB = UseAltMBB;
10262 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedAltVal)
10263 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2)
10264 .addImm(Val: 32).addImm(Val: 31 + BitSize).addImm(Val: 0);
10265 MBB->addSuccessor(Succ: UpdateMBB);
10266
10267 // UpdateMBB:
10268 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ],
10269 // [ %RotatedAltVal, UseAltMBB ]
10270 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10271 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10272 // JNE LoopMBB
10273 // # fall through to DoneMBB
10274 MBB = UpdateMBB;
10275 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RotatedNewVal)
10276 .addReg(RegNo: RotatedOldVal).addMBB(MBB: LoopMBB)
10277 .addReg(RegNo: RotatedAltVal).addMBB(MBB: UseAltMBB);
10278 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
10279 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
10280 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
10281 .addReg(RegNo: OldVal)
10282 .addReg(RegNo: NewVal)
10283 .add(MO: Base)
10284 .addImm(Val: Disp);
10285 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10286 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10287 MBB->addSuccessor(Succ: LoopMBB);
10288 MBB->addSuccessor(Succ: DoneMBB);
10289
10290 MI.eraseFromParent();
10291 return DoneMBB;
10292}
10293
10294// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_CMP_SWAPW
10295// instruction MI.
10296MachineBasicBlock *
10297SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
10298 MachineBasicBlock *MBB) const {
10299 MachineFunction &MF = *MBB->getParent();
10300 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10301 MachineRegisterInfo &MRI = MF.getRegInfo();
10302
10303 // Extract the operands. Base can be a register or a frame index.
10304 Register Dest = MI.getOperand(i: 0).getReg();
10305 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10306 int64_t Disp = MI.getOperand(i: 2).getImm();
10307 Register CmpVal = MI.getOperand(i: 3).getReg();
10308 Register OrigSwapVal = MI.getOperand(i: 4).getReg();
10309 Register BitShift = MI.getOperand(i: 5).getReg();
10310 Register NegBitShift = MI.getOperand(i: 6).getReg();
10311 int64_t BitSize = MI.getOperand(i: 7).getImm();
10312 DebugLoc DL = MI.getDebugLoc();
10313
10314 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
10315
10316 // Get the right opcodes for the displacement and zero-extension.
10317 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10318 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10319 unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
10320 assert(LOpcode && CSOpcode && "Displacement out of range");
10321
10322 // Create virtual registers for temporary results.
10323 Register OrigOldVal = MRI.createVirtualRegister(RegClass: RC);
10324 Register OldVal = MRI.createVirtualRegister(RegClass: RC);
10325 Register SwapVal = MRI.createVirtualRegister(RegClass: RC);
10326 Register StoreVal = MRI.createVirtualRegister(RegClass: RC);
10327 Register OldValRot = MRI.createVirtualRegister(RegClass: RC);
10328 Register RetryOldVal = MRI.createVirtualRegister(RegClass: RC);
10329 Register RetrySwapVal = MRI.createVirtualRegister(RegClass: RC);
10330
10331 // Insert 2 basic blocks for the loop.
10332 MachineBasicBlock *StartMBB = MBB;
10333 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10334 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10335 MachineBasicBlock *SetMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10336
10337 // StartMBB:
10338 // ...
10339 // %OrigOldVal = L Disp(%Base)
10340 // # fall through to LoopMBB
10341 MBB = StartMBB;
10342 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigOldVal)
10343 .add(MO: Base)
10344 .addImm(Val: Disp)
10345 .addReg(RegNo: 0);
10346 MBB->addSuccessor(Succ: LoopMBB);
10347
10348 // LoopMBB:
10349 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
10350 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
10351 // %OldValRot = RLL %OldVal, BitSize(%BitShift)
10352 // ^^ The low BitSize bits contain the field
10353 // of interest.
10354 // %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
10355 // ^^ Replace the upper 32-BitSize bits of the
10356 // swap value with those that we loaded and rotated.
10357 // %Dest = LL[CH] %OldValRot
10358 // CR %Dest, %CmpVal
10359 // JNE DoneMBB
10360 // # Fall through to SetMBB
10361 MBB = LoopMBB;
10362 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10363 .addReg(RegNo: OrigOldVal).addMBB(MBB: StartMBB)
10364 .addReg(RegNo: RetryOldVal).addMBB(MBB: SetMBB);
10365 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: SwapVal)
10366 .addReg(RegNo: OrigSwapVal).addMBB(MBB: StartMBB)
10367 .addReg(RegNo: RetrySwapVal).addMBB(MBB: SetMBB);
10368 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: OldValRot)
10369 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: BitSize);
10370 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RetrySwapVal)
10371 .addReg(RegNo: SwapVal).addReg(RegNo: OldValRot).addImm(Val: 32).addImm(Val: 63 - BitSize).addImm(Val: 0);
10372 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: ZExtOpcode), DestReg: Dest)
10373 .addReg(RegNo: OldValRot);
10374 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CR))
10375 .addReg(RegNo: Dest).addReg(RegNo: CmpVal);
10376 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10377 .addImm(Val: SystemZ::CCMASK_ICMP)
10378 .addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: DoneMBB);
10379 MBB->addSuccessor(Succ: DoneMBB);
10380 MBB->addSuccessor(Succ: SetMBB);
10381
10382 // SetMBB:
10383 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
10384 // ^^ Rotate the new field to its proper position.
10385 // %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
10386 // JNE LoopMBB
10387 // # fall through to ExitMBB
10388 MBB = SetMBB;
10389 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: StoreVal)
10390 .addReg(RegNo: RetrySwapVal).addReg(RegNo: NegBitShift).addImm(Val: -BitSize);
10391 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: RetryOldVal)
10392 .addReg(RegNo: OldVal)
10393 .addReg(RegNo: StoreVal)
10394 .add(MO: Base)
10395 .addImm(Val: Disp);
10396 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10397 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10398 MBB->addSuccessor(Succ: LoopMBB);
10399 MBB->addSuccessor(Succ: DoneMBB);
10400
10401 // If the CC def wasn't dead in the ATOMIC_CMP_SWAPW, mark CC as live-in
10402 // to the block after the loop. At this point, CC may have been defined
10403 // either by the CR in LoopMBB or by the CS in SetMBB.
10404 if (!MI.registerDefIsDead(Reg: SystemZ::CC, /*TRI=*/nullptr))
10405 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10406
10407 MI.eraseFromParent();
10408 return DoneMBB;
10409}
10410
10411// Emit a move from two GR64s to a GR128.
10412MachineBasicBlock *
10413SystemZTargetLowering::emitPair128(MachineInstr &MI,
10414 MachineBasicBlock *MBB) const {
10415 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10416 const DebugLoc &DL = MI.getDebugLoc();
10417
10418 Register Dest = MI.getOperand(i: 0).getReg();
10419 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest)
10420 .add(MO: MI.getOperand(i: 1))
10421 .addImm(Val: SystemZ::subreg_h64)
10422 .add(MO: MI.getOperand(i: 2))
10423 .addImm(Val: SystemZ::subreg_l64);
10424 MI.eraseFromParent();
10425 return MBB;
10426}
10427
10428// Emit an extension from a GR64 to a GR128. ClearEven is true
10429// if the high register of the GR128 value must be cleared or false if
10430// it's "don't care".
10431MachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI,
10432 MachineBasicBlock *MBB,
10433 bool ClearEven) const {
10434 MachineFunction &MF = *MBB->getParent();
10435 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10436 MachineRegisterInfo &MRI = MF.getRegInfo();
10437 DebugLoc DL = MI.getDebugLoc();
10438
10439 Register Dest = MI.getOperand(i: 0).getReg();
10440 Register Src = MI.getOperand(i: 1).getReg();
10441 Register In128 = MRI.createVirtualRegister(RegClass: &SystemZ::GR128BitRegClass);
10442
10443 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: In128);
10444 if (ClearEven) {
10445 Register NewIn128 = MRI.createVirtualRegister(RegClass: &SystemZ::GR128BitRegClass);
10446 Register Zero64 = MRI.createVirtualRegister(RegClass: &SystemZ::GR64BitRegClass);
10447
10448 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LLILL), DestReg: Zero64)
10449 .addImm(Val: 0);
10450 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: NewIn128)
10451 .addReg(RegNo: In128).addReg(RegNo: Zero64).addImm(Val: SystemZ::subreg_h64);
10452 In128 = NewIn128;
10453 }
10454 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Dest)
10455 .addReg(RegNo: In128).addReg(RegNo: Src).addImm(Val: SystemZ::subreg_l64);
10456
10457 MI.eraseFromParent();
10458 return MBB;
10459}
10460
10461MachineBasicBlock *
10462SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
10463 MachineBasicBlock *MBB,
10464 unsigned Opcode, bool IsMemset) const {
10465 MachineFunction &MF = *MBB->getParent();
10466 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10467 MachineRegisterInfo &MRI = MF.getRegInfo();
10468 DebugLoc DL = MI.getDebugLoc();
10469
10470 MachineOperand DestBase = earlyUseOperand(Op: MI.getOperand(i: 0));
10471 uint64_t DestDisp = MI.getOperand(i: 1).getImm();
10472 MachineOperand SrcBase = MachineOperand::CreateReg(Reg: 0U, isDef: false);
10473 uint64_t SrcDisp;
10474
10475 // Fold the displacement Disp if it is out of range.
10476 auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
10477 if (!isUInt<12>(x: Disp)) {
10478 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10479 unsigned Opcode = TII->getOpcodeForOffset(Opcode: SystemZ::LA, Offset: Disp);
10480 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode), DestReg: Reg)
10481 .add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10482 Base = MachineOperand::CreateReg(Reg, isDef: false);
10483 Disp = 0;
10484 }
10485 };
10486
10487 if (!IsMemset) {
10488 SrcBase = earlyUseOperand(Op: MI.getOperand(i: 2));
10489 SrcDisp = MI.getOperand(i: 3).getImm();
10490 } else {
10491 SrcBase = DestBase;
10492 SrcDisp = DestDisp++;
10493 foldDisplIfNeeded(DestBase, DestDisp);
10494 }
10495
10496 MachineOperand &LengthMO = MI.getOperand(i: IsMemset ? 2 : 4);
10497 bool IsImmForm = LengthMO.isImm();
10498 bool IsRegForm = !IsImmForm;
10499
10500 // Build and insert one Opcode of Length, with special treatment for memset.
10501 auto insertMemMemOp = [&](MachineBasicBlock *InsMBB,
10502 MachineBasicBlock::iterator InsPos,
10503 MachineOperand DBase, uint64_t DDisp,
10504 MachineOperand SBase, uint64_t SDisp,
10505 unsigned Length) -> void {
10506 assert(Length > 0 && Length <= 256 && "Building memory op with bad length.");
10507 if (IsMemset) {
10508 MachineOperand ByteMO = earlyUseOperand(Op: MI.getOperand(i: 3));
10509 if (ByteMO.isImm())
10510 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVI))
10511 .add(MO: SBase).addImm(Val: SDisp).add(MO: ByteMO);
10512 else
10513 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STC))
10514 .add(MO: ByteMO).add(MO: SBase).addImm(Val: SDisp).addReg(RegNo: 0);
10515 if (--Length == 0)
10516 return;
10517 }
10518 BuildMI(BB&: *MBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode))
10519 .add(MO: DBase).addImm(Val: DDisp).addImm(Val: Length)
10520 .add(MO: SBase).addImm(Val: SDisp)
10521 .setMemRefs(MI.memoperands());
10522 };
10523
10524 bool NeedsLoop = false;
10525 uint64_t ImmLength = 0;
10526 Register LenAdjReg = SystemZ::NoRegister;
10527 if (IsImmForm) {
10528 ImmLength = LengthMO.getImm();
10529 ImmLength += IsMemset ? 2 : 1; // Add back the subtracted adjustment.
10530 if (ImmLength == 0) {
10531 MI.eraseFromParent();
10532 return MBB;
10533 }
10534 if (Opcode == SystemZ::CLC) {
10535 if (ImmLength > 3 * 256)
10536 // A two-CLC sequence is a clear win over a loop, not least because
10537 // it needs only one branch. A three-CLC sequence needs the same
10538 // number of branches as a loop (i.e. 2), but is shorter. That
10539 // brings us to lengths greater than 768 bytes. It seems relatively
10540 // likely that a difference will be found within the first 768 bytes,
10541 // so we just optimize for the smallest number of branch
10542 // instructions, in order to avoid polluting the prediction buffer
10543 // too much.
10544 NeedsLoop = true;
10545 } else if (ImmLength > 6 * 256)
10546 // The heuristic we use is to prefer loops for anything that would
10547 // require 7 or more MVCs. With these kinds of sizes there isn't much
10548 // to choose between straight-line code and looping code, since the
10549 // time will be dominated by the MVCs themselves.
10550 NeedsLoop = true;
10551 } else {
10552 NeedsLoop = true;
10553 LenAdjReg = LengthMO.getReg();
10554 }
10555
10556 // When generating more than one CLC, all but the last will need to
10557 // branch to the end when a difference is found.
10558 MachineBasicBlock *EndMBB =
10559 (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
10560 ? SystemZ::splitBlockAfter(MI, MBB)
10561 : nullptr);
10562
10563 if (NeedsLoop) {
10564 Register StartCountReg =
10565 MRI.createVirtualRegister(RegClass: &SystemZ::GR64BitRegClass);
10566 if (IsImmForm) {
10567 TII->loadImmediate(MBB&: *MBB, MBBI: MI, Reg: StartCountReg, Value: ImmLength / 256);
10568 ImmLength &= 255;
10569 } else {
10570 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SRLG), DestReg: StartCountReg)
10571 .addReg(RegNo: LenAdjReg)
10572 .addReg(RegNo: 0)
10573 .addImm(Val: 8);
10574 }
10575
10576 bool HaveSingleBase = DestBase.isIdenticalTo(Other: SrcBase);
10577 auto loadZeroAddress = [&]() -> MachineOperand {
10578 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10579 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LGHI), DestReg: Reg).addImm(Val: 0);
10580 return MachineOperand::CreateReg(Reg, isDef: false);
10581 };
10582 if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
10583 DestBase = loadZeroAddress();
10584 if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
10585 SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
10586
10587 MachineBasicBlock *StartMBB = nullptr;
10588 MachineBasicBlock *LoopMBB = nullptr;
10589 MachineBasicBlock *NextMBB = nullptr;
10590 MachineBasicBlock *DoneMBB = nullptr;
10591 MachineBasicBlock *AllDoneMBB = nullptr;
10592
10593 Register StartSrcReg = forceReg(MI, Base&: SrcBase, TII);
10594 Register StartDestReg =
10595 (HaveSingleBase ? StartSrcReg : forceReg(MI, Base&: DestBase, TII));
10596
10597 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
10598 Register ThisSrcReg = MRI.createVirtualRegister(RegClass: RC);
10599 Register ThisDestReg =
10600 (HaveSingleBase ? ThisSrcReg : MRI.createVirtualRegister(RegClass: RC));
10601 Register NextSrcReg = MRI.createVirtualRegister(RegClass: RC);
10602 Register NextDestReg =
10603 (HaveSingleBase ? NextSrcReg : MRI.createVirtualRegister(RegClass: RC));
10604 RC = &SystemZ::GR64BitRegClass;
10605 Register ThisCountReg = MRI.createVirtualRegister(RegClass: RC);
10606 Register NextCountReg = MRI.createVirtualRegister(RegClass: RC);
10607
10608 if (IsRegForm) {
10609 AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10610 StartMBB = SystemZ::emitBlockAfter(MBB);
10611 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10612 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10613 DoneMBB = SystemZ::emitBlockAfter(MBB: NextMBB);
10614
10615 // MBB:
10616 // # Jump to AllDoneMBB if LenAdjReg means 0, or fall thru to StartMBB.
10617 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10618 .addReg(RegNo: LenAdjReg).addImm(Val: IsMemset ? -2 : -1);
10619 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10620 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10621 .addMBB(MBB: AllDoneMBB);
10622 MBB->addSuccessor(Succ: AllDoneMBB);
10623 if (!IsMemset)
10624 MBB->addSuccessor(Succ: StartMBB);
10625 else {
10626 // MemsetOneCheckMBB:
10627 // # Jump to MemsetOneMBB for a memset of length 1, or
10628 // # fall thru to StartMBB.
10629 MachineBasicBlock *MemsetOneCheckMBB = SystemZ::emitBlockAfter(MBB);
10630 MachineBasicBlock *MemsetOneMBB = SystemZ::emitBlockAfter(MBB: &*MF.rbegin());
10631 MBB->addSuccessor(Succ: MemsetOneCheckMBB);
10632 MBB = MemsetOneCheckMBB;
10633 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10634 .addReg(RegNo: LenAdjReg).addImm(Val: -1);
10635 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10636 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10637 .addMBB(MBB: MemsetOneMBB);
10638 MBB->addSuccessor(Succ: MemsetOneMBB, Prob: {10, 100});
10639 MBB->addSuccessor(Succ: StartMBB, Prob: {90, 100});
10640
10641 // MemsetOneMBB:
10642 // # Jump back to AllDoneMBB after a single MVI or STC.
10643 MBB = MemsetOneMBB;
10644 insertMemMemOp(MBB, MBB->end(),
10645 MachineOperand::CreateReg(Reg: StartDestReg, isDef: false), DestDisp,
10646 MachineOperand::CreateReg(Reg: StartSrcReg, isDef: false), SrcDisp,
10647 1);
10648 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: AllDoneMBB);
10649 MBB->addSuccessor(Succ: AllDoneMBB);
10650 }
10651
10652 // StartMBB:
10653 // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB.
10654 MBB = StartMBB;
10655 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10656 .addReg(RegNo: StartCountReg).addImm(Val: 0);
10657 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10658 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10659 .addMBB(MBB: DoneMBB);
10660 MBB->addSuccessor(Succ: DoneMBB);
10661 MBB->addSuccessor(Succ: LoopMBB);
10662 }
10663 else {
10664 StartMBB = MBB;
10665 DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10666 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10667 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10668
10669 // StartMBB:
10670 // # fall through to LoopMBB
10671 MBB->addSuccessor(Succ: LoopMBB);
10672
10673 DestBase = MachineOperand::CreateReg(Reg: NextDestReg, isDef: false);
10674 SrcBase = MachineOperand::CreateReg(Reg: NextSrcReg, isDef: false);
10675 if (EndMBB && !ImmLength)
10676 // If the loop handled the whole CLC range, DoneMBB will be empty with
10677 // CC live-through into EndMBB, so add it as live-in.
10678 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10679 }
10680
10681 // LoopMBB:
10682 // %ThisDestReg = phi [ %StartDestReg, StartMBB ],
10683 // [ %NextDestReg, NextMBB ]
10684 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ],
10685 // [ %NextSrcReg, NextMBB ]
10686 // %ThisCountReg = phi [ %StartCountReg, StartMBB ],
10687 // [ %NextCountReg, NextMBB ]
10688 // ( PFD 2, 768+DestDisp(%ThisDestReg) )
10689 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg)
10690 // ( JLH EndMBB )
10691 //
10692 // The prefetch is used only for MVC. The JLH is used only for CLC.
10693 MBB = LoopMBB;
10694 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisDestReg)
10695 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10696 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10697 if (!HaveSingleBase)
10698 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisSrcReg)
10699 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10700 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10701 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisCountReg)
10702 .addReg(RegNo: StartCountReg).addMBB(MBB: StartMBB)
10703 .addReg(RegNo: NextCountReg).addMBB(MBB: NextMBB);
10704 if (Opcode == SystemZ::MVC)
10705 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PFD))
10706 .addImm(Val: SystemZ::PFD_WRITE)
10707 .addReg(RegNo: ThisDestReg).addImm(Val: DestDisp - IsMemset + 768).addReg(RegNo: 0);
10708 insertMemMemOp(MBB, MBB->end(),
10709 MachineOperand::CreateReg(Reg: ThisDestReg, isDef: false), DestDisp,
10710 MachineOperand::CreateReg(Reg: ThisSrcReg, isDef: false), SrcDisp, 256);
10711 if (EndMBB) {
10712 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10713 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10714 .addMBB(MBB: EndMBB);
10715 MBB->addSuccessor(Succ: EndMBB);
10716 MBB->addSuccessor(Succ: NextMBB);
10717 }
10718
10719 // NextMBB:
10720 // %NextDestReg = LA 256(%ThisDestReg)
10721 // %NextSrcReg = LA 256(%ThisSrcReg)
10722 // %NextCountReg = AGHI %ThisCountReg, -1
10723 // CGHI %NextCountReg, 0
10724 // JLH LoopMBB
10725 // # fall through to DoneMBB
10726 //
10727 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes.
10728 MBB = NextMBB;
10729 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextDestReg)
10730 .addReg(RegNo: ThisDestReg).addImm(Val: 256).addReg(RegNo: 0);
10731 if (!HaveSingleBase)
10732 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextSrcReg)
10733 .addReg(RegNo: ThisSrcReg).addImm(Val: 256).addReg(RegNo: 0);
10734 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::AGHI), DestReg: NextCountReg)
10735 .addReg(RegNo: ThisCountReg).addImm(Val: -1);
10736 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10737 .addReg(RegNo: NextCountReg).addImm(Val: 0);
10738 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10739 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10740 .addMBB(MBB: LoopMBB);
10741 MBB->addSuccessor(Succ: LoopMBB);
10742 MBB->addSuccessor(Succ: DoneMBB);
10743
10744 MBB = DoneMBB;
10745 if (IsRegForm) {
10746 // DoneMBB:
10747 // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
10748 // # Use EXecute Relative Long for the remainder of the bytes. The target
10749 // instruction of the EXRL will have a length field of 1 since 0 is an
10750 // illegal value. The number of bytes processed becomes (%LenAdjReg &
10751 // 0xff) + 1.
10752 // # Fall through to AllDoneMBB.
10753 Register RemSrcReg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10754 Register RemDestReg = HaveSingleBase ? RemSrcReg
10755 : MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10756 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemDestReg)
10757 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10758 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10759 if (!HaveSingleBase)
10760 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemSrcReg)
10761 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10762 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10763 if (IsMemset)
10764 insertMemMemOp(MBB, MBB->end(),
10765 MachineOperand::CreateReg(Reg: RemDestReg, isDef: false), DestDisp,
10766 MachineOperand::CreateReg(Reg: RemSrcReg, isDef: false), SrcDisp, 1);
10767 MachineInstrBuilder EXRL_MIB =
10768 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EXRL_Pseudo))
10769 .addImm(Val: Opcode)
10770 .addReg(RegNo: LenAdjReg)
10771 .addReg(RegNo: RemDestReg).addImm(Val: DestDisp)
10772 .addReg(RegNo: RemSrcReg).addImm(Val: SrcDisp);
10773 MBB->addSuccessor(Succ: AllDoneMBB);
10774 MBB = AllDoneMBB;
10775 if (Opcode != SystemZ::MVC) {
10776 EXRL_MIB.addReg(RegNo: SystemZ::CC, Flags: RegState::ImplicitDefine);
10777 if (EndMBB)
10778 MBB->addLiveIn(PhysReg: SystemZ::CC);
10779 }
10780 }
10781 MF.getProperties().resetNoPHIs();
10782 }
10783
10784 // Handle any remaining bytes with straight-line code.
10785 while (ImmLength > 0) {
10786 uint64_t ThisLength = std::min(a: ImmLength, b: uint64_t(256));
10787 // The previous iteration might have created out-of-range displacements.
10788 // Apply them using LA/LAY if so.
10789 foldDisplIfNeeded(DestBase, DestDisp);
10790 foldDisplIfNeeded(SrcBase, SrcDisp);
10791 insertMemMemOp(MBB, MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
10792 DestDisp += ThisLength;
10793 SrcDisp += ThisLength;
10794 ImmLength -= ThisLength;
10795 // If there's another CLC to go, branch to the end if a difference
10796 // was found.
10797 if (EndMBB && ImmLength > 0) {
10798 MachineBasicBlock *NextMBB = SystemZ::splitBlockBefore(MI, MBB);
10799 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10800 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10801 .addMBB(MBB: EndMBB);
10802 MBB->addSuccessor(Succ: EndMBB);
10803 MBB->addSuccessor(Succ: NextMBB);
10804 MBB = NextMBB;
10805 }
10806 }
10807 if (EndMBB) {
10808 MBB->addSuccessor(Succ: EndMBB);
10809 MBB = EndMBB;
10810 MBB->addLiveIn(PhysReg: SystemZ::CC);
10811 }
10812
10813 MI.eraseFromParent();
10814 return MBB;
10815}
10816
10817// Decompose string pseudo-instruction MI into a loop that continually performs
10818// Opcode until CC != 3.
10819MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
10820 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
10821 MachineFunction &MF = *MBB->getParent();
10822 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10823 MachineRegisterInfo &MRI = MF.getRegInfo();
10824 DebugLoc DL = MI.getDebugLoc();
10825
10826 uint64_t End1Reg = MI.getOperand(i: 0).getReg();
10827 uint64_t Start1Reg = MI.getOperand(i: 1).getReg();
10828 uint64_t Start2Reg = MI.getOperand(i: 2).getReg();
10829 uint64_t CharReg = MI.getOperand(i: 3).getReg();
10830
10831 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass;
10832 uint64_t This1Reg = MRI.createVirtualRegister(RegClass: RC);
10833 uint64_t This2Reg = MRI.createVirtualRegister(RegClass: RC);
10834 uint64_t End2Reg = MRI.createVirtualRegister(RegClass: RC);
10835
10836 MachineBasicBlock *StartMBB = MBB;
10837 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10838 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10839
10840 // StartMBB:
10841 // # fall through to LoopMBB
10842 MBB->addSuccessor(Succ: LoopMBB);
10843
10844 // LoopMBB:
10845 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ]
10846 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ]
10847 // R0L = %CharReg
10848 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L
10849 // JO LoopMBB
10850 // # fall through to DoneMBB
10851 //
10852 // The load of R0L can be hoisted by post-RA LICM.
10853 MBB = LoopMBB;
10854
10855 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This1Reg)
10856 .addReg(RegNo: Start1Reg).addMBB(MBB: StartMBB)
10857 .addReg(RegNo: End1Reg).addMBB(MBB: LoopMBB);
10858 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This2Reg)
10859 .addReg(RegNo: Start2Reg).addMBB(MBB: StartMBB)
10860 .addReg(RegNo: End2Reg).addMBB(MBB: LoopMBB);
10861 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: SystemZ::R0L).addReg(RegNo: CharReg);
10862 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode))
10863 .addReg(RegNo: End1Reg, Flags: RegState::Define).addReg(RegNo: End2Reg, Flags: RegState::Define)
10864 .addReg(RegNo: This1Reg).addReg(RegNo: This2Reg);
10865 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10866 .addImm(Val: SystemZ::CCMASK_ANY).addImm(Val: SystemZ::CCMASK_3).addMBB(MBB: LoopMBB);
10867 MBB->addSuccessor(Succ: LoopMBB);
10868 MBB->addSuccessor(Succ: DoneMBB);
10869
10870 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10871
10872 MI.eraseFromParent();
10873 return DoneMBB;
10874}
10875
10876// Update TBEGIN instruction with final opcode and register clobbers.
10877MachineBasicBlock *SystemZTargetLowering::emitTransactionBegin(
10878 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode,
10879 bool NoFloat) const {
10880 MachineFunction &MF = *MBB->getParent();
10881 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
10882 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10883
10884 // Update opcode.
10885 MI.setDesc(TII->get(Opcode));
10886
10887 // We cannot handle a TBEGIN that clobbers the stack or frame pointer.
10888 // Make sure to add the corresponding GRSM bits if they are missing.
10889 uint64_t Control = MI.getOperand(i: 2).getImm();
10890 static const unsigned GPRControlBit[16] = {
10891 0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000,
10892 0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100
10893 };
10894 Control |= GPRControlBit[15];
10895 if (TFI->hasFP(MF))
10896 Control |= GPRControlBit[11];
10897 MI.getOperand(i: 2).setImm(Control);
10898
10899 // Add GPR clobbers.
10900 for (int I = 0; I < 16; I++) {
10901 if ((Control & GPRControlBit[I]) == 0) {
10902 unsigned Reg = SystemZMC::GR64Regs[I];
10903 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
10904 }
10905 }
10906
10907 // Add FPR/VR clobbers.
10908 if (!NoFloat && (Control & 4) != 0) {
10909 if (Subtarget.hasVector()) {
10910 for (unsigned Reg : SystemZMC::VR128Regs) {
10911 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
10912 }
10913 } else {
10914 for (unsigned Reg : SystemZMC::FP64Regs) {
10915 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
10916 }
10917 }
10918 }
10919
10920 return MBB;
10921}
10922
10923MachineBasicBlock *SystemZTargetLowering::emitLoadAndTestCmp0(
10924 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
10925 MachineFunction &MF = *MBB->getParent();
10926 MachineRegisterInfo *MRI = &MF.getRegInfo();
10927 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10928 DebugLoc DL = MI.getDebugLoc();
10929
10930 Register SrcReg = MI.getOperand(i: 0).getReg();
10931
10932 // Create new virtual register of the same class as source.
10933 const TargetRegisterClass *RC = MRI->getRegClass(Reg: SrcReg);
10934 Register DstReg = MRI->createVirtualRegister(RegClass: RC);
10935
10936 // Replace pseudo with a normal load-and-test that models the def as
10937 // well.
10938 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode), DestReg: DstReg)
10939 .addReg(RegNo: SrcReg)
10940 .setMIFlags(MI.getFlags());
10941 MI.eraseFromParent();
10942
10943 return MBB;
10944}
10945
10946MachineBasicBlock *SystemZTargetLowering::emitProbedAlloca(
10947 MachineInstr &MI, MachineBasicBlock *MBB) const {
10948 MachineFunction &MF = *MBB->getParent();
10949 MachineRegisterInfo *MRI = &MF.getRegInfo();
10950 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10951 DebugLoc DL = MI.getDebugLoc();
10952 const unsigned ProbeSize = getStackProbeSize(MF);
10953 Register DstReg = MI.getOperand(i: 0).getReg();
10954 Register SizeReg = MI.getOperand(i: 2).getReg();
10955
10956 MachineBasicBlock *StartMBB = MBB;
10957 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
10958 MachineBasicBlock *LoopTestMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10959 MachineBasicBlock *LoopBodyMBB = SystemZ::emitBlockAfter(MBB: LoopTestMBB);
10960 MachineBasicBlock *TailTestMBB = SystemZ::emitBlockAfter(MBB: LoopBodyMBB);
10961 MachineBasicBlock *TailMBB = SystemZ::emitBlockAfter(MBB: TailTestMBB);
10962
10963 MachineMemOperand *VolLdMMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
10964 F: MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, Size: 8, BaseAlignment: Align(1));
10965
10966 Register PHIReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10967 Register IncReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10968
10969 // LoopTestMBB
10970 // BRC TailTestMBB
10971 // # fallthrough to LoopBodyMBB
10972 StartMBB->addSuccessor(Succ: LoopTestMBB);
10973 MBB = LoopTestMBB;
10974 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: PHIReg)
10975 .addReg(RegNo: SizeReg)
10976 .addMBB(MBB: StartMBB)
10977 .addReg(RegNo: IncReg)
10978 .addMBB(MBB: LoopBodyMBB);
10979 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CLGFI))
10980 .addReg(RegNo: PHIReg)
10981 .addImm(Val: ProbeSize);
10982 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10983 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_LT)
10984 .addMBB(MBB: TailTestMBB);
10985 MBB->addSuccessor(Succ: LoopBodyMBB);
10986 MBB->addSuccessor(Succ: TailTestMBB);
10987
10988 // LoopBodyMBB: Allocate and probe by means of a volatile compare.
10989 // J LoopTestMBB
10990 MBB = LoopBodyMBB;
10991 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: IncReg)
10992 .addReg(RegNo: PHIReg)
10993 .addImm(Val: ProbeSize);
10994 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: SystemZ::R15D)
10995 .addReg(RegNo: SystemZ::R15D)
10996 .addImm(Val: ProbeSize);
10997 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG)).addReg(RegNo: SystemZ::R15D)
10998 .addReg(RegNo: SystemZ::R15D).addImm(Val: ProbeSize - 8).addReg(RegNo: 0)
10999 .setMemRefs(VolLdMMO);
11000 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: LoopTestMBB);
11001 MBB->addSuccessor(Succ: LoopTestMBB);
11002
11003 // TailTestMBB
11004 // BRC DoneMBB
11005 // # fallthrough to TailMBB
11006 MBB = TailTestMBB;
11007 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
11008 .addReg(RegNo: PHIReg)
11009 .addImm(Val: 0);
11010 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11011 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
11012 .addMBB(MBB: DoneMBB);
11013 MBB->addSuccessor(Succ: TailMBB);
11014 MBB->addSuccessor(Succ: DoneMBB);
11015
11016 // TailMBB
11017 // # fallthrough to DoneMBB
11018 MBB = TailMBB;
11019 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGR), DestReg: SystemZ::R15D)
11020 .addReg(RegNo: SystemZ::R15D)
11021 .addReg(RegNo: PHIReg);
11022 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG)).addReg(RegNo: SystemZ::R15D)
11023 .addReg(RegNo: SystemZ::R15D).addImm(Val: -8).addReg(RegNo: PHIReg)
11024 .setMemRefs(VolLdMMO);
11025 MBB->addSuccessor(Succ: DoneMBB);
11026
11027 // DoneMBB
11028 MBB = DoneMBB;
11029 BuildMI(BB&: *MBB, I: MBB->begin(), MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
11030 .addReg(RegNo: SystemZ::R15D);
11031
11032 MI.eraseFromParent();
11033 return DoneMBB;
11034}
11035
11036SDValue SystemZTargetLowering::
11037getBackchainAddress(SDValue SP, SelectionDAG &DAG) const {
11038 MachineFunction &MF = DAG.getMachineFunction();
11039 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
11040 SDLoc DL(SP);
11041 return DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: SP,
11042 N2: DAG.getIntPtrConstant(Val: TFL->getBackchainOffset(MF), DL));
11043}
11044
11045MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
11046 MachineInstr &MI, MachineBasicBlock *MBB) const {
11047 switch (MI.getOpcode()) {
11048 case SystemZ::ADJCALLSTACKDOWN:
11049 case SystemZ::ADJCALLSTACKUP:
11050 return emitAdjCallStack(MI, BB: MBB);
11051
11052 case SystemZ::Select32:
11053 case SystemZ::Select64:
11054 case SystemZ::Select128:
11055 case SystemZ::SelectF32:
11056 case SystemZ::SelectF64:
11057 case SystemZ::SelectF128:
11058 case SystemZ::SelectVR32:
11059 case SystemZ::SelectVR64:
11060 case SystemZ::SelectVR128:
11061 return emitSelect(MI, MBB);
11062
11063 case SystemZ::CondStore8Mux:
11064 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: false);
11065 case SystemZ::CondStore8MuxInv:
11066 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: true);
11067 case SystemZ::CondStore16Mux:
11068 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: false);
11069 case SystemZ::CondStore16MuxInv:
11070 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: true);
11071 case SystemZ::CondStore32Mux:
11072 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: false);
11073 case SystemZ::CondStore32MuxInv:
11074 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: true);
11075 case SystemZ::CondStore8:
11076 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: false);
11077 case SystemZ::CondStore8Inv:
11078 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: true);
11079 case SystemZ::CondStore16:
11080 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: false);
11081 case SystemZ::CondStore16Inv:
11082 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: true);
11083 case SystemZ::CondStore32:
11084 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: false);
11085 case SystemZ::CondStore32Inv:
11086 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: true);
11087 case SystemZ::CondStore64:
11088 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: false);
11089 case SystemZ::CondStore64Inv:
11090 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: true);
11091 case SystemZ::CondStoreF32:
11092 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: false);
11093 case SystemZ::CondStoreF32Inv:
11094 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: true);
11095 case SystemZ::CondStoreF64:
11096 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: false);
11097 case SystemZ::CondStoreF64Inv:
11098 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: true);
11099
11100 case SystemZ::SCmp128Hi:
11101 return emitICmp128Hi(MI, MBB, Unsigned: false);
11102 case SystemZ::UCmp128Hi:
11103 return emitICmp128Hi(MI, MBB, Unsigned: true);
11104
11105 case SystemZ::PAIR128:
11106 return emitPair128(MI, MBB);
11107 case SystemZ::AEXT128:
11108 return emitExt128(MI, MBB, ClearEven: false);
11109 case SystemZ::ZEXT128:
11110 return emitExt128(MI, MBB, ClearEven: true);
11111
11112 case SystemZ::ATOMIC_SWAPW:
11113 return emitAtomicLoadBinary(MI, MBB, BinOpcode: 0);
11114
11115 case SystemZ::ATOMIC_LOADW_AR:
11116 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AR);
11117 case SystemZ::ATOMIC_LOADW_AFI:
11118 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AFI);
11119
11120 case SystemZ::ATOMIC_LOADW_SR:
11121 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::SR);
11122
11123 case SystemZ::ATOMIC_LOADW_NR:
11124 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR);
11125 case SystemZ::ATOMIC_LOADW_NILH:
11126 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH);
11127
11128 case SystemZ::ATOMIC_LOADW_OR:
11129 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OR);
11130 case SystemZ::ATOMIC_LOADW_OILH:
11131 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OILH);
11132
11133 case SystemZ::ATOMIC_LOADW_XR:
11134 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XR);
11135 case SystemZ::ATOMIC_LOADW_XILF:
11136 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XILF);
11137
11138 case SystemZ::ATOMIC_LOADW_NRi:
11139 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR, Invert: true);
11140 case SystemZ::ATOMIC_LOADW_NILHi:
11141 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH, Invert: true);
11142
11143 case SystemZ::ATOMIC_LOADW_MIN:
11144 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
11145 case SystemZ::ATOMIC_LOADW_MAX:
11146 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
11147 case SystemZ::ATOMIC_LOADW_UMIN:
11148 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
11149 case SystemZ::ATOMIC_LOADW_UMAX:
11150 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
11151
11152 case SystemZ::ATOMIC_CMP_SWAPW:
11153 return emitAtomicCmpSwapW(MI, MBB);
11154 case SystemZ::MVCImm:
11155 case SystemZ::MVCReg:
11156 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC);
11157 case SystemZ::NCImm:
11158 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::NC);
11159 case SystemZ::OCImm:
11160 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::OC);
11161 case SystemZ::XCImm:
11162 case SystemZ::XCReg:
11163 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::XC);
11164 case SystemZ::CLCImm:
11165 case SystemZ::CLCReg:
11166 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::CLC);
11167 case SystemZ::MemsetImmImm:
11168 case SystemZ::MemsetImmReg:
11169 case SystemZ::MemsetRegImm:
11170 case SystemZ::MemsetRegReg:
11171 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC, IsMemset: true/*IsMemset*/);
11172 case SystemZ::CLSTLoop:
11173 return emitStringWrapper(MI, MBB, Opcode: SystemZ::CLST);
11174 case SystemZ::MVSTLoop:
11175 return emitStringWrapper(MI, MBB, Opcode: SystemZ::MVST);
11176 case SystemZ::SRSTLoop:
11177 return emitStringWrapper(MI, MBB, Opcode: SystemZ::SRST);
11178 case SystemZ::TBEGIN:
11179 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: false);
11180 case SystemZ::TBEGIN_nofloat:
11181 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: true);
11182 case SystemZ::TBEGINC:
11183 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGINC, NoFloat: true);
11184 case SystemZ::LTEBRCompare_Pseudo:
11185 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTEBR);
11186 case SystemZ::LTDBRCompare_Pseudo:
11187 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTDBR);
11188 case SystemZ::LTXBRCompare_Pseudo:
11189 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTXBR);
11190
11191 case SystemZ::PROBED_ALLOCA:
11192 return emitProbedAlloca(MI, MBB);
11193 case SystemZ::EH_SjLj_SetJmp:
11194 return emitEHSjLjSetJmp(MI, MBB);
11195 case SystemZ::EH_SjLj_LongJmp:
11196 return emitEHSjLjLongJmp(MI, MBB);
11197
11198 case TargetOpcode::STACKMAP:
11199 case TargetOpcode::PATCHPOINT:
11200 return emitPatchPoint(MI, MBB);
11201
11202 default:
11203 llvm_unreachable("Unexpected instr type to insert");
11204 }
11205}
11206
11207// This is only used by the isel schedulers, and is needed only to prevent
11208// compiler from crashing when list-ilp is used.
11209const TargetRegisterClass *
11210SystemZTargetLowering::getRepRegClassFor(MVT VT) const {
11211 if (VT == MVT::Untyped)
11212 return &SystemZ::ADDR128BitRegClass;
11213 return TargetLowering::getRepRegClassFor(VT);
11214}
11215
11216SDValue SystemZTargetLowering::lowerGET_ROUNDING(SDValue Op,
11217 SelectionDAG &DAG) const {
11218 SDLoc dl(Op);
11219 /*
11220 The rounding method is in FPC Byte 3 bits 6-7, and has the following
11221 settings:
11222 00 Round to nearest
11223 01 Round to 0
11224 10 Round to +inf
11225 11 Round to -inf
11226
11227 FLT_ROUNDS, on the other hand, expects the following:
11228 -1 Undefined
11229 0 Round to 0
11230 1 Round to nearest
11231 2 Round to +inf
11232 3 Round to -inf
11233 */
11234
11235 // Save FPC to register.
11236 SDValue Chain = Op.getOperand(i: 0);
11237 SDValue EFPC(
11238 DAG.getMachineNode(Opcode: SystemZ::EFPC, dl, ResultTys: {MVT::i32, MVT::Other}, Ops: Chain), 0);
11239 Chain = EFPC.getValue(R: 1);
11240
11241 // Transform as necessary
11242 SDValue CWD1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: EFPC,
11243 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
11244 // RetVal = (CWD1 ^ (CWD1 >> 1)) ^ 1
11245 SDValue CWD2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD1,
11246 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: CWD1,
11247 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32)));
11248
11249 SDValue RetVal = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD2,
11250 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11251 RetVal = DAG.getZExtOrTrunc(Op: RetVal, DL: dl, VT: Op.getValueType());
11252
11253 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl);
11254}
11255
11256SDValue SystemZTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
11257 SelectionDAG &DAG) const {
11258 EVT VT = Op.getValueType();
11259 Op = Op.getOperand(i: 0);
11260 EVT OpVT = Op.getValueType();
11261
11262 assert(OpVT.isVector() && "Operand type for VECREDUCE_ADD is not a vector.");
11263
11264 SDLoc DL(Op);
11265
11266 // load a 0 vector for the third operand of VSUM.
11267 SDValue Zero = DAG.getSplatBuildVector(VT: OpVT, DL, Op: DAG.getConstant(Val: 0, DL, VT));
11268
11269 // execute VSUM.
11270 switch (OpVT.getScalarSizeInBits()) {
11271 case 8:
11272 case 16:
11273 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Zero);
11274 [[fallthrough]];
11275 case 32:
11276 case 64:
11277 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::i128, N1: Op,
11278 N2: DAG.getBitcast(VT: Op.getValueType(), V: Zero));
11279 break;
11280 case 128:
11281 break; // VSUM over v1i128 should not happen and would be a noop
11282 default:
11283 llvm_unreachable("Unexpected scalar size.");
11284 }
11285 // Cast to original vector type, retrieve last element.
11286 return DAG.getNode(
11287 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: DAG.getBitcast(VT: OpVT, V: Op),
11288 N2: DAG.getConstant(Val: OpVT.getVectorNumElements() - 1, DL, VT: MVT::i32));
11289}
11290
11291static void printFunctionArgExts(const Function *F, raw_fd_ostream &OS) {
11292 FunctionType *FT = F->getFunctionType();
11293 const AttributeList &Attrs = F->getAttributes();
11294 if (Attrs.hasRetAttrs())
11295 OS << Attrs.getAsString(Index: AttributeList::ReturnIndex) << " ";
11296 OS << *F->getReturnType() << " @" << F->getName() << "(";
11297 for (unsigned I = 0, E = FT->getNumParams(); I != E; ++I) {
11298 if (I)
11299 OS << ", ";
11300 OS << *FT->getParamType(i: I);
11301 AttributeSet ArgAttrs = Attrs.getParamAttrs(ArgNo: I);
11302 for (auto A : {Attribute::SExt, Attribute::ZExt, Attribute::NoExt})
11303 if (ArgAttrs.hasAttribute(Kind: A))
11304 OS << " " << Attribute::getNameFromAttrKind(AttrKind: A);
11305 }
11306 OS << ")\n";
11307}
11308
11309bool SystemZTargetLowering::isInternal(const Function *Fn) const {
11310 std::map<const Function *, bool>::iterator Itr = IsInternalCache.find(x: Fn);
11311 if (Itr == IsInternalCache.end())
11312 Itr = IsInternalCache
11313 .insert(x: std::pair<const Function *, bool>(
11314 Fn, (Fn->hasLocalLinkage() && !Fn->hasAddressTaken())))
11315 .first;
11316 return Itr->second;
11317}
11318
11319void SystemZTargetLowering::
11320verifyNarrowIntegerArgs_Call(const SmallVectorImpl<ISD::OutputArg> &Outs,
11321 const Function *F, SDValue Callee) const {
11322 // Temporarily only do the check when explicitly requested, until it can be
11323 // enabled by default.
11324 if (!EnableIntArgExtCheck)
11325 return;
11326
11327 bool IsInternal = false;
11328 const Function *CalleeFn = nullptr;
11329 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
11330 if ((CalleeFn = dyn_cast<Function>(Val: G->getGlobal())))
11331 IsInternal = isInternal(Fn: CalleeFn);
11332 if (!IsInternal && !verifyNarrowIntegerArgs(Outs)) {
11333 errs() << "ERROR: Missing extension attribute of passed "
11334 << "value in call to function:\n" << "Callee: ";
11335 if (CalleeFn != nullptr)
11336 printFunctionArgExts(F: CalleeFn, OS&: errs());
11337 else
11338 errs() << "-\n";
11339 errs() << "Caller: ";
11340 printFunctionArgExts(F, OS&: errs());
11341 llvm_unreachable("");
11342 }
11343}
11344
11345void SystemZTargetLowering::
11346verifyNarrowIntegerArgs_Ret(const SmallVectorImpl<ISD::OutputArg> &Outs,
11347 const Function *F) const {
11348 // Temporarily only do the check when explicitly requested, until it can be
11349 // enabled by default.
11350 if (!EnableIntArgExtCheck)
11351 return;
11352
11353 if (!isInternal(Fn: F) && !verifyNarrowIntegerArgs(Outs)) {
11354 errs() << "ERROR: Missing extension attribute of returned "
11355 << "value from function:\n";
11356 printFunctionArgExts(F, OS&: errs());
11357 llvm_unreachable("");
11358 }
11359}
11360
11361// Verify that narrow integer arguments are extended as required by the ABI.
11362// Return false if an error is found.
11363bool SystemZTargetLowering::verifyNarrowIntegerArgs(
11364 const SmallVectorImpl<ISD::OutputArg> &Outs) const {
11365 if (!Subtarget.isTargetELF())
11366 return true;
11367
11368 if (EnableIntArgExtCheck.getNumOccurrences()) {
11369 if (!EnableIntArgExtCheck)
11370 return true;
11371 } else if (!getTargetMachine().Options.VerifyArgABICompliance)
11372 return true;
11373
11374 for (unsigned i = 0; i < Outs.size(); ++i) {
11375 MVT VT = Outs[i].VT;
11376 ISD::ArgFlagsTy Flags = Outs[i].Flags;
11377 if (VT.isInteger()) {
11378 assert((VT == MVT::i32 || VT.getSizeInBits() >= 64) &&
11379 "Unexpected integer argument VT.");
11380 if (VT == MVT::i32 &&
11381 !Flags.isSExt() && !Flags.isZExt() && !Flags.isNoExt())
11382 return false;
11383 }
11384 }
11385
11386 return true;
11387}
11388