1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SystemZTargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZISelLowering.h"
14#include "SystemZCallingConv.h"
15#include "SystemZConstantPoolValue.h"
16#include "SystemZMachineFunctionInfo.h"
17#include "llvm/ADT/SmallSet.h"
18#include "llvm/CodeGen/CallingConvLower.h"
19#include "llvm/CodeGen/ISDOpcodes.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
22#include "llvm/IR/GlobalAlias.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsS390.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/PatternMatch.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Target/TargetMachine.h"
32#include <cctype>
33#include <optional>
34
35using namespace llvm;
36
37#define DEBUG_TYPE "systemz-lower"
38
39// Temporarily let this be disabled by default until all known problems
40// related to argument extensions are fixed.
41static cl::opt<bool> EnableIntArgExtCheck(
42 "argext-abi-check", cl::init(Val: false),
43 cl::desc("Verify that narrow int args are properly extended per the "
44 "SystemZ ABI."));
45
46namespace {
47// Represents information about a comparison.
48struct Comparison {
49 Comparison(SDValue Op0In, SDValue Op1In, SDValue ChainIn)
50 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
51 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
52
53 // The operands to the comparison.
54 SDValue Op0, Op1;
55
56 // Chain if this is a strict floating-point comparison.
57 SDValue Chain;
58
59 // The opcode that should be used to compare Op0 and Op1.
60 unsigned Opcode;
61
62 // A SystemZICMP value. Only used for integer comparisons.
63 unsigned ICmpType;
64
65 // The mask of CC values that Opcode can produce.
66 unsigned CCValid;
67
68 // The mask of CC values for which the original condition is true.
69 unsigned CCMask;
70};
71} // end anonymous namespace
72
73// Classify VT as either 32 or 64 bit.
74static bool is32Bit(EVT VT) {
75 switch (VT.getSimpleVT().SimpleTy) {
76 case MVT::i32:
77 return true;
78 case MVT::i64:
79 return false;
80 default:
81 llvm_unreachable("Unsupported type");
82 }
83}
84
85// Return a version of MachineOperand that can be safely used before the
86// final use.
87static MachineOperand earlyUseOperand(MachineOperand Op) {
88 if (Op.isReg())
89 Op.setIsKill(false);
90 return Op;
91}
92
93SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
94 const SystemZSubtarget &STI)
95 : TargetLowering(TM, STI), Subtarget(STI) {
96 MVT PtrVT = MVT::i64;
97
98 auto *Regs = STI.getSpecialRegisters();
99
100 // Set up the register classes.
101 if (Subtarget.hasHighWord())
102 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GRX32BitRegClass);
103 else
104 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GR32BitRegClass);
105 addRegisterClass(VT: MVT::i64, RC: &SystemZ::GR64BitRegClass);
106 if (!useSoftFloat()) {
107 if (Subtarget.hasVector()) {
108 addRegisterClass(VT: MVT::f16, RC: &SystemZ::VR16BitRegClass);
109 addRegisterClass(VT: MVT::f32, RC: &SystemZ::VR32BitRegClass);
110 addRegisterClass(VT: MVT::f64, RC: &SystemZ::VR64BitRegClass);
111 } else {
112 addRegisterClass(VT: MVT::f16, RC: &SystemZ::FP16BitRegClass);
113 addRegisterClass(VT: MVT::f32, RC: &SystemZ::FP32BitRegClass);
114 addRegisterClass(VT: MVT::f64, RC: &SystemZ::FP64BitRegClass);
115 }
116 if (Subtarget.hasVectorEnhancements1())
117 addRegisterClass(VT: MVT::f128, RC: &SystemZ::VR128BitRegClass);
118 else
119 addRegisterClass(VT: MVT::f128, RC: &SystemZ::FP128BitRegClass);
120
121 if (Subtarget.hasVector()) {
122 addRegisterClass(VT: MVT::v16i8, RC: &SystemZ::VR128BitRegClass);
123 addRegisterClass(VT: MVT::v8i16, RC: &SystemZ::VR128BitRegClass);
124 addRegisterClass(VT: MVT::v4i32, RC: &SystemZ::VR128BitRegClass);
125 addRegisterClass(VT: MVT::v2i64, RC: &SystemZ::VR128BitRegClass);
126 addRegisterClass(VT: MVT::v8f16, RC: &SystemZ::VR128BitRegClass);
127 addRegisterClass(VT: MVT::v4f32, RC: &SystemZ::VR128BitRegClass);
128 addRegisterClass(VT: MVT::v2f64, RC: &SystemZ::VR128BitRegClass);
129 }
130
131 if (Subtarget.hasVector())
132 addRegisterClass(VT: MVT::i128, RC: &SystemZ::VR128BitRegClass);
133 }
134
135 // Compute derived properties from the register classes
136 computeRegisterProperties(TRI: Subtarget.getRegisterInfo());
137
138 // Set up special registers.
139 setStackPointerRegisterToSaveRestore(Regs->getStackPointerRegister());
140
141 // TODO: It may be better to default to latency-oriented scheduling, however
142 // LLVM's current latency-oriented scheduler can't handle physreg definitions
143 // such as SystemZ has with CC, so set this to the register-pressure
144 // scheduler, because it can.
145 setSchedulingPreference(Sched::RegPressure);
146
147 setBooleanContents(ZeroOrOneBooleanContent);
148 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
149
150 setMaxAtomicSizeInBitsSupported(128);
151
152 // Instructions are strings of 2-byte aligned 2-byte values.
153 setMinFunctionAlignment(Align(2));
154 // For performance reasons we prefer 16-byte alignment.
155 setPrefFunctionAlignment(Align(16));
156
157 // Handle operations that are handled in a similar way for all types.
158 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
159 I <= MVT::LAST_FP_VALUETYPE;
160 ++I) {
161 MVT VT = MVT::SimpleValueType(I);
162 if (isTypeLegal(VT)) {
163 // Lower SET_CC into an IPM-based sequence.
164 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
165 setOperationAction(Op: ISD::STRICT_FSETCC, VT, Action: Custom);
166 setOperationAction(Op: ISD::STRICT_FSETCCS, VT, Action: Custom);
167
168 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
169 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
170
171 // Lower SELECT_CC and BR_CC into separate comparisons and branches.
172 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Custom);
173 setOperationAction(Op: ISD::BR_CC, VT, Action: Custom);
174 }
175 }
176
177 // Expand jump table branches as address arithmetic followed by an
178 // indirect jump.
179 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
180
181 // Expand BRCOND into a BR_CC (see above).
182 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
183
184 // Handle integer types except i128.
185 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
186 I <= MVT::LAST_INTEGER_VALUETYPE;
187 ++I) {
188 MVT VT = MVT::SimpleValueType(I);
189 if (isTypeLegal(VT) && VT != MVT::i128) {
190 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
191
192 // Expand individual DIV and REMs into DIVREMs.
193 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
194 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
195 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
196 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
197 setOperationAction(Op: ISD::SDIVREM, VT, Action: Custom);
198 setOperationAction(Op: ISD::UDIVREM, VT, Action: Custom);
199
200 // Support addition/subtraction with overflow.
201 setOperationAction(Op: ISD::SADDO, VT, Action: Custom);
202 setOperationAction(Op: ISD::SSUBO, VT, Action: Custom);
203
204 // Support addition/subtraction with carry.
205 setOperationAction(Op: ISD::UADDO, VT, Action: Custom);
206 setOperationAction(Op: ISD::USUBO, VT, Action: Custom);
207
208 // Support carry in as value rather than glue.
209 setOperationAction(Op: ISD::UADDO_CARRY, VT, Action: Custom);
210 setOperationAction(Op: ISD::USUBO_CARRY, VT, Action: Custom);
211
212 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
213 // available, or if the operand is constant.
214 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT, Action: Custom);
215
216 // Use POPCNT on z196 and above.
217 if (Subtarget.hasPopulationCount())
218 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
219 else
220 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
221
222 // No special instructions for these.
223 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
224 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
225
226 // Use *MUL_LOHI where possible instead of MULH*.
227 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
228 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
229 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Custom);
230 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Custom);
231
232 // The fp<=>i32/i64 conversions are all Legal except for f16 and for
233 // unsigned on z10 (only z196 and above have native support for
234 // unsigned conversions).
235 for (auto Op : {ISD::FP_TO_SINT, ISD::STRICT_FP_TO_SINT,
236 ISD::SINT_TO_FP, ISD::STRICT_SINT_TO_FP})
237 setOperationAction(Op, VT, Action: Custom);
238 for (auto Op : {ISD::FP_TO_UINT, ISD::STRICT_FP_TO_UINT})
239 setOperationAction(Op, VT, Action: Custom);
240 for (auto Op : {ISD::UINT_TO_FP, ISD::STRICT_UINT_TO_FP}) {
241 // Handle unsigned 32-bit input types as signed 64-bit types on z10.
242 auto OpAction =
243 (!Subtarget.hasFPExtension() && VT == MVT::i32) ? Promote : Custom;
244 setOperationAction(Op, VT, Action: OpAction);
245 }
246 }
247 }
248
249 // Handle i128 if legal.
250 if (isTypeLegal(VT: MVT::i128)) {
251 // No special instructions for these.
252 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i128, Action: Expand);
253 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i128, Action: Expand);
254 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i128, Action: Expand);
255 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i128, Action: Expand);
256 setOperationAction(Op: ISD::ROTR, VT: MVT::i128, Action: Expand);
257 setOperationAction(Op: ISD::ROTL, VT: MVT::i128, Action: Expand);
258
259 // We may be able to use VSLDB/VSLD/VSRD for these.
260 setOperationAction(Op: ISD::FSHL, VT: MVT::i128, Action: Custom);
261 setOperationAction(Op: ISD::FSHR, VT: MVT::i128, Action: Custom);
262
263 // No special instructions for these before z17.
264 if (!Subtarget.hasVectorEnhancements3()) {
265 setOperationAction(Op: ISD::MUL, VT: MVT::i128, Action: Expand);
266 setOperationAction(Op: ISD::MULHS, VT: MVT::i128, Action: Expand);
267 setOperationAction(Op: ISD::MULHU, VT: MVT::i128, Action: Expand);
268 setOperationAction(Op: ISD::SDIV, VT: MVT::i128, Action: Expand);
269 setOperationAction(Op: ISD::UDIV, VT: MVT::i128, Action: Expand);
270 setOperationAction(Op: ISD::SREM, VT: MVT::i128, Action: Expand);
271 setOperationAction(Op: ISD::UREM, VT: MVT::i128, Action: Expand);
272 setOperationAction(Op: ISD::CTLZ, VT: MVT::i128, Action: Expand);
273 setOperationAction(Op: ISD::CTTZ, VT: MVT::i128, Action: Expand);
274 } else {
275 // Even if we do have a legal 128-bit multiply, we do not
276 // want 64-bit multiply-high operations to use it.
277 setOperationAction(Op: ISD::MULHS, VT: MVT::i64, Action: Custom);
278 setOperationAction(Op: ISD::MULHU, VT: MVT::i64, Action: Custom);
279 }
280
281 // Support addition/subtraction with carry.
282 setOperationAction(Op: ISD::UADDO, VT: MVT::i128, Action: Custom);
283 setOperationAction(Op: ISD::USUBO, VT: MVT::i128, Action: Custom);
284 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i128, Action: Custom);
285 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i128, Action: Custom);
286
287 // Use VPOPCT and add up partial results.
288 setOperationAction(Op: ISD::CTPOP, VT: MVT::i128, Action: Custom);
289
290 // Additional instructions available with z17.
291 if (Subtarget.hasVectorEnhancements3()) {
292 setOperationAction(Op: ISD::ABS, VT: MVT::i128, Action: Legal);
293
294 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
295 VT: MVT::i128, Action: Legal);
296 }
297 }
298
299 // These need custom handling in order to handle the f16 conversions.
300 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i128, Action: Custom);
301 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i128, Action: Custom);
302 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i128, Action: Custom);
303 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i128, Action: Custom);
304 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i128, Action: Custom);
305 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i128, Action: Custom);
306 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i128, Action: Custom);
307 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i128, Action: Custom);
308
309 // Type legalization will convert 8- and 16-bit atomic operations into
310 // forms that operate on i32s (but still keeping the original memory VT).
311 // Lower them into full i32 operations.
312 setOperationAction(Op: ISD::ATOMIC_SWAP, VT: MVT::i32, Action: Custom);
313 setOperationAction(Op: ISD::ATOMIC_LOAD_ADD, VT: MVT::i32, Action: Custom);
314 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT: MVT::i32, Action: Custom);
315 setOperationAction(Op: ISD::ATOMIC_LOAD_AND, VT: MVT::i32, Action: Custom);
316 setOperationAction(Op: ISD::ATOMIC_LOAD_OR, VT: MVT::i32, Action: Custom);
317 setOperationAction(Op: ISD::ATOMIC_LOAD_XOR, VT: MVT::i32, Action: Custom);
318 setOperationAction(Op: ISD::ATOMIC_LOAD_NAND, VT: MVT::i32, Action: Custom);
319 setOperationAction(Op: ISD::ATOMIC_LOAD_MIN, VT: MVT::i32, Action: Custom);
320 setOperationAction(Op: ISD::ATOMIC_LOAD_MAX, VT: MVT::i32, Action: Custom);
321 setOperationAction(Op: ISD::ATOMIC_LOAD_UMIN, VT: MVT::i32, Action: Custom);
322 setOperationAction(Op: ISD::ATOMIC_LOAD_UMAX, VT: MVT::i32, Action: Custom);
323
324 // Whether or not i128 is not a legal type, we need to custom lower
325 // the atomic operations in order to exploit SystemZ instructions.
326 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i128, Action: Custom);
327 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i128, Action: Custom);
328 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f128, Action: Custom);
329 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f128, Action: Custom);
330
331 // Mark sign/zero extending atomic loads as legal, which will make
332 // DAGCombiner fold extensions into atomic loads if possible.
333 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64,
334 MemVTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Legal);
335 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i32,
336 MemVTs: {MVT::i8, MVT::i16}, Action: Legal);
337 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i16,
338 MemVT: MVT::i8, Action: Legal);
339
340 // We can use the CC result of compare-and-swap to implement
341 // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
342 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i32, Action: Custom);
343 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i64, Action: Custom);
344 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i128, Action: Custom);
345
346 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
347
348 // Traps are legal, as we will convert them to "j .+2".
349 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
350
351 // We have native support for a 64-bit CTLZ, via FLOGR.
352 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Promote);
353 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: MVT::i32, Action: Promote);
354 setOperationAction(Op: ISD::CTLZ, VT: MVT::i64, Action: Legal);
355
356 // On z17 we have native support for a 64-bit CTTZ.
357 if (Subtarget.hasMiscellaneousExtensions4()) {
358 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Promote);
359 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::i32, Action: Promote);
360 setOperationAction(Op: ISD::CTTZ, VT: MVT::i64, Action: Legal);
361 }
362
363 // On z15 we have native support for a 64-bit CTPOP.
364 if (Subtarget.hasMiscellaneousExtensions3()) {
365 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Promote);
366 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Legal);
367 }
368
369 // Give LowerOperation the chance to replace 64-bit ORs with subregs.
370 setOperationAction(Op: ISD::OR, VT: MVT::i64, Action: Custom);
371
372 // Expand 128 bit shifts without using a libcall.
373 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64, Action: Expand);
374 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64, Action: Expand);
375 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64, Action: Expand);
376
377 // Also expand 256 bit shifts if i128 is a legal type.
378 if (isTypeLegal(VT: MVT::i128)) {
379 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i128, Action: Expand);
380 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i128, Action: Expand);
381 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i128, Action: Expand);
382 }
383
384 // Handle bitcast from fp128 to i128.
385 if (!isTypeLegal(VT: MVT::i128))
386 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
387
388 // We have native instructions for i8, i16 and i32 extensions, but not i1.
389 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
390 for (MVT VT : MVT::integer_valuetypes()) {
391 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
392 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
393 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
394 }
395
396 // Handle the various types of symbolic address.
397 setOperationAction(Op: ISD::ConstantPool, VT: PtrVT, Action: Custom);
398 setOperationAction(Op: ISD::GlobalAddress, VT: PtrVT, Action: Custom);
399 setOperationAction(Op: ISD::GlobalTLSAddress, VT: PtrVT, Action: Custom);
400 setOperationAction(Op: ISD::BlockAddress, VT: PtrVT, Action: Custom);
401 setOperationAction(Op: ISD::JumpTable, VT: PtrVT, Action: Custom);
402
403 // We need to handle dynamic allocations specially because of the
404 // 160-byte area at the bottom of the stack.
405 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: PtrVT, Action: Custom);
406 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: PtrVT, Action: Custom);
407
408 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
409 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Custom);
410
411 // Handle prefetches with PFD or PFDRL.
412 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
413
414 // Handle readcyclecounter with STCKF.
415 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Custom);
416
417 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
418 // Assume by default that all vector operations need to be expanded.
419 for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
420 if (getOperationAction(Op: Opcode, VT) == Legal)
421 setOperationAction(Op: Opcode, VT, Action: Expand);
422
423 // Likewise all truncating stores and extending loads.
424 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
425 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
426 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
427 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
428 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
429 }
430
431 if (isTypeLegal(VT)) {
432 // These operations are legal for anything that can be stored in a
433 // vector register, even if there is no native support for the format
434 // as such. In particular, we can do these for v4f32 even though there
435 // are no specific instructions for that format.
436 setOperationAction(Op: ISD::LOAD, VT, Action: Legal);
437 setOperationAction(Op: ISD::STORE, VT, Action: Legal);
438 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
439 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
440 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Legal);
441
442 // Likewise, except that we need to replace the nodes with something
443 // more specific.
444 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
445 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
446 }
447 }
448
449 // Handle integer vector types.
450 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
451 if (isTypeLegal(VT)) {
452 // These operations have direct equivalents.
453 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Legal);
454 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Legal);
455 setOperationAction(Op: ISD::ADD, VT, Action: Legal);
456 setOperationAction(Op: ISD::SUB, VT, Action: Legal);
457 if (VT != MVT::v2i64 || Subtarget.hasVectorEnhancements3()) {
458 setOperationAction(Op: ISD::MUL, VT, Action: Legal);
459 setOperationAction(Op: ISD::MULHS, VT, Action: Legal);
460 setOperationAction(Op: ISD::MULHU, VT, Action: Legal);
461 }
462 if (Subtarget.hasVectorEnhancements3() &&
463 VT != MVT::v16i8 && VT != MVT::v8i16) {
464 setOperationAction(Op: ISD::SDIV, VT, Action: Legal);
465 setOperationAction(Op: ISD::UDIV, VT, Action: Legal);
466 setOperationAction(Op: ISD::SREM, VT, Action: Legal);
467 setOperationAction(Op: ISD::UREM, VT, Action: Legal);
468 }
469 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
470 setOperationAction(Op: ISD::AND, VT, Action: Legal);
471 setOperationAction(Op: ISD::OR, VT, Action: Legal);
472 setOperationAction(Op: ISD::XOR, VT, Action: Legal);
473 if (Subtarget.hasVectorEnhancements1())
474 setOperationAction(Op: ISD::CTPOP, VT, Action: Legal);
475 else
476 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
477 setOperationAction(Op: ISD::CTTZ, VT, Action: Legal);
478 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
479
480 // Convert a GPR scalar to a vector by inserting it into element 0.
481 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
482
483 // Use a series of unpacks for extensions.
484 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Custom);
485 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT, Action: Custom);
486
487 // Detect shifts/rotates by a scalar amount and convert them into
488 // V*_BY_SCALAR.
489 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
490 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
491 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
492 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
493
494 // Add ISD::VECREDUCE_ADD as custom in order to implement
495 // it with VZERO+VSUM
496 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
497
498 // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
499 // and inverting the result as necessary.
500 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
501
502 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT,
503 Action: Legal);
504 }
505 }
506
507 if (Subtarget.hasVector()) {
508 // There should be no need to check for float types other than v2f64
509 // since <2 x f32> isn't a legal type.
510 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
511 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
512 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
513 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
514 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
515 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
516 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
517 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
518
519 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
520 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
521 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
522 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
523 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
524 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
525 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
526 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
527 }
528
529 if (Subtarget.hasVectorEnhancements2()) {
530 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
531 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
532 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
533 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
534 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
535 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
536 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
537 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
538
539 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
540 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
541 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
542 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
543 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
544 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
545 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
546 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
547 }
548
549 // Handle floating-point types.
550 if (!useSoftFloat()) {
551 // Promote all f16 operations to float, with some exceptions below.
552 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
553 setOperationAction(Op: Opc, VT: MVT::f16, Action: Promote);
554 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Expand);
555 for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) {
556 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f16, Action: Expand);
557 setTruncStoreAction(ValVT: VT, MemVT: MVT::f16, Action: Expand);
558 }
559 for (auto Op : {ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE})
560 setOperationAction(Op, VT: MVT::f16, Action: Subtarget.hasVector() ? Legal : Custom);
561 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f16, Action: LibCall);
562 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: LibCall);
563 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
564
565 for (auto Op : {ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN})
566 setOperationAction(Op, VT: MVT::f16, Action: Legal);
567 }
568
569 for (unsigned I = MVT::FIRST_FP_VALUETYPE;
570 I <= MVT::LAST_FP_VALUETYPE;
571 ++I) {
572 MVT VT = MVT::SimpleValueType(I);
573 if (isTypeLegal(VT) && VT != MVT::f16) {
574 // We can use FI for FRINT.
575 setOperationAction(Op: ISD::FRINT, VT, Action: Legal);
576
577 // We can use the extended form of FI for other rounding operations.
578 if (Subtarget.hasFPExtension()) {
579 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Legal);
580 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
581 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
582 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
583 setOperationAction(Op: ISD::FROUND, VT, Action: Legal);
584 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
585 }
586
587 // No special instructions for these.
588 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
589 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
590 setOperationAction(Op: ISD::FSINCOS, VT, Action: Expand);
591 setOperationAction(Op: ISD::FREM, VT, Action: LibCall);
592 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
593
594 // Special treatment.
595 setOperationAction(Op: ISD::IS_FPCLASS, VT, Action: Custom);
596
597 // Handle constrained floating-point operations.
598 setOperationAction(Op: ISD::STRICT_FADD, VT, Action: Legal);
599 setOperationAction(Op: ISD::STRICT_FSUB, VT, Action: Legal);
600 setOperationAction(Op: ISD::STRICT_FMUL, VT, Action: Legal);
601 setOperationAction(Op: ISD::STRICT_FDIV, VT, Action: Legal);
602 setOperationAction(Op: ISD::STRICT_FMA, VT, Action: Legal);
603 setOperationAction(Op: ISD::STRICT_FSQRT, VT, Action: Legal);
604 setOperationAction(Op: ISD::STRICT_FRINT, VT, Action: Legal);
605 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT, Action: Legal);
606 if (Subtarget.hasFPExtension()) {
607 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT, Action: Legal);
608 setOperationAction(Op: ISD::STRICT_FFLOOR, VT, Action: Legal);
609 setOperationAction(Op: ISD::STRICT_FCEIL, VT, Action: Legal);
610 setOperationAction(Op: ISD::STRICT_FTRUNC, VT, Action: Legal);
611 setOperationAction(Op: ISD::STRICT_FROUND, VT, Action: Legal);
612 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT, Action: Legal);
613 }
614
615 // Extension from f16 needs libcall.
616 setOperationAction(Op: ISD::FP_EXTEND, VT, Action: Custom);
617 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT, Action: Custom);
618 }
619 }
620
621 // Handle floating-point vector types.
622 if (Subtarget.hasVector()) {
623 // Scalar-to-vector conversion is just a subreg.
624 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8f16, Action: Legal);
625 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Legal);
626 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
627
628 // Some insertions and extractions can be done directly but others
629 // need to go via integers.
630 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8f16, Action: Custom);
631 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
632 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
633 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v8f16, Action: Custom);
634 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
635 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
636
637 // These operations have direct equivalents.
638 setOperationAction(Op: ISD::FADD, VT: MVT::v2f64, Action: Legal);
639 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Legal);
640 setOperationAction(Op: ISD::FSUB, VT: MVT::v2f64, Action: Legal);
641 setOperationAction(Op: ISD::FMUL, VT: MVT::v2f64, Action: Legal);
642 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Legal);
643 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Legal);
644 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Legal);
645 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Legal);
646 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Legal);
647 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f64, Action: Legal);
648 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Legal);
649 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Legal);
650 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Legal);
651 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f64, Action: Legal);
652 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
653
654 // Handle constrained floating-point operations.
655 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v2f64, Action: Legal);
656 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v2f64, Action: Legal);
657 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v2f64, Action: Legal);
658 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v2f64, Action: Legal);
659 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v2f64, Action: Legal);
660 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v2f64, Action: Legal);
661 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v2f64, Action: Legal);
662 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v2f64, Action: Legal);
663 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v2f64, Action: Legal);
664 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v2f64, Action: Legal);
665 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v2f64, Action: Legal);
666 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v2f64, Action: Legal);
667 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
668
669 setOperationAction(Op: ISD::SETCC, VT: MVT::v2f64, Action: Custom);
670 setOperationAction(Op: ISD::SETCC, VT: MVT::v4f32, Action: Custom);
671 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v2f64, Action: Custom);
672 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v4f32, Action: Custom);
673 if (Subtarget.hasVectorEnhancements1()) {
674 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v2f64, Action: Custom);
675 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v4f32, Action: Custom);
676 }
677 }
678
679 // The vector enhancements facility 1 has instructions for these.
680 if (Subtarget.hasVectorEnhancements1()) {
681 setOperationAction(Op: ISD::FADD, VT: MVT::v4f32, Action: Legal);
682 setOperationAction(Op: ISD::FNEG, VT: MVT::v4f32, Action: Legal);
683 setOperationAction(Op: ISD::FSUB, VT: MVT::v4f32, Action: Legal);
684 setOperationAction(Op: ISD::FMUL, VT: MVT::v4f32, Action: Legal);
685 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Legal);
686 setOperationAction(Op: ISD::FDIV, VT: MVT::v4f32, Action: Legal);
687 setOperationAction(Op: ISD::FABS, VT: MVT::v4f32, Action: Legal);
688 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Legal);
689 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Legal);
690 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Legal);
691 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Legal);
692 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Legal);
693 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Legal);
694 setOperationAction(Op: ISD::FROUND, VT: MVT::v4f32, Action: Legal);
695 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
696
697 for (MVT Type : {MVT::f64, MVT::v2f64, MVT::f32, MVT::v4f32, MVT::f128}) {
698 setOperationAction(Op: ISD::FMAXNUM, VT: Type, Action: Legal);
699 setOperationAction(Op: ISD::FMAXIMUM, VT: Type, Action: Legal);
700 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: Type, Action: Legal);
701 setOperationAction(Op: ISD::PSEUDO_FMAX, VT: Type, Action: Legal);
702 setOperationAction(Op: ISD::FMINNUM, VT: Type, Action: Legal);
703 setOperationAction(Op: ISD::FMINIMUM, VT: Type, Action: Legal);
704 setOperationAction(Op: ISD::FMINIMUMNUM, VT: Type, Action: Legal);
705 setOperationAction(Op: ISD::PSEUDO_FMIN, VT: Type, Action: Legal);
706 }
707
708 // Handle constrained floating-point operations.
709 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v4f32, Action: Legal);
710 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v4f32, Action: Legal);
711 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v4f32, Action: Legal);
712 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v4f32, Action: Legal);
713 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v4f32, Action: Legal);
714 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v4f32, Action: Legal);
715 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v4f32, Action: Legal);
716 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v4f32, Action: Legal);
717 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v4f32, Action: Legal);
718 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v4f32, Action: Legal);
719 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v4f32, Action: Legal);
720 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v4f32, Action: Legal);
721 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
722 for (auto VT : { MVT::f32, MVT::f64, MVT::f128,
723 MVT::v4f32, MVT::v2f64 }) {
724 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT, Action: Legal);
725 setOperationAction(Op: ISD::STRICT_FMINNUM, VT, Action: Legal);
726 setOperationAction(Op: ISD::STRICT_FMAXIMUM, VT, Action: Legal);
727 setOperationAction(Op: ISD::STRICT_FMINIMUM, VT, Action: Legal);
728 setOperationAction(Op: ISD::STRICT_PSEUDO_FMAX, VT, Action: Legal);
729 setOperationAction(Op: ISD::STRICT_PSEUDO_FMIN, VT, Action: Legal);
730 }
731 }
732
733 // We only have fused f128 multiply-addition on vector registers.
734 if (!Subtarget.hasVectorEnhancements1()) {
735 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Expand);
736 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f128, Action: Expand);
737 }
738
739 // We don't have a copysign instruction on vector registers.
740 if (Subtarget.hasVectorEnhancements1())
741 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f128, Action: Expand);
742
743 // Needed so that we don't try to implement f128 constant loads using
744 // a load-and-extend of a f80 constant (in cases where the constant
745 // would fit in an f80).
746 for (MVT VT : MVT::fp_valuetypes())
747 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f80, Action: Expand);
748
749 // We don't have extending load instruction on vector registers.
750 if (Subtarget.hasVectorEnhancements1()) {
751 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
752 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
753 }
754
755 // Floating-point truncation and stores need to be done separately.
756 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
757 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
758 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
759
760 // We have 64-bit FPR<->GPR moves, but need special handling for
761 // 32-bit forms.
762 if (!Subtarget.hasVector()) {
763 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
764 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Custom);
765 }
766
767 // VASTART and VACOPY need to deal with the SystemZ-specific varargs
768 // structure, but VAEND is a no-op.
769 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
770 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Custom);
771 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
772
773 if (Subtarget.isTargetzOS()) {
774 // Handle address space casts between mixed sized pointers.
775 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i32, Action: Custom);
776 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i64, Action: Custom);
777 }
778
779 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
780
781 // Codes for which we want to perform some z-specific combinations.
782 setTargetDAGCombine({ISD::ZERO_EXTEND,
783 ISD::SIGN_EXTEND,
784 ISD::SIGN_EXTEND_INREG,
785 ISD::LOAD,
786 ISD::STORE,
787 ISD::VECTOR_SHUFFLE,
788 ISD::EXTRACT_VECTOR_ELT,
789 ISD::FP_ROUND,
790 ISD::STRICT_FP_ROUND,
791 ISD::FP_EXTEND,
792 ISD::SINT_TO_FP,
793 ISD::UINT_TO_FP,
794 ISD::STRICT_FP_EXTEND,
795 ISD::FCOPYSIGN,
796 ISD::BSWAP,
797 ISD::SETCC,
798 ISD::SRL,
799 ISD::SRA,
800 ISD::MUL,
801 ISD::SDIV,
802 ISD::UDIV,
803 ISD::SREM,
804 ISD::UREM,
805 ISD::INTRINSIC_VOID,
806 ISD::INTRINSIC_W_CHAIN});
807
808 // Handle intrinsics.
809 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
810 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
811
812 // We're not using SJLJ for exception handling, but they're implemented
813 // solely to support use of __builtin_setjmp / __builtin_longjmp.
814 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
815 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
816
817 // We want to use MVC in preference to even a single load/store pair.
818 MaxStoresPerMemcpy = Subtarget.hasVector() ? 2 : 0;
819 MaxStoresPerMemcpyOptSize = 0;
820
821 // Same with memmove.
822 MaxStoresPerMemmove = Subtarget.hasVector() ? 2 : 0;
823 MaxStoresPerMemmoveOptSize = 0;
824
825 // The main memset sequence is a byte store followed by an MVC.
826 // Two STC or MV..I stores win over that, but the kind of fused stores
827 // generated by target-independent code don't when the byte value is
828 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better
829 // than "STC;MVC". Handle the choice in target-specific code instead.
830 MaxStoresPerMemset = Subtarget.hasVector() ? 2 : 0;
831 MaxStoresPerMemsetOptSize = 0;
832
833 // Default to having -disable-strictnode-mutation on
834 IsStrictFPEnabled = true;
835}
836
837bool SystemZTargetLowering::useSoftFloat() const {
838 return Subtarget.hasSoftFloat();
839}
840
841TargetLoweringBase::LegalizeTypeAction
842SystemZTargetLowering::getPreferredVectorAction(MVT VT) const {
843 // Widen subvectors to the full width rather than promoting integer
844 // elements. This is better because:
845 //
846 // (a) it means that we can handle the ABI for passing and returning
847 // sub-128 vectors without having to handle them as legal types.
848 //
849 // (b) we don't have instructions to extend on load and truncate on store,
850 // so promoting the integers is less efficient.
851 //
852 // (c) there are no multiplication instructions for the widest integer
853 // type (v2i64).
854
855 // Expand (narrow) f16 vectors during type legalization to avoid
856 // operations for all elements as with expansion after widening.
857 if (VT.getScalarType() == MVT::f16)
858 return VT.getVectorElementCount().isScalar() ? TypeScalarizeVector
859 : TypeSplitVector;
860 if (VT.getScalarSizeInBits() % 8 == 0)
861 return TypeWidenVector;
862 return TargetLoweringBase::getPreferredVectorAction(VT);
863}
864
865unsigned SystemZTargetLowering::getVectorTypeBreakdownForCallingConv(
866 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
867 unsigned &NumIntermediates, MVT &RegisterVT) const {
868 // Pass fp16 vectors in VR(s).
869 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16)) {
870 IntermediateVT = RegisterVT = MVT::v8f16;
871 return NumIntermediates =
872 divideCeil(Numerator: VT.getVectorNumElements(), Denominator: SystemZ::VectorBytes / 2);
873 }
874 return TargetLowering::getVectorTypeBreakdownForCallingConv(
875 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
876}
877
878MVT SystemZTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
879 CallingConv::ID CC,
880 EVT VT) const {
881 // 128-bit single-element vector types are passed like other vectors,
882 // not like their element type.
883 if (Subtarget.hasVector() && VT.isVector() && VT.getSizeInBits() == 128 &&
884 VT.getVectorNumElements() == 1)
885 return MVT::v16i8;
886 // Pass fp16 vectors in VR(s).
887 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16))
888 return MVT::v8f16;
889 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
890}
891
892unsigned SystemZTargetLowering::getNumRegistersForCallingConv(
893 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
894 // Pass fp16 vectors in VR(s).
895 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16))
896 return divideCeil(Numerator: VT.getVectorNumElements(), Denominator: SystemZ::VectorBytes / 2);
897 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
898}
899
900EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,
901 LLVMContext &, EVT VT) const {
902 if (!VT.isVector())
903 return MVT::i32;
904 return VT.changeVectorElementTypeToInteger();
905}
906
907bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(
908 const MachineFunction &MF, EVT VT) const {
909 if (useSoftFloat())
910 return false;
911
912 VT = VT.getScalarType();
913
914 if (!VT.isSimple())
915 return false;
916
917 switch (VT.getSimpleVT().SimpleTy) {
918 case MVT::f32:
919 case MVT::f64:
920 return true;
921 case MVT::f128:
922 return Subtarget.hasVectorEnhancements1();
923 default:
924 break;
925 }
926
927 return false;
928}
929
930// Return true if the constant can be generated with a vector instruction,
931// such as VGM, VGMB or VREPI.
932bool SystemZVectorConstantInfo::isVectorConstantLegal(
933 const SystemZSubtarget &Subtarget) {
934 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
935 if (!Subtarget.hasVector() ||
936 (isFP128 && !Subtarget.hasVectorEnhancements1()))
937 return false;
938
939 // Try using VECTOR GENERATE BYTE MASK. This is the architecturally-
940 // preferred way of creating all-zero and all-one vectors so give it
941 // priority over other methods below.
942 unsigned Mask = 0;
943 unsigned I = 0;
944 for (; I < SystemZ::VectorBytes; ++I) {
945 uint64_t Byte = IntBits.lshr(shiftAmt: I * 8).trunc(width: 8).getZExtValue();
946 if (Byte == 0xff)
947 Mask |= 1ULL << I;
948 else if (Byte != 0)
949 break;
950 }
951 if (I == SystemZ::VectorBytes) {
952 Opcode = SystemZISD::BYTE_MASK;
953 OpVals.push_back(Elt: Mask);
954 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 8), NumElements: 16);
955 return true;
956 }
957
958 if (SplatBitSize > 64)
959 return false;
960
961 auto TryValue = [&](uint64_t Value) -> bool {
962 // Try VECTOR REPLICATE IMMEDIATE
963 int64_t SignedValue = SignExtend64(X: Value, B: SplatBitSize);
964 if (isInt<16>(x: SignedValue)) {
965 OpVals.push_back(Elt: ((unsigned) SignedValue));
966 Opcode = SystemZISD::REPLICATE;
967 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
968 NumElements: SystemZ::VectorBits / SplatBitSize);
969 return true;
970 }
971 // Try VECTOR GENERATE MASK
972 unsigned Start, End;
973 if (TII->isRxSBGMask(Mask: Value, BitSize: SplatBitSize, Start, End)) {
974 // isRxSBGMask returns the bit numbers for a full 64-bit value, with 0
975 // denoting 1 << 63 and 63 denoting 1. Convert them to bit numbers for
976 // an SplatBitSize value, so that 0 denotes 1 << (SplatBitSize-1).
977 OpVals.push_back(Elt: Start - (64 - SplatBitSize));
978 OpVals.push_back(Elt: End - (64 - SplatBitSize));
979 Opcode = SystemZISD::ROTATE_MASK;
980 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
981 NumElements: SystemZ::VectorBits / SplatBitSize);
982 return true;
983 }
984 return false;
985 };
986
987 // First try assuming that any undefined bits above the highest set bit
988 // and below the lowest set bit are 1s. This increases the likelihood of
989 // being able to use a sign-extended element value in VECTOR REPLICATE
990 // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
991 uint64_t SplatBitsZ = SplatBits.getZExtValue();
992 uint64_t SplatUndefZ = SplatUndef.getZExtValue();
993 unsigned LowerBits = llvm::countr_zero(Val: SplatBitsZ);
994 unsigned UpperBits = llvm::countl_zero(Val: SplatBitsZ);
995 uint64_t Lower = SplatUndefZ & maskTrailingOnes<uint64_t>(N: LowerBits);
996 uint64_t Upper = SplatUndefZ & maskLeadingOnes<uint64_t>(N: UpperBits);
997 if (TryValue(SplatBitsZ | Upper | Lower))
998 return true;
999
1000 // Now try assuming that any undefined bits between the first and
1001 // last defined set bits are set. This increases the chances of
1002 // using a non-wraparound mask.
1003 uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
1004 return TryValue(SplatBitsZ | Middle);
1005}
1006
1007SystemZVectorConstantInfo::SystemZVectorConstantInfo(APInt IntImm) {
1008 if (IntImm.isSingleWord()) {
1009 IntBits = APInt(128, IntImm.getZExtValue());
1010 IntBits <<= (SystemZ::VectorBits - IntImm.getBitWidth());
1011 } else
1012 IntBits = IntImm;
1013 assert(IntBits.getBitWidth() == 128 && "Unsupported APInt.");
1014
1015 // Find the smallest splat.
1016 SplatBits = IntImm;
1017 unsigned Width = SplatBits.getBitWidth();
1018 while (Width > 8) {
1019 unsigned HalfSize = Width / 2;
1020 APInt HighValue = SplatBits.lshr(shiftAmt: HalfSize).trunc(width: HalfSize);
1021 APInt LowValue = SplatBits.trunc(width: HalfSize);
1022
1023 // If the two halves do not match, stop here.
1024 if (HighValue != LowValue || 8 > HalfSize)
1025 break;
1026
1027 SplatBits = HighValue;
1028 Width = HalfSize;
1029 }
1030 SplatUndef = 0;
1031 SplatBitSize = Width;
1032}
1033
1034SystemZVectorConstantInfo::SystemZVectorConstantInfo(BuildVectorSDNode *BVN) {
1035 assert(BVN->isConstant() && "Expected a constant BUILD_VECTOR");
1036 bool HasAnyUndefs;
1037
1038 // Get IntBits by finding the 128 bit splat.
1039 BVN->isConstantSplat(SplatValue&: IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 128,
1040 isBigEndian: true);
1041
1042 // Get SplatBits by finding the 8 bit or greater splat.
1043 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 8,
1044 isBigEndian: true);
1045}
1046
1047bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1048 bool ForCodeSize) const {
1049 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
1050 if (Imm.isZero() || Imm.isNegZero())
1051 return true;
1052
1053 return SystemZVectorConstantInfo(Imm).isVectorConstantLegal(Subtarget);
1054}
1055
1056MachineBasicBlock *
1057SystemZTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
1058 MachineBasicBlock *MBB) const {
1059 DebugLoc DL = MI.getDebugLoc();
1060 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1061 const SystemZRegisterInfo *TRI = Subtarget.getRegisterInfo();
1062
1063 MachineFunction *MF = MBB->getParent();
1064 MachineRegisterInfo &MRI = MF->getRegInfo();
1065
1066 const BasicBlock *BB = MBB->getBasicBlock();
1067 MachineFunction::iterator I = ++MBB->getIterator();
1068
1069 Register DstReg = MI.getOperand(i: 0).getReg();
1070 const TargetRegisterClass *RC = MRI.getRegClass(Reg: DstReg);
1071 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
1072 (void)TRI;
1073 Register MainDstReg = MRI.createVirtualRegister(RegClass: RC);
1074 Register RestoreDstReg = MRI.createVirtualRegister(RegClass: RC);
1075
1076 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1077 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1078 // For v = setjmp(buf), we generate.
1079 // Algorithm:
1080 //
1081 // ---------
1082 // | thisMBB |
1083 // ---------
1084 // |
1085 // ------------------------
1086 // | |
1087 // ---------- ---------------
1088 // | mainMBB | | restoreMBB |
1089 // | v = 0 | | v = 1 |
1090 // ---------- ---------------
1091 // | |
1092 // -------------------------
1093 // |
1094 // -----------------------------
1095 // | sinkMBB |
1096 // | phi(v_mainMBB,v_restoreMBB) |
1097 // -----------------------------
1098 // thisMBB:
1099 // buf[FPOffset] = Frame Pointer if hasFP.
1100 // buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB.
1101 // buf[BCOffset] = Backchain value if building with -mbackchain.
1102 // buf[SPOffset] = Stack Pointer.
1103 // buf[LPOffset] = We never write this slot with R13, gcc stores R13 always.
1104 // SjLjSetup restoreMBB
1105 // mainMBB:
1106 // v_main = 0
1107 // sinkMBB:
1108 // v = phi(v_main, v_restore)
1109 // restoreMBB:
1110 // v_restore = 1
1111
1112 MachineBasicBlock *ThisMBB = MBB;
1113 MachineBasicBlock *MainMBB = MF->CreateMachineBasicBlock(BB);
1114 MachineBasicBlock *SinkMBB = MF->CreateMachineBasicBlock(BB);
1115 MachineBasicBlock *RestoreMBB = MF->CreateMachineBasicBlock(BB);
1116
1117 MF->insert(MBBI: I, MBB: MainMBB);
1118 MF->insert(MBBI: I, MBB: SinkMBB);
1119 MF->push_back(MBB: RestoreMBB);
1120 RestoreMBB->setMachineBlockAddressTaken();
1121
1122 MachineInstrBuilder MIB;
1123
1124 // Transfer the remainder of BB and its successor edges to sinkMBB.
1125 SinkMBB->splice(Where: SinkMBB->begin(), Other: MBB,
1126 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
1127 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
1128
1129 // thisMBB:
1130 const int64_t FPOffset = 0; // Slot 1.
1131 const int64_t LabelOffset = 1 * PVT.getStoreSize(); // Slot 2.
1132 const int64_t BCOffset = 2 * PVT.getStoreSize(); // Slot 3.
1133 const int64_t SPOffset = 3 * PVT.getStoreSize(); // Slot 4.
1134
1135 // Buf address.
1136 Register BufReg = MI.getOperand(i: 1).getReg();
1137
1138 const TargetRegisterClass *PtrRC = getRegClassFor(VT: PVT);
1139 Register LabelReg = MRI.createVirtualRegister(RegClass: PtrRC);
1140
1141 // Prepare IP for longjmp.
1142 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LARL), DestReg: LabelReg)
1143 .addMBB(MBB: RestoreMBB);
1144 // Store IP for return from jmp, slot 2, offset = 1.
1145 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1146 .addReg(RegNo: LabelReg)
1147 .addReg(RegNo: BufReg)
1148 .addImm(Val: LabelOffset)
1149 .addReg(RegNo: 0);
1150
1151 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1152 bool HasFP = Subtarget.getFrameLowering()->hasFP(MF: *MF);
1153 if (HasFP) {
1154 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1155 .addReg(RegNo: SpecialRegs->getFramePointerRegister())
1156 .addReg(RegNo: BufReg)
1157 .addImm(Val: FPOffset)
1158 .addReg(RegNo: 0);
1159 }
1160
1161 // Store SP.
1162 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1163 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1164 .addReg(RegNo: BufReg)
1165 .addImm(Val: SPOffset)
1166 .addReg(RegNo: 0);
1167
1168 // Slot 3(Offset = 2) Backchain value (if building with -mbackchain).
1169 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1170 if (BackChain) {
1171 Register BCReg = MRI.createVirtualRegister(RegClass: PtrRC);
1172 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1173 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1174 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1175 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1176 .addReg(RegNo: 0);
1177
1178 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1179 .addReg(RegNo: BCReg)
1180 .addReg(RegNo: BufReg)
1181 .addImm(Val: BCOffset)
1182 .addReg(RegNo: 0);
1183 }
1184
1185 // Setup.
1186 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EH_SjLj_Setup))
1187 .addMBB(MBB: RestoreMBB);
1188
1189 const SystemZRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1190 MIB.addRegMask(Mask: RegInfo->getNoPreservedMask());
1191
1192 ThisMBB->addSuccessor(Succ: MainMBB);
1193 ThisMBB->addSuccessor(Succ: RestoreMBB);
1194
1195 // mainMBB:
1196 BuildMI(BB: MainMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: MainDstReg).addImm(Val: 0);
1197 MainMBB->addSuccessor(Succ: SinkMBB);
1198
1199 // sinkMBB:
1200 BuildMI(BB&: *SinkMBB, I: SinkMBB->begin(), MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: DstReg)
1201 .addReg(RegNo: MainDstReg)
1202 .addMBB(MBB: MainMBB)
1203 .addReg(RegNo: RestoreDstReg)
1204 .addMBB(MBB: RestoreMBB);
1205
1206 // restoreMBB.
1207 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: RestoreDstReg).addImm(Val: 1);
1208 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: SinkMBB);
1209 RestoreMBB->addSuccessor(Succ: SinkMBB);
1210
1211 MI.eraseFromParent();
1212
1213 return SinkMBB;
1214}
1215
1216MachineBasicBlock *
1217SystemZTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
1218 MachineBasicBlock *MBB) const {
1219
1220 DebugLoc DL = MI.getDebugLoc();
1221 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1222
1223 MachineFunction *MF = MBB->getParent();
1224 MachineRegisterInfo &MRI = MF->getRegInfo();
1225
1226 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1227 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1228 Register BufReg = MI.getOperand(i: 0).getReg();
1229 const TargetRegisterClass *RC = MRI.getRegClass(Reg: BufReg);
1230 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1231
1232 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
1233 Register BCReg = MRI.createVirtualRegister(RegClass: RC);
1234
1235 MachineInstrBuilder MIB;
1236
1237 const int64_t FPOffset = 0;
1238 const int64_t LabelOffset = 1 * PVT.getStoreSize();
1239 const int64_t BCOffset = 2 * PVT.getStoreSize();
1240 const int64_t SPOffset = 3 * PVT.getStoreSize();
1241 const int64_t LPOffset = 4 * PVT.getStoreSize();
1242
1243 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: Tmp)
1244 .addReg(RegNo: BufReg)
1245 .addImm(Val: LabelOffset)
1246 .addReg(RegNo: 0);
1247
1248 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1249 DestReg: SpecialRegs->getFramePointerRegister())
1250 .addReg(RegNo: BufReg)
1251 .addImm(Val: FPOffset)
1252 .addReg(RegNo: 0);
1253
1254 // We are restoring R13 even though we never stored in setjmp from llvm,
1255 // as gcc always stores R13 in builtin_setjmp. We could have mixed code
1256 // gcc setjmp and llvm longjmp.
1257 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: SystemZ::R13D)
1258 .addReg(RegNo: BufReg)
1259 .addImm(Val: LPOffset)
1260 .addReg(RegNo: 0);
1261
1262 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1263 if (BackChain) {
1264 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1265 .addReg(RegNo: BufReg)
1266 .addImm(Val: BCOffset)
1267 .addReg(RegNo: 0);
1268 }
1269
1270 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1271 DestReg: SpecialRegs->getStackPointerRegister())
1272 .addReg(RegNo: BufReg)
1273 .addImm(Val: SPOffset)
1274 .addReg(RegNo: 0);
1275
1276 if (BackChain) {
1277 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1278 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1279 .addReg(RegNo: BCReg)
1280 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1281 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1282 .addReg(RegNo: 0);
1283 }
1284
1285 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BR)).addReg(RegNo: Tmp);
1286
1287 MI.eraseFromParent();
1288 return MBB;
1289}
1290
1291/// Returns true if stack probing through inline assembly is requested.
1292bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
1293 // If the function specifically requests inline stack probes, emit them.
1294 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
1295 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
1296 "inline-asm";
1297 return false;
1298}
1299
1300TargetLowering::AtomicExpansionKind
1301SystemZTargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
1302 return AtomicExpansionKind::None;
1303}
1304
1305TargetLowering::AtomicExpansionKind
1306SystemZTargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
1307 return AtomicExpansionKind::None;
1308}
1309
1310TargetLowering::AtomicExpansionKind
1311SystemZTargetLowering::shouldExpandAtomicRMWInIR(
1312 const AtomicRMWInst *RMW) const {
1313 // Don't expand subword operations as they require special treatment.
1314 if (RMW->getType()->isIntegerTy(BitWidth: 8) || RMW->getType()->isIntegerTy(BitWidth: 16))
1315 return AtomicExpansionKind::None;
1316
1317 // Don't expand if there is a target instruction available.
1318 if (Subtarget.hasInterlockedAccess1() &&
1319 (RMW->getType()->isIntegerTy(BitWidth: 32) || RMW->getType()->isIntegerTy(BitWidth: 64)) &&
1320 (RMW->getOperation() == AtomicRMWInst::BinOp::Add ||
1321 RMW->getOperation() == AtomicRMWInst::BinOp::Sub ||
1322 RMW->getOperation() == AtomicRMWInst::BinOp::And ||
1323 RMW->getOperation() == AtomicRMWInst::BinOp::Or ||
1324 RMW->getOperation() == AtomicRMWInst::BinOp::Xor))
1325 return AtomicExpansionKind::None;
1326
1327 return AtomicExpansionKind::CmpXChg;
1328}
1329
1330bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1331 // We can use CGFI or CLGFI.
1332 return isInt<32>(x: Imm) || isUInt<32>(x: Imm);
1333}
1334
1335bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1336 // We can use ALGFI or SLGFI.
1337 return isUInt<32>(x: Imm) || isUInt<32>(x: -Imm);
1338}
1339
1340bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(
1341 EVT VT, unsigned, Align, MachineMemOperand::Flags, unsigned *Fast) const {
1342 // Unaligned accesses should never be slower than the expanded version.
1343 // We check specifically for aligned accesses in the few cases where
1344 // they are required.
1345 if (Fast)
1346 *Fast = 1;
1347 return true;
1348}
1349
1350bool SystemZTargetLowering::hasAndNot(SDValue Y) const {
1351 EVT VT = Y.getValueType();
1352
1353 // We can use NC(G)RK for types in GPRs ...
1354 if (VT == MVT::i32 || VT == MVT::i64)
1355 return Subtarget.hasMiscellaneousExtensions3();
1356
1357 // ... or VNC for types in VRs.
1358 if (VT.isVector() || VT == MVT::i128)
1359 return Subtarget.hasVector();
1360
1361 return false;
1362}
1363
1364// Information about the addressing mode for a memory access.
1365struct AddressingMode {
1366 // True if a long displacement is supported.
1367 bool LongDisplacement;
1368
1369 // True if use of index register is supported.
1370 bool IndexReg;
1371
1372 AddressingMode(bool LongDispl, bool IdxReg) :
1373 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
1374};
1375
1376// Return the desired addressing mode for a Load which has only one use (in
1377// the same block) which is a Store.
1378static AddressingMode getLoadStoreAddrMode(bool HasVector,
1379 Type *Ty) {
1380 // With vector support a Load->Store combination may be combined to either
1381 // an MVC or vector operations and it seems to work best to allow the
1382 // vector addressing mode.
1383 if (HasVector)
1384 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1385
1386 // Otherwise only the MVC case is special.
1387 bool MVC = Ty->isIntegerTy(BitWidth: 8);
1388 return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/);
1389}
1390
1391// Return the addressing mode which seems most desirable given an LLVM
1392// Instruction pointer.
1393static AddressingMode
1394supportedAddressingMode(Instruction *I, bool HasVector) {
1395 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1396 switch (II->getIntrinsicID()) {
1397 default: break;
1398 case Intrinsic::memset:
1399 case Intrinsic::memmove:
1400 case Intrinsic::memcpy:
1401 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1402 }
1403 }
1404
1405 if (isa<LoadInst>(Val: I) && I->hasOneUse()) {
1406 auto *SingleUser = cast<Instruction>(Val: *I->user_begin());
1407 if (SingleUser->getParent() == I->getParent()) {
1408 if (isa<ICmpInst>(Val: SingleUser)) {
1409 if (auto *C = dyn_cast<ConstantInt>(Val: SingleUser->getOperand(i: 1)))
1410 if (C->getBitWidth() <= 64 &&
1411 (isInt<16>(x: C->getSExtValue()) || isUInt<16>(x: C->getZExtValue())))
1412 // Comparison of memory with 16 bit signed / unsigned immediate
1413 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1414 } else if (isa<StoreInst>(Val: SingleUser))
1415 // Load->Store
1416 return getLoadStoreAddrMode(HasVector, Ty: I->getType());
1417 }
1418 } else if (auto *StoreI = dyn_cast<StoreInst>(Val: I)) {
1419 if (auto *LoadI = dyn_cast<LoadInst>(Val: StoreI->getValueOperand()))
1420 if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())
1421 // Load->Store
1422 return getLoadStoreAddrMode(HasVector, Ty: LoadI->getType());
1423 }
1424
1425 if (HasVector && (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I))) {
1426
1427 // * Use LDE instead of LE/LEY for z13 to avoid partial register
1428 // dependencies (LDE only supports small offsets).
1429 // * Utilize the vector registers to hold floating point
1430 // values (vector load / store instructions only support small
1431 // offsets).
1432
1433 Type *MemAccessTy = (isa<LoadInst>(Val: I) ? I->getType() :
1434 I->getOperand(i: 0)->getType());
1435 bool IsFPAccess = MemAccessTy->isFloatingPointTy();
1436 bool IsVectorAccess = MemAccessTy->isVectorTy();
1437
1438 // A store of an extracted vector element will be combined into a VSTE type
1439 // instruction.
1440 if (!IsVectorAccess && isa<StoreInst>(Val: I)) {
1441 Value *DataOp = I->getOperand(i: 0);
1442 if (isa<ExtractElementInst>(Val: DataOp))
1443 IsVectorAccess = true;
1444 }
1445
1446 // A load which gets inserted into a vector element will be combined into a
1447 // VLE type instruction.
1448 if (!IsVectorAccess && isa<LoadInst>(Val: I) && I->hasOneUse()) {
1449 User *LoadUser = *I->user_begin();
1450 if (isa<InsertElementInst>(Val: LoadUser))
1451 IsVectorAccess = true;
1452 }
1453
1454 if (IsFPAccess || IsVectorAccess)
1455 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1456 }
1457
1458 return AddressingMode(true/*LongDispl*/, true/*IdxReg*/);
1459}
1460
1461bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1462 const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const {
1463 // Punt on globals for now, although they can be used in limited
1464 // RELATIVE LONG cases.
1465 if (AM.BaseGV)
1466 return false;
1467
1468 // Require a 20-bit signed offset.
1469 if (!isInt<20>(x: AM.BaseOffs))
1470 return false;
1471
1472 bool RequireD12 =
1473 Subtarget.hasVector() && (Ty->isVectorTy() || Ty->isIntegerTy(BitWidth: 128));
1474 AddressingMode SupportedAM(!RequireD12, true);
1475 if (I != nullptr)
1476 SupportedAM = supportedAddressingMode(I, HasVector: Subtarget.hasVector());
1477
1478 if (!SupportedAM.LongDisplacement && !isUInt<12>(x: AM.BaseOffs))
1479 return false;
1480
1481 if (!SupportedAM.IndexReg)
1482 // No indexing allowed.
1483 return AM.Scale == 0;
1484 else
1485 // Indexing is OK but no scale factor can be applied.
1486 return AM.Scale == 0 || AM.Scale == 1;
1487}
1488
1489bool SystemZTargetLowering::findOptimalMemOpLowering(
1490 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
1491 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
1492 const AttributeList &FuncAttributes, EVT *LargestVT) const {
1493
1494 assert(Limit != ~0U &&
1495 "Expected EmitTargetCodeForMemXXX() to handle AlwaysInline cases.");
1496
1497 if (Op.isZeroMemset())
1498 return false; // Memset zero: Use XC.
1499
1500 const int MVCFastLen = 16;
1501 // Use MVC up to 16 bytes for memcpy. Small memset uses STC/MVI for first
1502 // byte.
1503 if (Op.isMemcpy() && Op.size() <= MVCFastLen)
1504 return false;
1505 if (Op.isMemset() && Op.size() - 1 <= MVCFastLen)
1506 return false;
1507
1508 // Avoid unaligned VL/VST:s.
1509 if ((Op.size() >= 16 && !Op.isAligned(AlignCheck: Align(8))) ||
1510 (Op.size() >= 25 && Op.size() <= 31))
1511 return false;
1512
1513 return TargetLowering::findOptimalMemOpLowering(
1514 Context, MemOps, Limit, Op, DstAS, SrcAS, FuncAttributes, LargestVT);
1515}
1516
1517EVT SystemZTargetLowering::getOptimalMemOpType(
1518 LLVMContext &Context, const MemOp &Op,
1519 const AttributeList &FuncAttributes) const {
1520 return Subtarget.hasVector() ? MVT::v2i64 : MVT::Other;
1521}
1522
1523bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const {
1524 if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
1525 return false;
1526 unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();
1527 unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedValue();
1528 return FromBits > ToBits;
1529}
1530
1531bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const {
1532 if (!FromVT.isInteger() || !ToVT.isInteger())
1533 return false;
1534 unsigned FromBits = FromVT.getFixedSizeInBits();
1535 unsigned ToBits = ToVT.getFixedSizeInBits();
1536 return FromBits > ToBits;
1537}
1538
1539//===----------------------------------------------------------------------===//
1540// Inline asm support
1541//===----------------------------------------------------------------------===//
1542
1543TargetLowering::ConstraintType
1544SystemZTargetLowering::getConstraintType(StringRef Constraint) const {
1545 if (Constraint.size() == 1) {
1546 switch (Constraint[0]) {
1547 case 'a': // Address register
1548 case 'd': // Data register (equivalent to 'r')
1549 case 'f': // Floating-point register
1550 case 'h': // High-part register
1551 case 'r': // General-purpose register
1552 case 'v': // Vector register
1553 return C_RegisterClass;
1554
1555 case 'Q': // Memory with base and unsigned 12-bit displacement
1556 case 'R': // Likewise, plus an index
1557 case 'S': // Memory with base and signed 20-bit displacement
1558 case 'T': // Likewise, plus an index
1559 case 'm': // Equivalent to 'T'.
1560 return C_Memory;
1561
1562 case 'I': // Unsigned 8-bit constant
1563 case 'J': // Unsigned 12-bit constant
1564 case 'K': // Signed 16-bit constant
1565 case 'L': // Signed 20-bit displacement (on all targets we support)
1566 case 'M': // 0x7fffffff
1567 return C_Immediate;
1568
1569 default:
1570 break;
1571 }
1572 } else if (Constraint.size() == 2 && Constraint[0] == 'Z') {
1573 switch (Constraint[1]) {
1574 case 'Q': // Address with base and unsigned 12-bit displacement
1575 case 'R': // Likewise, plus an index
1576 case 'S': // Address with base and signed 20-bit displacement
1577 case 'T': // Likewise, plus an index
1578 return C_Address;
1579
1580 default:
1581 break;
1582 }
1583 } else if (Constraint.size() == 5 && Constraint.starts_with(Prefix: "{")) {
1584 if (StringRef("{@cc}").compare(RHS: Constraint) == 0)
1585 return C_Other;
1586 }
1587 return TargetLowering::getConstraintType(Constraint);
1588}
1589
1590TargetLowering::ConstraintWeight
1591SystemZTargetLowering::getSingleConstraintMatchWeight(
1592 AsmOperandInfo &Info, const char *Constraint) const {
1593 ConstraintWeight Weight = CW_Invalid;
1594 Value *CallOperandVal = Info.CallOperandVal;
1595 // If we don't have a value, we can't do a match,
1596 // but allow it at the lowest weight.
1597 if (!CallOperandVal)
1598 return CW_Default;
1599 Type *type = CallOperandVal->getType();
1600 // Look at the constraint type.
1601 switch (*Constraint) {
1602 default:
1603 Weight = TargetLowering::getSingleConstraintMatchWeight(info&: Info, constraint: Constraint);
1604 break;
1605
1606 case 'a': // Address register
1607 case 'd': // Data register (equivalent to 'r')
1608 case 'h': // High-part register
1609 case 'r': // General-purpose register
1610 Weight =
1611 CallOperandVal->getType()->isIntegerTy() ? CW_Register : CW_Default;
1612 break;
1613
1614 case 'f': // Floating-point register
1615 if (!useSoftFloat())
1616 Weight = type->isFloatingPointTy() ? CW_Register : CW_Default;
1617 break;
1618
1619 case 'v': // Vector register
1620 if (Subtarget.hasVector())
1621 Weight = (type->isVectorTy() || type->isFloatingPointTy()) ? CW_Register
1622 : CW_Default;
1623 break;
1624
1625 case 'I': // Unsigned 8-bit constant
1626 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1627 if (isUInt<8>(x: C->getZExtValue()))
1628 Weight = CW_Constant;
1629 break;
1630
1631 case 'J': // Unsigned 12-bit constant
1632 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1633 if (isUInt<12>(x: C->getZExtValue()))
1634 Weight = CW_Constant;
1635 break;
1636
1637 case 'K': // Signed 16-bit constant
1638 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1639 if (isInt<16>(x: C->getSExtValue()))
1640 Weight = CW_Constant;
1641 break;
1642
1643 case 'L': // Signed 20-bit displacement (on all targets we support)
1644 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1645 if (isInt<20>(x: C->getSExtValue()))
1646 Weight = CW_Constant;
1647 break;
1648
1649 case 'M': // 0x7fffffff
1650 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1651 if (C->getZExtValue() == 0x7fffffff)
1652 Weight = CW_Constant;
1653 break;
1654 }
1655 return Weight;
1656}
1657
1658// Parse a "{tNNN}" register constraint for which the register type "t"
1659// has already been verified. MC is the class associated with "t" and
1660// Map maps 0-based register numbers to LLVM register numbers.
1661static std::pair<unsigned, const TargetRegisterClass *>
1662parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC,
1663 const unsigned *Map, unsigned Size) {
1664 assert(*(Constraint.end()-1) == '}' && "Missing '}'");
1665 if (isdigit(Constraint[2])) {
1666 unsigned Index;
1667 bool Failed =
1668 Constraint.slice(Start: 2, End: Constraint.size() - 1).getAsInteger(Radix: 10, Result&: Index);
1669 if (!Failed && Index < Size && Map[Index])
1670 return std::make_pair(x: Map[Index], y&: RC);
1671 }
1672 return std::make_pair(x: 0U, y: nullptr);
1673}
1674
1675std::pair<unsigned, const TargetRegisterClass *>
1676SystemZTargetLowering::getRegForInlineAsmConstraint(
1677 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
1678 if (Constraint.size() == 1) {
1679 // GCC Constraint Letters
1680 switch (Constraint[0]) {
1681 default: break;
1682 case 'd': // Data register (equivalent to 'r')
1683 case 'r': // General-purpose register
1684 if (VT.getSizeInBits() == 64)
1685 return std::make_pair(x: 0U, y: &SystemZ::GR64BitRegClass);
1686 else if (VT.getSizeInBits() == 128)
1687 return std::make_pair(x: 0U, y: &SystemZ::GR128BitRegClass);
1688 return std::make_pair(x: 0U, y: &SystemZ::GR32BitRegClass);
1689
1690 case 'a': // Address register
1691 if (VT == MVT::i64)
1692 return std::make_pair(x: 0U, y: &SystemZ::ADDR64BitRegClass);
1693 else if (VT == MVT::i128)
1694 return std::make_pair(x: 0U, y: &SystemZ::ADDR128BitRegClass);
1695 return std::make_pair(x: 0U, y: &SystemZ::ADDR32BitRegClass);
1696
1697 case 'h': // High-part register (an LLVM extension)
1698 return std::make_pair(x: 0U, y: &SystemZ::GRH32BitRegClass);
1699
1700 case 'f': // Floating-point register
1701 if (!useSoftFloat()) {
1702 if (VT.getSizeInBits() == 16)
1703 return std::make_pair(x: 0U, y: &SystemZ::FP16BitRegClass);
1704 else if (VT.getSizeInBits() == 64)
1705 return std::make_pair(x: 0U, y: &SystemZ::FP64BitRegClass);
1706 else if (VT.getSizeInBits() == 128)
1707 return std::make_pair(x: 0U, y: &SystemZ::FP128BitRegClass);
1708 return std::make_pair(x: 0U, y: &SystemZ::FP32BitRegClass);
1709 }
1710 break;
1711
1712 case 'v': // Vector register
1713 if (Subtarget.hasVector()) {
1714 if (VT.getSizeInBits() == 16)
1715 return std::make_pair(x: 0U, y: &SystemZ::VR16BitRegClass);
1716 if (VT.getSizeInBits() == 32)
1717 return std::make_pair(x: 0U, y: &SystemZ::VR32BitRegClass);
1718 if (VT.getSizeInBits() == 64)
1719 return std::make_pair(x: 0U, y: &SystemZ::VR64BitRegClass);
1720 return std::make_pair(x: 0U, y: &SystemZ::VR128BitRegClass);
1721 }
1722 break;
1723 }
1724 }
1725 if (Constraint.starts_with(Prefix: "{")) {
1726
1727 // A clobber constraint (e.g. ~{f0}) will have MVT::Other which is illegal
1728 // to check the size on.
1729 auto getVTSizeInBits = [&VT]() {
1730 return VT == MVT::Other ? 0 : VT.getSizeInBits();
1731 };
1732
1733 // We need to override the default register parsing for GPRs and FPRs
1734 // because the interpretation depends on VT. The internal names of
1735 // the registers are also different from the external names
1736 // (F0D and F0S instead of F0, etc.).
1737 if (Constraint[1] == 'r') {
1738 if (getVTSizeInBits() == 32)
1739 return parseRegisterNumber(Constraint, RC: &SystemZ::GR32BitRegClass,
1740 Map: SystemZMC::GR32Regs, Size: 16);
1741 if (getVTSizeInBits() == 128)
1742 return parseRegisterNumber(Constraint, RC: &SystemZ::GR128BitRegClass,
1743 Map: SystemZMC::GR128Regs, Size: 16);
1744 return parseRegisterNumber(Constraint, RC: &SystemZ::GR64BitRegClass,
1745 Map: SystemZMC::GR64Regs, Size: 16);
1746 }
1747 if (Constraint[1] == 'f') {
1748 if (useSoftFloat())
1749 return std::make_pair(
1750 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1751 if (getVTSizeInBits() == 16)
1752 return parseRegisterNumber(Constraint, RC: &SystemZ::FP16BitRegClass,
1753 Map: SystemZMC::FP16Regs, Size: 16);
1754 if (getVTSizeInBits() == 32)
1755 return parseRegisterNumber(Constraint, RC: &SystemZ::FP32BitRegClass,
1756 Map: SystemZMC::FP32Regs, Size: 16);
1757 if (getVTSizeInBits() == 128)
1758 return parseRegisterNumber(Constraint, RC: &SystemZ::FP128BitRegClass,
1759 Map: SystemZMC::FP128Regs, Size: 16);
1760 return parseRegisterNumber(Constraint, RC: &SystemZ::FP64BitRegClass,
1761 Map: SystemZMC::FP64Regs, Size: 16);
1762 }
1763 if (Constraint[1] == 'v') {
1764 if (!Subtarget.hasVector())
1765 return std::make_pair(
1766 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1767 if (getVTSizeInBits() == 16)
1768 return parseRegisterNumber(Constraint, RC: &SystemZ::VR16BitRegClass,
1769 Map: SystemZMC::VR16Regs, Size: 32);
1770 if (getVTSizeInBits() == 32)
1771 return parseRegisterNumber(Constraint, RC: &SystemZ::VR32BitRegClass,
1772 Map: SystemZMC::VR32Regs, Size: 32);
1773 if (getVTSizeInBits() == 64)
1774 return parseRegisterNumber(Constraint, RC: &SystemZ::VR64BitRegClass,
1775 Map: SystemZMC::VR64Regs, Size: 32);
1776 return parseRegisterNumber(Constraint, RC: &SystemZ::VR128BitRegClass,
1777 Map: SystemZMC::VR128Regs, Size: 32);
1778 }
1779 if (Constraint[1] == '@') {
1780 if (StringRef("{@cc}").compare(RHS: Constraint) == 0)
1781 return std::make_pair(x: SystemZ::CC, y: &SystemZ::CCRRegClass);
1782 }
1783 }
1784 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
1785}
1786
1787// FIXME? Maybe this could be a TableGen attribute on some registers and
1788// this table could be generated automatically from RegInfo.
1789Register
1790SystemZTargetLowering::getRegisterByName(const char *RegName, LLT VT,
1791 const MachineFunction &MF) const {
1792 Register Reg =
1793 StringSwitch<Register>(RegName)
1794 .Case(S: "r4", Value: Subtarget.isTargetXPLINK64() ? SystemZ::R4D
1795 : SystemZ::NoRegister)
1796 .Case(S: "r15",
1797 Value: Subtarget.isTargetELF() ? SystemZ::R15D : SystemZ::NoRegister)
1798 .Default(Value: Register());
1799
1800 return Reg;
1801}
1802
1803Register SystemZTargetLowering::getExceptionPointerRegister(
1804 ExceptionHandling EH, const Constant *PersonalityFn) const {
1805 return Subtarget.isTargetXPLINK64() ? SystemZ::R1D : SystemZ::R6D;
1806}
1807
1808Register SystemZTargetLowering::getExceptionSelectorRegister(
1809 ExceptionHandling EH, const Constant *PersonalityFn) const {
1810 return Subtarget.isTargetXPLINK64() ? SystemZ::R2D : SystemZ::R7D;
1811}
1812
1813// Convert condition code in CCReg to an i32 value.
1814static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg) {
1815 SDLoc DL(CCReg);
1816 SDValue IPM = DAG.getNode(Opcode: SystemZISD::IPM, DL, VT: MVT::i32, Operand: CCReg);
1817 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: IPM,
1818 N2: DAG.getConstant(Val: SystemZ::IPM_CC, DL, VT: MVT::i32));
1819}
1820
1821// Lower @cc targets via setcc.
1822SDValue SystemZTargetLowering::LowerAsmOutputForConstraint(
1823 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
1824 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
1825 if (StringRef("{@cc}").compare(RHS: OpInfo.ConstraintCode) != 0)
1826 return SDValue();
1827
1828 // Check that return type is valid.
1829 if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
1830 OpInfo.ConstraintVT.getSizeInBits() < 8)
1831 report_fatal_error(reason: "Glue output operand is of invalid type");
1832
1833 if (Glue.getNode()) {
1834 Glue = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::CC, VT: MVT::i32, Glue);
1835 Chain = Glue.getValue(R: 1);
1836 } else
1837 Glue = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::CC, VT: MVT::i32);
1838 return getCCResult(DAG, CCReg: Glue);
1839}
1840
1841void SystemZTargetLowering::LowerAsmOperandForConstraint(
1842 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
1843 SelectionDAG &DAG) const {
1844 // Only support length 1 constraints for now.
1845 if (Constraint.size() == 1) {
1846 switch (Constraint[0]) {
1847 case 'I': // Unsigned 8-bit constant
1848 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1849 if (isUInt<8>(x: C->getZExtValue()))
1850 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1851 VT: Op.getValueType()));
1852 return;
1853
1854 case 'J': // Unsigned 12-bit constant
1855 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1856 if (isUInt<12>(x: C->getZExtValue()))
1857 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1858 VT: Op.getValueType()));
1859 return;
1860
1861 case 'K': // Signed 16-bit constant
1862 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1863 if (isInt<16>(x: C->getSExtValue()))
1864 Ops.push_back(x: DAG.getSignedTargetConstant(
1865 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1866 return;
1867
1868 case 'L': // Signed 20-bit displacement (on all targets we support)
1869 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1870 if (isInt<20>(x: C->getSExtValue()))
1871 Ops.push_back(x: DAG.getSignedTargetConstant(
1872 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1873 return;
1874
1875 case 'M': // 0x7fffffff
1876 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1877 if (C->getZExtValue() == 0x7fffffff)
1878 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1879 VT: Op.getValueType()));
1880 return;
1881 }
1882 }
1883 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
1884}
1885
1886//===----------------------------------------------------------------------===//
1887// Calling conventions
1888//===----------------------------------------------------------------------===//
1889
1890#define GET_CALLING_CONV_IMPL
1891#include "SystemZGenCallingConv.inc"
1892
1893const MCPhysReg *SystemZTargetLowering::getScratchRegisters(
1894 CallingConv::ID) const {
1895 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1896 SystemZ::R14D, 0 };
1897 return ScratchRegs;
1898}
1899
1900bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType,
1901 Type *ToType) const {
1902 return isTruncateFree(FromType, ToType);
1903}
1904
1905bool SystemZTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
1906 return CI->isTailCall();
1907}
1908
1909// Value is a value that has been passed to us in the location described by VA
1910// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining
1911// any loads onto Chain.
1912static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
1913 CCValAssign &VA, SDValue Chain,
1914 SDValue Value) {
1915 // If the argument has been promoted from a smaller type, insert an
1916 // assertion to capture this.
1917 if (VA.getLocInfo() == CCValAssign::SExt)
1918 Value = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Value,
1919 N2: DAG.getValueType(VA.getValVT()));
1920 else if (VA.getLocInfo() == CCValAssign::ZExt)
1921 Value = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Value,
1922 N2: DAG.getValueType(VA.getValVT()));
1923
1924 if (VA.isExtInLoc())
1925 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Value);
1926 else if (VA.getLocInfo() == CCValAssign::BCvt) {
1927 // If this is a short vector argument loaded from the stack,
1928 // extend from i64 to full vector size and then bitcast.
1929 assert(VA.getLocVT() == MVT::i64);
1930 assert(VA.getValVT().isVector());
1931 Value = DAG.getBuildVector(VT: MVT::v2i64, DL, Ops: {Value, DAG.getUNDEF(VT: MVT::i64)});
1932 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Value);
1933 } else
1934 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo");
1935 return Value;
1936}
1937
1938// Value is a value of type VA.getValVT() that we need to copy into
1939// the location described by VA. Return a copy of Value converted to
1940// VA.getValVT(). The caller is responsible for handling indirect values.
1941static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
1942 CCValAssign &VA, SDValue Value) {
1943 switch (VA.getLocInfo()) {
1944 case CCValAssign::SExt:
1945 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1946 case CCValAssign::ZExt:
1947 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1948 case CCValAssign::AExt:
1949 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1950 case CCValAssign::BCvt: {
1951 assert(VA.getLocVT() == MVT::i64 || VA.getLocVT() == MVT::i128);
1952 assert(VA.getValVT().isVector() || VA.getValVT() == MVT::f32 ||
1953 VA.getValVT() == MVT::f64 || VA.getValVT() == MVT::f128);
1954 // For an f32 vararg we need to first promote it to an f64 and then
1955 // bitcast it to an i64.
1956 if (VA.getValVT() == MVT::f32 && VA.getLocVT() == MVT::i64)
1957 Value = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f64, Operand: Value);
1958 MVT BitCastToType = VA.getValVT().isVector() && VA.getLocVT() == MVT::i64
1959 ? MVT::v2i64
1960 : VA.getLocVT();
1961 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: BitCastToType, Operand: Value);
1962 // For ELF, this is a short vector argument to be stored to the stack,
1963 // bitcast to v2i64 and then extract first element.
1964 if (BitCastToType == MVT::v2i64)
1965 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: VA.getLocVT(), N1: Value,
1966 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
1967 return Value;
1968 }
1969 case CCValAssign::Full:
1970 return Value;
1971 default:
1972 llvm_unreachable("Unhandled getLocInfo()");
1973 }
1974}
1975
1976static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
1977 SDLoc DL(In);
1978 SDValue Lo, Hi;
1979 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
1980 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64, Operand: In);
1981 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64,
1982 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i128, N1: In,
1983 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32)));
1984 } else {
1985 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: In, DL, LoVT: MVT::i64, HiVT: MVT::i64);
1986 }
1987
1988 // FIXME: If v2i64 were a legal type, we could use it instead of
1989 // Untyped here. This might enable improved folding.
1990 SDNode *Pair = DAG.getMachineNode(Opcode: SystemZ::PAIR128, dl: DL,
1991 VT: MVT::Untyped, Op1: Hi, Op2: Lo);
1992 return SDValue(Pair, 0);
1993}
1994
1995static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
1996 SDLoc DL(In);
1997 SDValue Hi = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64,
1998 DL, VT: MVT::i64, Operand: In);
1999 SDValue Lo = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64,
2000 DL, VT: MVT::i64, Operand: In);
2001
2002 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
2003 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Lo);
2004 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Hi);
2005 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i128, N1: Hi,
2006 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32));
2007 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
2008 } else {
2009 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
2010 }
2011}
2012
2013bool SystemZTargetLowering::splitValueIntoRegisterParts(
2014 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
2015 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
2016 EVT ValueVT = Val.getValueType();
2017 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
2018 // Inline assembly operand.
2019 Parts[0] = lowerI128ToGR128(DAG, In: DAG.getBitcast(VT: MVT::i128, V: Val));
2020 return true;
2021 }
2022
2023 return false;
2024}
2025
2026SDValue SystemZTargetLowering::joinRegisterPartsIntoValue(
2027 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
2028 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
2029 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
2030 // Inline assembly operand.
2031 SDValue Res = lowerGR128ToI128(DAG, In: Parts[0]);
2032 return DAG.getBitcast(VT: ValueVT, V: Res);
2033 }
2034
2035 return SDValue();
2036}
2037
2038// The first part of a split stack argument is at index I in Args (and
2039// ArgLocs). Return the type of a part and the number of them by reference.
2040template <class ArgTy>
2041static bool analyzeArgSplit(const SmallVectorImpl<ArgTy> &Args,
2042 SmallVector<CCValAssign, 16> &ArgLocs, unsigned I,
2043 MVT &PartVT, unsigned &NumParts) {
2044 if (!Args[I].Flags.isSplit())
2045 return false;
2046 assert(I < ArgLocs.size() && ArgLocs.size() == Args.size() &&
2047 "ArgLocs havoc.");
2048 PartVT = ArgLocs[I].getValVT();
2049 NumParts = 1;
2050 for (unsigned PartIdx = I + 1;; ++PartIdx) {
2051 assert(PartIdx != ArgLocs.size() && "SplitEnd not found.");
2052 assert(ArgLocs[PartIdx].getValVT() == PartVT && "Unsupported split.");
2053 ++NumParts;
2054 if (Args[PartIdx].Flags.isSplitEnd())
2055 break;
2056 }
2057 return true;
2058}
2059
2060SDValue SystemZTargetLowering::LowerFormalArguments(
2061 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
2062 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2063 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2064 MachineFunction &MF = DAG.getMachineFunction();
2065 MachineFrameInfo &MFI = MF.getFrameInfo();
2066 MachineRegisterInfo &MRI = MF.getRegInfo();
2067 SystemZMachineFunctionInfo *FuncInfo =
2068 MF.getInfo<SystemZMachineFunctionInfo>();
2069 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
2070 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
2071
2072 // Assign locations to all of the incoming arguments.
2073 SmallVector<CCValAssign, 16> ArgLocs;
2074 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2075 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_SystemZ);
2076 FuncInfo->setSizeOfFnParams(CCInfo.getStackSize());
2077
2078 unsigned NumFixedGPRs = 0;
2079 unsigned NumFixedFPRs = 0;
2080 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2081 SDValue ArgValue;
2082 CCValAssign &VA = ArgLocs[I];
2083 EVT LocVT = VA.getLocVT();
2084 if (VA.isRegLoc()) {
2085 // Arguments passed in registers
2086 const TargetRegisterClass *RC;
2087 switch (LocVT.getSimpleVT().SimpleTy) {
2088 default:
2089 // Integers smaller than i64 should be promoted to i64.
2090 llvm_unreachable("Unexpected argument type");
2091 case MVT::i32:
2092 NumFixedGPRs += 1;
2093 RC = &SystemZ::GR32BitRegClass;
2094 break;
2095 case MVT::i64:
2096 NumFixedGPRs += 1;
2097 RC = &SystemZ::GR64BitRegClass;
2098 break;
2099 case MVT::f16:
2100 NumFixedFPRs += 1;
2101 RC = &SystemZ::FP16BitRegClass;
2102 break;
2103 case MVT::f32:
2104 NumFixedFPRs += 1;
2105 RC = &SystemZ::FP32BitRegClass;
2106 break;
2107 case MVT::f64:
2108 NumFixedFPRs += 1;
2109 RC = &SystemZ::FP64BitRegClass;
2110 break;
2111 case MVT::f128:
2112 NumFixedFPRs += 2;
2113 RC = &SystemZ::FP128BitRegClass;
2114 break;
2115 case MVT::v16i8:
2116 case MVT::v8i16:
2117 case MVT::v4i32:
2118 case MVT::v2i64:
2119 case MVT::v8f16:
2120 case MVT::v4f32:
2121 case MVT::v2f64:
2122 RC = &SystemZ::VR128BitRegClass;
2123 break;
2124 }
2125
2126 Register VReg = MRI.createVirtualRegister(RegClass: RC);
2127 MRI.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
2128 ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
2129 } else {
2130 assert(VA.isMemLoc() && "Argument not register or memory");
2131
2132 // Create the frame index object for this incoming parameter.
2133 // FIXME: Pre-include call frame size in the offset, should not
2134 // need to manually add it here.
2135 int64_t ArgSPOffset = VA.getLocMemOffset();
2136 if (Subtarget.isTargetXPLINK64()) {
2137 auto &XPRegs =
2138 Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
2139 ArgSPOffset += XPRegs.getCallFrameSize();
2140 }
2141 int FI =
2142 MFI.CreateFixedObject(Size: LocVT.getSizeInBits() / 8, SPOffset: ArgSPOffset, IsImmutable: true);
2143
2144 // Create the SelectionDAG nodes corresponding to a load
2145 // from this parameter. Unpromoted ints and floats are
2146 // passed as right-justified 8-byte values.
2147 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
2148 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32 ||
2149 VA.getLocVT() == MVT::f16) {
2150 unsigned SlotOffs = VA.getLocVT() == MVT::f16 ? 6 : 4;
2151 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FIN,
2152 N2: DAG.getIntPtrConstant(Val: SlotOffs, DL));
2153 }
2154 ArgValue = DAG.getLoad(VT: LocVT, dl: DL, Chain, Ptr: FIN,
2155 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2156 }
2157
2158 // Convert the value of the argument register into the value that's
2159 // being passed.
2160 if (VA.getLocInfo() == CCValAssign::Indirect) {
2161 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
2162 PtrInfo: MachinePointerInfo()));
2163 // If the original argument was split (e.g. i128), we need
2164 // to load all parts of it here (using the same address).
2165 MVT PartVT;
2166 unsigned NumParts;
2167 if (analyzeArgSplit(Args: Ins, ArgLocs, I, PartVT, NumParts)) {
2168 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2169 ++I;
2170 CCValAssign &PartVA = ArgLocs[I];
2171 unsigned PartOffset = Ins[I].PartOffset;
2172 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue,
2173 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2174 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
2175 PtrInfo: MachinePointerInfo()));
2176 assert(PartOffset && "Offset should be non-zero.");
2177 }
2178 }
2179 } else if (Subtarget.isTargetXPLINK64() &&
2180 (VA.getLocInfo() == CCValAssign::SExt ||
2181 VA.getLocInfo() == CCValAssign::ZExt) &&
2182 Ins[I].ArgVT.isSimple()) {
2183 // Some prior z/OS compilers do not always perform the extension of
2184 // short integer arguments or pointers. To accommodate those, do not
2185 // rely on that extension by avoiding any AssertSext/AssertZext nodes by
2186 // directly truncating ArgValue to the original argument type.
2187 MVT OrigVT = Ins[I].ArgVT.getSimpleVT();
2188 InVals.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OrigVT, Operand: ArgValue));
2189 } else
2190 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: ArgValue));
2191 }
2192
2193 if (IsVarArg && Subtarget.isTargetXPLINK64()) {
2194 // Save the number of non-varargs registers for later use by va_start, etc.
2195 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2196 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2197
2198 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2199 Subtarget.getSpecialRegisters());
2200
2201 // Likewise the address (in the form of a frame index) of where the
2202 // first stack vararg would be. The 1-byte size here is arbitrary.
2203 // FIXME: Pre-include call frame size in the offset, should not
2204 // need to manually add it here.
2205 int64_t VarArgOffset = CCInfo.getStackSize() + Regs->getCallFrameSize();
2206 int FI = MFI.CreateFixedObject(Size: 1, SPOffset: VarArgOffset, IsImmutable: true);
2207 FuncInfo->setVarArgsFrameIndex(FI);
2208 }
2209
2210 if (IsVarArg && Subtarget.isTargetELF()) {
2211 // Save the number of non-varargs registers for later use by va_start, etc.
2212 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2213 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2214
2215 // Likewise the address (in the form of a frame index) of where the
2216 // first stack vararg would be. The 1-byte size here is arbitrary.
2217 int64_t VarArgsOffset = CCInfo.getStackSize();
2218 FuncInfo->setVarArgsFrameIndex(
2219 MFI.CreateFixedObject(Size: 1, SPOffset: VarArgsOffset, IsImmutable: true));
2220
2221 // ...and a similar frame index for the caller-allocated save area
2222 // that will be used to store the incoming registers.
2223 int64_t RegSaveOffset =
2224 -SystemZMC::ELFCallFrameSize + TFL->getRegSpillOffset(MF, Reg: SystemZ::R2D) - 16;
2225 unsigned RegSaveIndex = MFI.CreateFixedObject(Size: 1, SPOffset: RegSaveOffset, IsImmutable: true);
2226 FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
2227
2228 // Store the FPR varargs in the reserved frame slots. (We store the
2229 // GPRs as part of the prologue.)
2230 if (NumFixedFPRs < SystemZ::ELFNumArgFPRs && !useSoftFloat()) {
2231 SDValue MemOps[SystemZ::ELFNumArgFPRs];
2232 for (unsigned I = NumFixedFPRs; I < SystemZ::ELFNumArgFPRs; ++I) {
2233 unsigned Offset = TFL->getRegSpillOffset(MF, Reg: SystemZ::ELFArgFPRs[I]);
2234 int FI =
2235 MFI.CreateFixedObject(Size: 8, SPOffset: -SystemZMC::ELFCallFrameSize + Offset, IsImmutable: true);
2236 SDValue FIN = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
2237 Register VReg = MF.addLiveIn(PReg: SystemZ::ELFArgFPRs[I],
2238 RC: &SystemZ::FP64BitRegClass);
2239 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::f64);
2240 MemOps[I] = DAG.getStore(Chain: ArgValue.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: FIN,
2241 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2242 }
2243 // Join the stores, which are independent of one another.
2244 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
2245 Ops: ArrayRef(&MemOps[NumFixedFPRs],
2246 SystemZ::ELFNumArgFPRs - NumFixedFPRs));
2247 }
2248 }
2249
2250 if (Subtarget.isTargetXPLINK64()) {
2251 // Create virual register for handling incoming "ADA" special register (R5)
2252 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
2253 Register ADAvReg = MRI.createVirtualRegister(RegClass: RC);
2254 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2255 Subtarget.getSpecialRegisters());
2256 MRI.addLiveIn(Reg: Regs->getADARegister(), vreg: ADAvReg);
2257 FuncInfo->setADAVirtualRegister(ADAvReg);
2258 }
2259 return Chain;
2260}
2261
2262static bool canUseSiblingCall(const CCState &ArgCCInfo,
2263 SmallVectorImpl<CCValAssign> &ArgLocs,
2264 SmallVectorImpl<ISD::OutputArg> &Outs) {
2265 // Punt if there are any indirect or stack arguments, or if the call
2266 // needs the callee-saved argument register R6, or if the call uses
2267 // the callee-saved register arguments SwiftSelf and SwiftError.
2268 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2269 CCValAssign &VA = ArgLocs[I];
2270 if (VA.getLocInfo() == CCValAssign::Indirect)
2271 return false;
2272 if (!VA.isRegLoc())
2273 return false;
2274 Register Reg = VA.getLocReg();
2275 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
2276 return false;
2277 if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError())
2278 return false;
2279 }
2280 return true;
2281}
2282
2283static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL,
2284 unsigned Offset, bool LoadAdr = false) {
2285 MachineFunction &MF = DAG.getMachineFunction();
2286 SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
2287 Register ADAvReg = MFI->getADAVirtualRegister();
2288 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2289
2290 SDValue Reg = DAG.getRegister(Reg: ADAvReg, VT: PtrVT);
2291 SDValue Ofs = DAG.getTargetConstant(Val: Offset, DL, VT: PtrVT);
2292
2293 SDValue Result = DAG.getNode(Opcode: SystemZISD::ADA_ENTRY, DL, VT: PtrVT, N1: Val, N2: Reg, N3: Ofs);
2294 if (!LoadAdr)
2295 Result = DAG.getLoad(
2296 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result, PtrInfo: MachinePointerInfo(), Alignment: Align(8),
2297 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2298
2299 return Result;
2300}
2301
2302// ADA access using Global value
2303// Note: for functions, address of descriptor is returned
2304static SDValue getADAEntry(SelectionDAG &DAG, const GlobalValue *GV, SDLoc DL,
2305 EVT PtrVT) {
2306 unsigned ADAtype;
2307 bool LoadAddr = false;
2308 const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: GV);
2309 bool IsFunction =
2310 (isa<Function>(Val: GV)) || (GA && isa<Function>(Val: GA->getAliaseeObject()));
2311 bool IsInternal = (GV->hasInternalLinkage() || GV->hasPrivateLinkage());
2312
2313 if (IsFunction) {
2314 if (IsInternal) {
2315 ADAtype = SystemZII::MO_ADA_DIRECT_FUNC_DESC;
2316 LoadAddr = true;
2317 } else
2318 ADAtype = SystemZII::MO_ADA_INDIRECT_FUNC_DESC;
2319 } else {
2320 ADAtype = SystemZII::MO_ADA_DATA_SYMBOL_ADDR;
2321 }
2322 SDValue Val = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: ADAtype);
2323
2324 return getADAEntry(DAG, Val, DL, Offset: 0, LoadAdr: LoadAddr);
2325}
2326
2327static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA,
2328 SDLoc &DL, SDValue &Chain) {
2329 unsigned ADADelta = 0; // ADA offset in desc.
2330 unsigned EPADelta = 8; // EPA offset in desc.
2331 MachineFunction &MF = DAG.getMachineFunction();
2332 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2333
2334 // XPLink calling convention.
2335 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2336 bool IsInternal = (G->getGlobal()->hasInternalLinkage() ||
2337 G->getGlobal()->hasPrivateLinkage());
2338 if (IsInternal) {
2339 SystemZMachineFunctionInfo *MFI =
2340 MF.getInfo<SystemZMachineFunctionInfo>();
2341 Register ADAvReg = MFI->getADAVirtualRegister();
2342 ADA = DAG.getCopyFromReg(Chain, dl: DL, Reg: ADAvReg, VT: PtrVT);
2343 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2344 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2345 return true;
2346 } else {
2347 SDValue GA = DAG.getTargetGlobalAddress(
2348 GV: G->getGlobal(), DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2349 ADA = getADAEntry(DAG, Val: GA, DL, Offset: ADADelta);
2350 Callee = getADAEntry(DAG, Val: GA, DL, Offset: EPADelta);
2351 }
2352 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2353 SDValue ES = DAG.getTargetExternalSymbol(
2354 Sym: E->getSymbol(), VT: PtrVT, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2355 ADA = getADAEntry(DAG, Val: ES, DL, Offset: ADADelta);
2356 Callee = getADAEntry(DAG, Val: ES, DL, Offset: EPADelta);
2357 } else {
2358 // Function pointer case
2359 ADA = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2360 N2: DAG.getConstant(Val: ADADelta, DL, VT: PtrVT));
2361 ADA = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: ADA,
2362 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2363 Callee = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2364 N2: DAG.getConstant(Val: EPADelta, DL, VT: PtrVT));
2365 Callee = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Callee,
2366 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2367 }
2368 return false;
2369}
2370
2371SDValue
2372SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
2373 SmallVectorImpl<SDValue> &InVals) const {
2374 SelectionDAG &DAG = CLI.DAG;
2375 SDLoc &DL = CLI.DL;
2376 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2377 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2378 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2379 SDValue Chain = CLI.Chain;
2380 SDValue Callee = CLI.Callee;
2381 bool &IsTailCall = CLI.IsTailCall;
2382 CallingConv::ID CallConv = CLI.CallConv;
2383 bool IsVarArg = CLI.IsVarArg;
2384 MachineFunction &MF = DAG.getMachineFunction();
2385 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
2386 LLVMContext &Ctx = *DAG.getContext();
2387 SystemZCallingConventionRegisters *Regs = Subtarget.getSpecialRegisters();
2388
2389 // FIXME: z/OS support to be added in later.
2390 if (Subtarget.isTargetXPLINK64())
2391 IsTailCall = false;
2392
2393 // Integer args <=32 bits should have an extension attribute.
2394 verifyNarrowIntegerArgs_Call(Outs, F: &MF.getFunction(), Callee);
2395
2396 // Analyze the operands of the call, assigning locations to each operand.
2397 SmallVector<CCValAssign, 16> ArgLocs;
2398 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
2399 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_SystemZ);
2400
2401 // We don't support GuaranteedTailCallOpt, only automatically-detected
2402 // sibling calls.
2403 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs))
2404 IsTailCall = false;
2405
2406 // Get a count of how many bytes are to be pushed on the stack.
2407 unsigned NumBytes = ArgCCInfo.getStackSize();
2408
2409 // Mark the start of the call.
2410 if (!IsTailCall)
2411 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL);
2412
2413 // Copy argument values to their designated locations.
2414 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass;
2415 SmallVector<SDValue, 8> MemOpChains;
2416 SDValue StackPtr;
2417 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2418 CCValAssign &VA = ArgLocs[I];
2419 SDValue ArgValue = OutVals[I];
2420
2421 if (VA.getLocInfo() == CCValAssign::Indirect) {
2422 // Store the argument in a stack slot and pass its address.
2423 EVT SlotVT;
2424 MVT PartVT;
2425 unsigned NumParts = 1;
2426 if (analyzeArgSplit(Args: Outs, ArgLocs, I, PartVT, NumParts))
2427 SlotVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: PartVT.getSizeInBits() * NumParts);
2428 else
2429 SlotVT = Outs[I].VT;
2430 SDValue SpillSlot = DAG.CreateStackTemporary(VT: SlotVT);
2431 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
2432
2433 MachinePointerInfo StackPtrInfo =
2434 MachinePointerInfo::getFixedStack(MF, FI);
2435 MemOpChains.push_back(
2436 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot, PtrInfo: StackPtrInfo));
2437 // If the original argument was split (e.g. i128), we need
2438 // to store all parts of it here (and pass just one address).
2439 assert(Outs[I].PartOffset == 0);
2440 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2441 ++I;
2442 SDValue PartValue = OutVals[I];
2443 unsigned PartOffset = Outs[I].PartOffset;
2444 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot,
2445 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2446 MemOpChains.push_back(
2447 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
2448 PtrInfo: StackPtrInfo.getWithOffset(O: PartOffset)));
2449 assert(PartOffset && "Offset should be non-zero.");
2450 assert((PartOffset + PartValue.getValueType().getStoreSize() <=
2451 SlotVT.getStoreSize()) && "Not enough space for argument part!");
2452 }
2453 ArgValue = SpillSlot;
2454 } else
2455 ArgValue = convertValVTToLocVT(DAG, DL, VA, Value: ArgValue);
2456
2457 if (VA.isRegLoc()) {
2458 // In XPLINK64, for the 128-bit vararg case, ArgValue is bitcasted to a
2459 // MVT::i128 type. We decompose the 128-bit type to a pair of its high
2460 // and low values.
2461 if (VA.getLocVT() == MVT::i128)
2462 ArgValue = lowerI128ToGR128(DAG, In: ArgValue);
2463 // Queue up the argument copies and emit them at the end.
2464 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
2465 } else {
2466 assert(VA.isMemLoc() && "Argument not register or memory");
2467
2468 // Work out the address of the stack slot. Unpromoted ints and
2469 // floats are passed as right-justified 8-byte values.
2470 if (!StackPtr.getNode())
2471 StackPtr = DAG.getCopyFromReg(Chain, dl: DL,
2472 Reg: Regs->getStackPointerRegister(), VT: PtrVT);
2473 unsigned Offset = Regs->getStackPointerBias() + Regs->getCallFrameSize() +
2474 VA.getLocMemOffset();
2475 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
2476 Offset += 4;
2477 else if (VA.getLocVT() == MVT::f16)
2478 Offset += 6;
2479 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
2480 N2: DAG.getIntPtrConstant(Val: Offset, DL));
2481
2482 // Emit the store.
2483 MemOpChains.push_back(
2484 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address, PtrInfo: MachinePointerInfo()));
2485
2486 // Although long doubles or vectors are passed through the stack when
2487 // they are vararg (non-fixed arguments), if a long double or vector
2488 // occupies the third and fourth slot of the argument list GPR3 should
2489 // still shadow the third slot of the argument list.
2490 if (Subtarget.isTargetXPLINK64() && VA.needsCustom()) {
2491 SDValue ShadowArgValue =
2492 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: ArgValue,
2493 N2: DAG.getIntPtrConstant(Val: 1, DL));
2494 RegsToPass.push_back(Elt: std::make_pair(x: SystemZ::R3D, y&: ShadowArgValue));
2495 }
2496 }
2497 }
2498
2499 // Join the stores, which are independent of one another.
2500 if (!MemOpChains.empty())
2501 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
2502
2503 // Accept direct calls by converting symbolic call addresses to the
2504 // associated Target* opcodes. Force %r1 to be used for indirect
2505 // tail calls.
2506 SDValue Glue;
2507
2508 if (Subtarget.isTargetXPLINK64()) {
2509 SDValue ADA;
2510 bool IsBRASL = getzOSCalleeAndADA(DAG, Callee, ADA, DL, Chain);
2511 if (!IsBRASL) {
2512 unsigned CalleeReg = static_cast<SystemZXPLINK64Registers *>(Regs)
2513 ->getAddressOfCalleeRegister();
2514 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CalleeReg, N: Callee, Glue);
2515 Glue = Chain.getValue(R: 1);
2516 Callee = DAG.getRegister(Reg: CalleeReg, VT: Callee.getValueType());
2517 }
2518 RegsToPass.push_back(Elt: std::make_pair(
2519 x: static_cast<SystemZXPLINK64Registers *>(Regs)->getADARegister(), y&: ADA));
2520 } else {
2521 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2522 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2523 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2524 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2525 Callee = DAG.getTargetExternalSymbol(Sym: E->getSymbol(), VT: PtrVT);
2526 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2527 } else if (IsTailCall) {
2528 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R1D, N: Callee, Glue);
2529 Glue = Chain.getValue(R: 1);
2530 Callee = DAG.getRegister(Reg: SystemZ::R1D, VT: Callee.getValueType());
2531 }
2532 }
2533
2534 // Build a sequence of copy-to-reg nodes, chained and glued together.
2535 for (const auto &[Reg, N] : RegsToPass) {
2536 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N, Glue);
2537 Glue = Chain.getValue(R: 1);
2538 }
2539
2540 // The first call operand is the chain and the second is the target address.
2541 SmallVector<SDValue, 8> Ops;
2542 Ops.push_back(Elt: Chain);
2543 Ops.push_back(Elt: Callee);
2544
2545 // Add argument registers to the end of the list so that they are
2546 // known live into the call.
2547 for (const auto &[Reg, N] : RegsToPass)
2548 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
2549
2550 // Add a register mask operand representing the call-preserved registers.
2551 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2552 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2553 assert(Mask && "Missing call preserved mask for calling convention");
2554 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
2555
2556 // Glue the call to the argument copies, if any.
2557 if (Glue.getNode())
2558 Ops.push_back(Elt: Glue);
2559
2560 // Emit the call.
2561 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2562 if (IsTailCall) {
2563 SDValue Ret = DAG.getNode(Opcode: SystemZISD::SIBCALL, DL, VTList: NodeTys, Ops);
2564 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
2565 return Ret;
2566 }
2567 Chain = DAG.getNode(Opcode: SystemZISD::CALL, DL, VTList: NodeTys, Ops);
2568 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
2569 Glue = Chain.getValue(R: 1);
2570
2571 // Mark the end of the call, which is glued to the call itself.
2572 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
2573 Glue = Chain.getValue(R: 1);
2574
2575 // Assign locations to each value returned by this call.
2576 SmallVector<CCValAssign, 16> RetLocs;
2577 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
2578 RetCCInfo.AnalyzeCallResult(Ins, Fn: RetCC_SystemZ);
2579
2580 // Copy all of the result registers out of their specified physreg.
2581 for (CCValAssign &VA : RetLocs) {
2582 // Copy the value out, gluing the copy to the end of the call sequence.
2583 SDValue RetValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(),
2584 VT: VA.getLocVT(), Glue);
2585 Chain = RetValue.getValue(R: 1);
2586 Glue = RetValue.getValue(R: 2);
2587
2588 // Convert the value of the return register into the value that's
2589 // being returned.
2590 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: RetValue));
2591 }
2592
2593 return Chain;
2594}
2595
2596// Generate a call taking the given operands as arguments and returning a
2597// result of type RetVT.
2598std::pair<SDValue, SDValue> SystemZTargetLowering::makeExternalCall(
2599 SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT,
2600 ArrayRef<SDValue> Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL,
2601 bool DoesNotReturn, bool IsReturnValueUsed) const {
2602 TargetLowering::ArgListTy Args;
2603 Args.reserve(n: Ops.size());
2604
2605 for (SDValue Op : Ops) {
2606 TargetLowering::ArgListEntry Entry(
2607 Op, Op.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
2608 Entry.IsSExt = shouldSignExtendTypeInLibCall(Ty: Entry.Ty, IsSigned);
2609 Entry.IsZExt = !Entry.IsSExt;
2610 Args.push_back(x: Entry);
2611 }
2612
2613 SDValue Callee =
2614 DAG.getExternalSymbol(Sym: CalleeName, VT: getPointerTy(DL: DAG.getDataLayout()));
2615
2616 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2617 TargetLowering::CallLoweringInfo CLI(DAG);
2618 bool SignExtend = shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2619 CLI.setDebugLoc(DL)
2620 .setChain(Chain)
2621 .setCallee(CC: CallConv, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2622 .setNoReturn(DoesNotReturn)
2623 .setDiscardResult(!IsReturnValueUsed)
2624 .setSExtResult(SignExtend)
2625 .setZExtResult(!SignExtend);
2626 return LowerCallTo(CLI);
2627}
2628
2629bool SystemZTargetLowering::CanLowerReturn(
2630 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2631 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
2632 const Type *RetTy) const {
2633 // Special case that we cannot easily detect in RetCC_SystemZ since
2634 // i128 may not be a legal type.
2635 for (auto &Out : Outs)
2636 if (Out.ArgVT.isScalarInteger() && Out.ArgVT.getSizeInBits() > 64)
2637 return false;
2638
2639 SmallVector<CCValAssign, 16> RetLocs;
2640 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Context);
2641 return RetCCInfo.CheckReturn(Outs, Fn: RetCC_SystemZ);
2642}
2643
2644SDValue
2645SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2646 bool IsVarArg,
2647 const SmallVectorImpl<ISD::OutputArg> &Outs,
2648 const SmallVectorImpl<SDValue> &OutVals,
2649 const SDLoc &DL, SelectionDAG &DAG) const {
2650 MachineFunction &MF = DAG.getMachineFunction();
2651
2652 // Integer args <=32 bits should have an extension attribute.
2653 verifyNarrowIntegerArgs_Ret(Outs, F: &MF.getFunction());
2654
2655 // Assign locations to each returned value.
2656 SmallVector<CCValAssign, 16> RetLocs;
2657 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
2658 RetCCInfo.AnalyzeReturn(Outs, Fn: RetCC_SystemZ);
2659
2660 // Quick exit for void returns
2661 if (RetLocs.empty())
2662 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Operand: Chain);
2663
2664 if (CallConv == CallingConv::GHC)
2665 report_fatal_error(reason: "GHC functions return void only");
2666
2667 // Copy the result values into the output registers.
2668 SDValue Glue;
2669 SmallVector<SDValue, 4> RetOps;
2670 RetOps.push_back(Elt: Chain);
2671 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
2672 CCValAssign &VA = RetLocs[I];
2673 SDValue RetValue = OutVals[I];
2674
2675 // Make the return register live on exit.
2676 assert(VA.isRegLoc() && "Can only return in registers!");
2677
2678 // Promote the value as required.
2679 RetValue = convertValVTToLocVT(DAG, DL, VA, Value: RetValue);
2680
2681 // Chain and glue the copies together.
2682 Register Reg = VA.getLocReg();
2683 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: RetValue, Glue);
2684 Glue = Chain.getValue(R: 1);
2685 RetOps.push_back(Elt: DAG.getRegister(Reg, VT: VA.getLocVT()));
2686 }
2687
2688 // Update chain and glue.
2689 RetOps[0] = Chain;
2690 if (Glue.getNode())
2691 RetOps.push_back(Elt: Glue);
2692
2693 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Ops: RetOps);
2694}
2695
2696// Return true if Op is an intrinsic node with chain that returns the CC value
2697// as its only (other) argument. Provide the associated SystemZISD opcode and
2698// the mask of valid CC values if so.
2699static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
2700 unsigned &CCValid) {
2701 unsigned Id = Op.getConstantOperandVal(i: 1);
2702 switch (Id) {
2703 case Intrinsic::s390_tbegin:
2704 Opcode = SystemZISD::TBEGIN;
2705 CCValid = SystemZ::CCMASK_TBEGIN;
2706 return true;
2707
2708 case Intrinsic::s390_tbegin_nofloat:
2709 Opcode = SystemZISD::TBEGIN_NOFLOAT;
2710 CCValid = SystemZ::CCMASK_TBEGIN;
2711 return true;
2712
2713 case Intrinsic::s390_tend:
2714 Opcode = SystemZISD::TEND;
2715 CCValid = SystemZ::CCMASK_TEND;
2716 return true;
2717
2718 default:
2719 return false;
2720 }
2721}
2722
2723// Return true if Op is an intrinsic node without chain that returns the
2724// CC value as its final argument. Provide the associated SystemZISD
2725// opcode and the mask of valid CC values if so.
2726static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
2727 unsigned Id = Op.getConstantOperandVal(i: 0);
2728 switch (Id) {
2729 case Intrinsic::s390_vpkshs:
2730 case Intrinsic::s390_vpksfs:
2731 case Intrinsic::s390_vpksgs:
2732 Opcode = SystemZISD::PACKS_CC;
2733 CCValid = SystemZ::CCMASK_VCMP;
2734 return true;
2735
2736 case Intrinsic::s390_vpklshs:
2737 case Intrinsic::s390_vpklsfs:
2738 case Intrinsic::s390_vpklsgs:
2739 Opcode = SystemZISD::PACKLS_CC;
2740 CCValid = SystemZ::CCMASK_VCMP;
2741 return true;
2742
2743 case Intrinsic::s390_vceqbs:
2744 case Intrinsic::s390_vceqhs:
2745 case Intrinsic::s390_vceqfs:
2746 case Intrinsic::s390_vceqgs:
2747 case Intrinsic::s390_vceqqs:
2748 Opcode = SystemZISD::VICMPES;
2749 CCValid = SystemZ::CCMASK_VCMP;
2750 return true;
2751
2752 case Intrinsic::s390_vchbs:
2753 case Intrinsic::s390_vchhs:
2754 case Intrinsic::s390_vchfs:
2755 case Intrinsic::s390_vchgs:
2756 case Intrinsic::s390_vchqs:
2757 Opcode = SystemZISD::VICMPHS;
2758 CCValid = SystemZ::CCMASK_VCMP;
2759 return true;
2760
2761 case Intrinsic::s390_vchlbs:
2762 case Intrinsic::s390_vchlhs:
2763 case Intrinsic::s390_vchlfs:
2764 case Intrinsic::s390_vchlgs:
2765 case Intrinsic::s390_vchlqs:
2766 Opcode = SystemZISD::VICMPHLS;
2767 CCValid = SystemZ::CCMASK_VCMP;
2768 return true;
2769
2770 case Intrinsic::s390_vtm:
2771 Opcode = SystemZISD::VTM;
2772 CCValid = SystemZ::CCMASK_VCMP;
2773 return true;
2774
2775 case Intrinsic::s390_vfaebs:
2776 case Intrinsic::s390_vfaehs:
2777 case Intrinsic::s390_vfaefs:
2778 Opcode = SystemZISD::VFAE_CC;
2779 CCValid = SystemZ::CCMASK_ANY;
2780 return true;
2781
2782 case Intrinsic::s390_vfaezbs:
2783 case Intrinsic::s390_vfaezhs:
2784 case Intrinsic::s390_vfaezfs:
2785 Opcode = SystemZISD::VFAEZ_CC;
2786 CCValid = SystemZ::CCMASK_ANY;
2787 return true;
2788
2789 case Intrinsic::s390_vfeebs:
2790 case Intrinsic::s390_vfeehs:
2791 case Intrinsic::s390_vfeefs:
2792 Opcode = SystemZISD::VFEE_CC;
2793 CCValid = SystemZ::CCMASK_ANY;
2794 return true;
2795
2796 case Intrinsic::s390_vfeezbs:
2797 case Intrinsic::s390_vfeezhs:
2798 case Intrinsic::s390_vfeezfs:
2799 Opcode = SystemZISD::VFEEZ_CC;
2800 CCValid = SystemZ::CCMASK_ANY;
2801 return true;
2802
2803 case Intrinsic::s390_vfenebs:
2804 case Intrinsic::s390_vfenehs:
2805 case Intrinsic::s390_vfenefs:
2806 Opcode = SystemZISD::VFENE_CC;
2807 CCValid = SystemZ::CCMASK_ANY;
2808 return true;
2809
2810 case Intrinsic::s390_vfenezbs:
2811 case Intrinsic::s390_vfenezhs:
2812 case Intrinsic::s390_vfenezfs:
2813 Opcode = SystemZISD::VFENEZ_CC;
2814 CCValid = SystemZ::CCMASK_ANY;
2815 return true;
2816
2817 case Intrinsic::s390_vistrbs:
2818 case Intrinsic::s390_vistrhs:
2819 case Intrinsic::s390_vistrfs:
2820 Opcode = SystemZISD::VISTR_CC;
2821 CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3;
2822 return true;
2823
2824 case Intrinsic::s390_vstrcbs:
2825 case Intrinsic::s390_vstrchs:
2826 case Intrinsic::s390_vstrcfs:
2827 Opcode = SystemZISD::VSTRC_CC;
2828 CCValid = SystemZ::CCMASK_ANY;
2829 return true;
2830
2831 case Intrinsic::s390_vstrczbs:
2832 case Intrinsic::s390_vstrczhs:
2833 case Intrinsic::s390_vstrczfs:
2834 Opcode = SystemZISD::VSTRCZ_CC;
2835 CCValid = SystemZ::CCMASK_ANY;
2836 return true;
2837
2838 case Intrinsic::s390_vstrsb:
2839 case Intrinsic::s390_vstrsh:
2840 case Intrinsic::s390_vstrsf:
2841 Opcode = SystemZISD::VSTRS_CC;
2842 CCValid = SystemZ::CCMASK_ANY;
2843 return true;
2844
2845 case Intrinsic::s390_vstrszb:
2846 case Intrinsic::s390_vstrszh:
2847 case Intrinsic::s390_vstrszf:
2848 Opcode = SystemZISD::VSTRSZ_CC;
2849 CCValid = SystemZ::CCMASK_ANY;
2850 return true;
2851
2852 case Intrinsic::s390_vfcedbs:
2853 case Intrinsic::s390_vfcesbs:
2854 Opcode = SystemZISD::VFCMPES;
2855 CCValid = SystemZ::CCMASK_VCMP;
2856 return true;
2857
2858 case Intrinsic::s390_vfchdbs:
2859 case Intrinsic::s390_vfchsbs:
2860 Opcode = SystemZISD::VFCMPHS;
2861 CCValid = SystemZ::CCMASK_VCMP;
2862 return true;
2863
2864 case Intrinsic::s390_vfchedbs:
2865 case Intrinsic::s390_vfchesbs:
2866 Opcode = SystemZISD::VFCMPHES;
2867 CCValid = SystemZ::CCMASK_VCMP;
2868 return true;
2869
2870 case Intrinsic::s390_vftcidb:
2871 case Intrinsic::s390_vftcisb:
2872 Opcode = SystemZISD::VFTCI;
2873 CCValid = SystemZ::CCMASK_VCMP;
2874 return true;
2875
2876 case Intrinsic::s390_tdc:
2877 Opcode = SystemZISD::TDC;
2878 CCValid = SystemZ::CCMASK_TDC;
2879 return true;
2880
2881 default:
2882 return false;
2883 }
2884}
2885
2886// Emit an intrinsic with chain and an explicit CC register result.
2887static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op,
2888 unsigned Opcode) {
2889 // Copy all operands except the intrinsic ID.
2890 unsigned NumOps = Op.getNumOperands();
2891 SmallVector<SDValue, 6> Ops;
2892 Ops.reserve(N: NumOps - 1);
2893 Ops.push_back(Elt: Op.getOperand(i: 0));
2894 for (unsigned I = 2; I < NumOps; ++I)
2895 Ops.push_back(Elt: Op.getOperand(i: I));
2896
2897 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
2898 SDVTList RawVTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
2899 SDValue Intr = DAG.getNode(Opcode, DL: SDLoc(Op), VTList: RawVTs, Ops);
2900 SDValue OldChain = SDValue(Op.getNode(), 1);
2901 SDValue NewChain = SDValue(Intr.getNode(), 1);
2902 DAG.ReplaceAllUsesOfValueWith(From: OldChain, To: NewChain);
2903 return Intr.getNode();
2904}
2905
2906// Emit an intrinsic with an explicit CC register result.
2907static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op,
2908 unsigned Opcode) {
2909 // Copy all operands except the intrinsic ID.
2910 SDLoc DL(Op);
2911 unsigned NumOps = Op.getNumOperands();
2912 SmallVector<SDValue, 6> Ops;
2913 Ops.reserve(N: NumOps - 1);
2914 for (unsigned I = 1; I < NumOps; ++I) {
2915 SDValue CurrOper = Op.getOperand(i: I);
2916 if (CurrOper.getValueType() == MVT::f16) {
2917 assert((Op.getConstantOperandVal(0) == Intrinsic::s390_tdc && I == 1) &&
2918 "Unhandled intrinsic with f16 operand.");
2919 CurrOper = DAG.getFPExtendOrRound(Op: CurrOper, DL, VT: MVT::f32);
2920 }
2921 Ops.push_back(Elt: CurrOper);
2922 }
2923
2924 SDValue Intr = DAG.getNode(Opcode, DL, VTList: Op->getVTList(), Ops);
2925 return Intr.getNode();
2926}
2927
2928// CC is a comparison that will be implemented using an integer or
2929// floating-point comparison. Return the condition code mask for
2930// a branch on true. In the integer case, CCMASK_CMP_UO is set for
2931// unsigned comparisons and clear for signed ones. In the floating-point
2932// case, CCMASK_CMP_UO has its normal mask meaning (unordered).
2933static unsigned CCMaskForCondCode(ISD::CondCode CC) {
2934#define CONV(X) \
2935 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
2936 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
2937 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
2938
2939 switch (CC) {
2940 default:
2941 llvm_unreachable("Invalid integer condition!");
2942
2943 CONV(EQ);
2944 CONV(NE);
2945 CONV(GT);
2946 CONV(GE);
2947 CONV(LT);
2948 CONV(LE);
2949
2950 case ISD::SETO: return SystemZ::CCMASK_CMP_O;
2951 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO;
2952 }
2953#undef CONV
2954}
2955
2956// If C can be converted to a comparison against zero, adjust the operands
2957// as necessary.
2958static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2959 if (C.ICmpType == SystemZICMP::UnsignedOnly)
2960 return;
2961
2962 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1.getNode());
2963 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
2964 return;
2965
2966 int64_t Value = ConstOp1->getSExtValue();
2967 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
2968 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
2969 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
2970 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
2971 C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
2972 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op1.getValueType());
2973 }
2974}
2975
2976// If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
2977// adjust the operands as necessary.
2978static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
2979 Comparison &C) {
2980 // For us to make any changes, it must a comparison between a single-use
2981 // load and a constant.
2982 if (!C.Op0.hasOneUse() ||
2983 C.Op0.getOpcode() != ISD::LOAD ||
2984 C.Op1.getOpcode() != ISD::Constant)
2985 return;
2986
2987 // We must have an 8- or 16-bit load.
2988 auto *Load = cast<LoadSDNode>(Val&: C.Op0);
2989 unsigned NumBits = Load->getMemoryVT().getSizeInBits();
2990 if ((NumBits != 8 && NumBits != 16) ||
2991 NumBits != Load->getMemoryVT().getStoreSizeInBits())
2992 return;
2993
2994 // The load must be an extending one and the constant must be within the
2995 // range of the unextended value.
2996 auto *ConstOp1 = cast<ConstantSDNode>(Val&: C.Op1);
2997 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
2998 return;
2999 uint64_t Value = ConstOp1->getZExtValue();
3000 uint64_t Mask = (1 << NumBits) - 1;
3001 if (Load->getExtensionType() == ISD::SEXTLOAD) {
3002 // Make sure that ConstOp1 is in range of C.Op0.
3003 int64_t SignedValue = ConstOp1->getSExtValue();
3004 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)
3005 return;
3006 if (C.ICmpType != SystemZICMP::SignedOnly) {
3007 // Unsigned comparison between two sign-extended values is equivalent
3008 // to unsigned comparison between two zero-extended values.
3009 Value &= Mask;
3010 } else if (NumBits == 8) {
3011 // Try to treat the comparison as unsigned, so that we can use CLI.
3012 // Adjust CCMask and Value as necessary.
3013 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
3014 // Test whether the high bit of the byte is set.
3015 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
3016 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
3017 // Test whether the high bit of the byte is clear.
3018 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
3019 else
3020 // No instruction exists for this combination.
3021 return;
3022 C.ICmpType = SystemZICMP::UnsignedOnly;
3023 }
3024 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {
3025 if (Value > Mask)
3026 return;
3027 // If the constant is in range, we can use any comparison.
3028 C.ICmpType = SystemZICMP::Any;
3029 } else
3030 return;
3031
3032 // Make sure that the first operand is an i32 of the right extension type.
3033 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ?
3034 ISD::SEXTLOAD :
3035 ISD::ZEXTLOAD);
3036 if (C.Op0.getValueType() != MVT::i32 ||
3037 Load->getExtensionType() != ExtType) {
3038 C.Op0 = DAG.getExtLoad(ExtType, dl: SDLoc(Load), VT: MVT::i32, Chain: Load->getChain(),
3039 Ptr: Load->getBasePtr(), PtrInfo: Load->getPointerInfo(),
3040 MemVT: Load->getMemoryVT(), Alignment: Load->getAlign(),
3041 MMOFlags: Load->getMemOperand()->getFlags());
3042 // Update the chain uses.
3043 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 1), To: C.Op0.getValue(R: 1));
3044 }
3045
3046 // Make sure that the second operand is an i32 with the right value.
3047 if (C.Op1.getValueType() != MVT::i32 ||
3048 Value != ConstOp1->getZExtValue())
3049 C.Op1 = DAG.getConstant(Val: (uint32_t)Value, DL, VT: MVT::i32);
3050}
3051
3052// Return true if Op is either an unextended load, or a load suitable
3053// for integer register-memory comparisons of type ICmpType.
3054static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
3055 auto *Load = dyn_cast<LoadSDNode>(Val: Op.getNode());
3056 if (Load) {
3057 // There are no instructions to compare a register with a memory byte.
3058 if (Load->getMemoryVT() == MVT::i8)
3059 return false;
3060 // Otherwise decide on extension type.
3061 switch (Load->getExtensionType()) {
3062 case ISD::NON_EXTLOAD:
3063 return true;
3064 case ISD::SEXTLOAD:
3065 return ICmpType != SystemZICMP::UnsignedOnly;
3066 case ISD::ZEXTLOAD:
3067 return ICmpType != SystemZICMP::SignedOnly;
3068 default:
3069 break;
3070 }
3071 }
3072 return false;
3073}
3074
3075// Return true if it is better to swap the operands of C.
3076static bool shouldSwapCmpOperands(const Comparison &C) {
3077 // If one side of the compare is a load of the stackguard reference value,
3078 // then that load should be Op1.
3079 if (C.Op0.isMachineOpcode() &&
3080 (C.Op0.getMachineOpcode() == SystemZ::LOAD_STACK_GUARD))
3081 return true;
3082
3083 // Leave i128 and f128 comparisons alone, since they have no memory forms.
3084 if (C.Op0.getValueType() == MVT::i128)
3085 return false;
3086 if (C.Op0.getValueType() == MVT::f128)
3087 return false;
3088
3089 // Always keep a floating-point constant second, since comparisons with
3090 // zero can use LOAD TEST and comparisons with other constants make a
3091 // natural memory operand.
3092 if (isa<ConstantFPSDNode>(Val: C.Op1))
3093 return false;
3094
3095 // Never swap comparisons with zero since there are many ways to optimize
3096 // those later.
3097 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1);
3098 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
3099 return false;
3100
3101 // Also keep natural memory operands second if the loaded value is
3102 // only used here. Several comparisons have memory forms.
3103 if (isNaturalMemoryOperand(Op: C.Op1, ICmpType: C.ICmpType) && C.Op1.hasOneUse())
3104 return false;
3105
3106 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.
3107 // In that case we generally prefer the memory to be second.
3108 if (isNaturalMemoryOperand(Op: C.Op0, ICmpType: C.ICmpType) && C.Op0.hasOneUse()) {
3109 // The only exceptions are when the second operand is a constant and
3110 // we can use things like CHHSI.
3111 if (!ConstOp1)
3112 return true;
3113 // The unsigned memory-immediate instructions can handle 16-bit
3114 // unsigned integers.
3115 if (C.ICmpType != SystemZICMP::SignedOnly &&
3116 isUInt<16>(x: ConstOp1->getZExtValue()))
3117 return false;
3118 // The signed memory-immediate instructions can handle 16-bit
3119 // signed integers.
3120 if (C.ICmpType != SystemZICMP::UnsignedOnly &&
3121 isInt<16>(x: ConstOp1->getSExtValue()))
3122 return false;
3123 return true;
3124 }
3125
3126 // Try to promote the use of CGFR and CLGFR.
3127 unsigned Opcode0 = C.Op0.getOpcode();
3128 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND)
3129 return true;
3130 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND)
3131 return true;
3132 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::AND &&
3133 C.Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
3134 C.Op0.getConstantOperandVal(i: 1) == 0xffffffff)
3135 return true;
3136
3137 return false;
3138}
3139
3140// Check whether C tests for equality between X and Y and whether X - Y
3141// or Y - X is also computed. In that case it's better to compare the
3142// result of the subtraction against zero.
3143static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
3144 Comparison &C) {
3145 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3146 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3147 for (SDNode *N : C.Op0->users()) {
3148 if (N->getOpcode() == ISD::SUB &&
3149 ((N->getOperand(Num: 0) == C.Op0 && N->getOperand(Num: 1) == C.Op1) ||
3150 (N->getOperand(Num: 0) == C.Op1 && N->getOperand(Num: 1) == C.Op0))) {
3151 // Disable the nsw and nuw flags: the backend needs to handle
3152 // overflow as well during comparison elimination.
3153 N->dropFlags(Mask: SDNodeFlags::NoWrap);
3154 C.Op0 = SDValue(N, 0);
3155 C.Op1 = DAG.getConstant(Val: 0, DL, VT: N->getValueType(ResNo: 0));
3156 return;
3157 }
3158 }
3159 }
3160}
3161
3162// Check whether C compares a floating-point value with zero and if that
3163// floating-point value is also negated. In this case we can use the
3164// negation to set CC, so avoiding separate LOAD AND TEST and
3165// LOAD (NEGATIVE/COMPLEMENT) instructions.
3166static void adjustForFNeg(Comparison &C) {
3167 // This optimization is invalid for strict comparisons, since FNEG
3168 // does not raise any exceptions.
3169 if (C.Chain)
3170 return;
3171 auto *C1 = dyn_cast<ConstantFPSDNode>(Val&: C.Op1);
3172 if (C1 && C1->isZero()) {
3173 for (SDNode *N : C.Op0->users()) {
3174 if (N->getOpcode() == ISD::FNEG) {
3175 C.Op0 = SDValue(N, 0);
3176 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3177 return;
3178 }
3179 }
3180 }
3181}
3182
3183// Check whether C compares (shl X, 32) with 0 and whether X is
3184// also sign-extended. In that case it is better to test the result
3185// of the sign extension using LTGFR.
3186//
3187// This case is important because InstCombine transforms a comparison
3188// with (sext (trunc X)) into a comparison with (shl X, 32).
3189static void adjustForLTGFR(Comparison &C) {
3190 // Check for a comparison between (shl X, 32) and 0.
3191 if (C.Op0.getOpcode() == ISD::SHL && C.Op0.getValueType() == MVT::i64 &&
3192 C.Op1.getOpcode() == ISD::Constant && C.Op1->getAsZExtVal() == 0) {
3193 auto *C1 = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3194 if (C1 && C1->getZExtValue() == 32) {
3195 SDValue ShlOp0 = C.Op0.getOperand(i: 0);
3196 // See whether X has any SIGN_EXTEND_INREG uses.
3197 for (SDNode *N : ShlOp0->users()) {
3198 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3199 cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT() == MVT::i32) {
3200 C.Op0 = SDValue(N, 0);
3201 return;
3202 }
3203 }
3204 }
3205 }
3206}
3207
3208// If C compares the truncation of an extending load, try to compare
3209// the untruncated value instead. This exposes more opportunities to
3210// reuse CC.
3211static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
3212 Comparison &C) {
3213 if (C.Op0.getOpcode() == ISD::TRUNCATE &&
3214 C.Op0.getOperand(i: 0).getOpcode() == ISD::LOAD &&
3215 C.Op1.getOpcode() == ISD::Constant &&
3216 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
3217 C.Op1->getAsZExtVal() == 0) {
3218 auto *L = cast<LoadSDNode>(Val: C.Op0.getOperand(i: 0));
3219 if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=
3220 C.Op0.getValueSizeInBits().getFixedValue()) {
3221 unsigned Type = L->getExtensionType();
3222 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
3223 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
3224 C.Op0 = C.Op0.getOperand(i: 0);
3225 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op0.getValueType());
3226 }
3227 }
3228 }
3229}
3230
3231// Adjust if a given Compare is a check of the stack guard against a stack
3232// guard instance on the stack. Specifically, this checks if:
3233// - The operands are a load of the stack guard, and a load from a stack slot
3234// - The original opcode is ICMP
3235// - ICMPType is compatible with unsigned comparison.
3236static void adjustForStackGuardCompare(SelectionDAG &DAG, const SDLoc &DL,
3237 Comparison &C) {
3238
3239 // Opcode must be ICMP.
3240 if (C.Opcode != SystemZISD::ICMP)
3241 return;
3242 // ICmpType must be Unsigned or Any.
3243 if (C.ICmpType == SystemZICMP::SignedOnly)
3244 return;
3245 // Op0 must be FrameIndex Load.
3246 if (!(ISD::isNormalLoad(N: C.Op0.getNode()) &&
3247 dyn_cast<FrameIndexSDNode>(Val: C.Op0.getOperand(i: 1))))
3248 return;
3249 // Op1 must be LOAD_STACK_GUARD.
3250 if (!C.Op1.isMachineOpcode() ||
3251 C.Op1.getMachineOpcode() != SystemZ::LOAD_STACK_GUARD)
3252 return;
3253
3254 // At this point we are sure that this is a proper CMP_STACKGUARD
3255 // case, update the opcode to reflect this.
3256 C.Opcode = SystemZISD::CMP_STACKGUARD;
3257 C.Op1 = SDValue();
3258}
3259
3260// Return true if shift operation N has an in-range constant shift value.
3261// Store it in ShiftVal if so.
3262static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
3263 auto *Shift = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
3264 if (!Shift)
3265 return false;
3266
3267 uint64_t Amount = Shift->getZExtValue();
3268 if (Amount >= N.getValueSizeInBits())
3269 return false;
3270
3271 ShiftVal = Amount;
3272 return true;
3273}
3274
3275// Check whether an AND with Mask is suitable for a TEST UNDER MASK
3276// instruction and whether the CC value is descriptive enough to handle
3277// a comparison of type Opcode between the AND result and CmpVal.
3278// CCMask says which comparison result is being tested and BitSize is
3279// the number of bits in the operands. If TEST UNDER MASK can be used,
3280// return the corresponding CC mask, otherwise return 0.
3281static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
3282 uint64_t Mask, uint64_t CmpVal,
3283 unsigned ICmpType) {
3284 assert(Mask != 0 && "ANDs with zero should have been removed by now");
3285
3286 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL.
3287 if (!SystemZ::isImmLL(Val: Mask) && !SystemZ::isImmLH(Val: Mask) &&
3288 !SystemZ::isImmHL(Val: Mask) && !SystemZ::isImmHH(Val: Mask))
3289 return 0;
3290
3291 // Work out the masks for the lowest and highest bits.
3292 uint64_t High = llvm::bit_floor(Value: Mask);
3293 uint64_t Low = uint64_t(1) << llvm::countr_zero(Val: Mask);
3294
3295 // Signed ordered comparisons are effectively unsigned if the sign
3296 // bit is dropped.
3297 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly);
3298
3299 // Check for equality comparisons with 0, or the equivalent.
3300 if (CmpVal == 0) {
3301 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3302 return SystemZ::CCMASK_TM_ALL_0;
3303 if (CCMask == SystemZ::CCMASK_CMP_NE)
3304 return SystemZ::CCMASK_TM_SOME_1;
3305 }
3306 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) {
3307 if (CCMask == SystemZ::CCMASK_CMP_LT)
3308 return SystemZ::CCMASK_TM_ALL_0;
3309 if (CCMask == SystemZ::CCMASK_CMP_GE)
3310 return SystemZ::CCMASK_TM_SOME_1;
3311 }
3312 if (EffectivelyUnsigned && CmpVal < Low) {
3313 if (CCMask == SystemZ::CCMASK_CMP_LE)
3314 return SystemZ::CCMASK_TM_ALL_0;
3315 if (CCMask == SystemZ::CCMASK_CMP_GT)
3316 return SystemZ::CCMASK_TM_SOME_1;
3317 }
3318
3319 // Check for equality comparisons with the mask, or the equivalent.
3320 if (CmpVal == Mask) {
3321 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3322 return SystemZ::CCMASK_TM_ALL_1;
3323 if (CCMask == SystemZ::CCMASK_CMP_NE)
3324 return SystemZ::CCMASK_TM_SOME_0;
3325 }
3326 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {
3327 if (CCMask == SystemZ::CCMASK_CMP_GT)
3328 return SystemZ::CCMASK_TM_ALL_1;
3329 if (CCMask == SystemZ::CCMASK_CMP_LE)
3330 return SystemZ::CCMASK_TM_SOME_0;
3331 }
3332 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {
3333 if (CCMask == SystemZ::CCMASK_CMP_GE)
3334 return SystemZ::CCMASK_TM_ALL_1;
3335 if (CCMask == SystemZ::CCMASK_CMP_LT)
3336 return SystemZ::CCMASK_TM_SOME_0;
3337 }
3338
3339 // Check for ordered comparisons with the top bit.
3340 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {
3341 if (CCMask == SystemZ::CCMASK_CMP_LE)
3342 return SystemZ::CCMASK_TM_MSB_0;
3343 if (CCMask == SystemZ::CCMASK_CMP_GT)
3344 return SystemZ::CCMASK_TM_MSB_1;
3345 }
3346 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {
3347 if (CCMask == SystemZ::CCMASK_CMP_LT)
3348 return SystemZ::CCMASK_TM_MSB_0;
3349 if (CCMask == SystemZ::CCMASK_CMP_GE)
3350 return SystemZ::CCMASK_TM_MSB_1;
3351 }
3352
3353 // If there are just two bits, we can do equality checks for Low and High
3354 // as well.
3355 if (Mask == Low + High) {
3356 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
3357 return SystemZ::CCMASK_TM_MIXED_MSB_0;
3358 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
3359 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY;
3360 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
3361 return SystemZ::CCMASK_TM_MIXED_MSB_1;
3362 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
3363 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY;
3364 }
3365
3366 // Looks like we've exhausted our options.
3367 return 0;
3368}
3369
3370// See whether C can be implemented as a TEST UNDER MASK instruction.
3371// Update the arguments with the TM version if so.
3372static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
3373 Comparison &C) {
3374 // Use VECTOR TEST UNDER MASK for i128 operations.
3375 if (C.Op0.getValueType() == MVT::i128) {
3376 // We can use VTM for EQ/NE comparisons of x & y against 0.
3377 if (C.Op0.getOpcode() == ISD::AND &&
3378 (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3379 C.CCMask == SystemZ::CCMASK_CMP_NE)) {
3380 auto *Mask = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3381 if (Mask && Mask->getAPIntValue() == 0) {
3382 C.Opcode = SystemZISD::VTM;
3383 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 1));
3384 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 0));
3385 C.CCValid = SystemZ::CCMASK_VCMP;
3386 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3387 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3388 else
3389 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3390 }
3391 }
3392 return;
3393 }
3394
3395 // Check that we have a comparison with a constant.
3396 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3397 if (!ConstOp1)
3398 return;
3399 uint64_t CmpVal = ConstOp1->getZExtValue();
3400
3401 // Check whether the nonconstant input is an AND with a constant mask.
3402 Comparison NewC(C);
3403 uint64_t MaskVal;
3404 ConstantSDNode *Mask = nullptr;
3405 if (C.Op0.getOpcode() == ISD::AND) {
3406 NewC.Op0 = C.Op0.getOperand(i: 0);
3407 NewC.Op1 = C.Op0.getOperand(i: 1);
3408 Mask = dyn_cast<ConstantSDNode>(Val&: NewC.Op1);
3409 if (!Mask)
3410 return;
3411 MaskVal = Mask->getZExtValue();
3412 } else {
3413 // There is no instruction to compare with a 64-bit immediate
3414 // so use TMHH instead if possible. We need an unsigned ordered
3415 // comparison with an i64 immediate.
3416 if (NewC.Op0.getValueType() != MVT::i64 ||
3417 NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
3418 NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
3419 NewC.ICmpType == SystemZICMP::SignedOnly)
3420 return;
3421 // Convert LE and GT comparisons into LT and GE.
3422 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
3423 NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
3424 if (CmpVal == uint64_t(-1))
3425 return;
3426 CmpVal += 1;
3427 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
3428 }
3429 // If the low N bits of Op1 are zero than the low N bits of Op0 can
3430 // be masked off without changing the result.
3431 MaskVal = -(CmpVal & -CmpVal);
3432 NewC.ICmpType = SystemZICMP::UnsignedOnly;
3433 }
3434 if (!MaskVal)
3435 return;
3436
3437 // Check whether the combination of mask, comparison value and comparison
3438 // type are suitable.
3439 unsigned BitSize = NewC.Op0.getValueSizeInBits();
3440 unsigned NewCCMask, ShiftVal;
3441 if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3442 NewC.Op0.getOpcode() == ISD::SHL &&
3443 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3444 (MaskVal >> ShiftVal != 0) &&
3445 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
3446 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3447 Mask: MaskVal >> ShiftVal,
3448 CmpVal: CmpVal >> ShiftVal,
3449 ICmpType: SystemZICMP::Any))) {
3450 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3451 MaskVal >>= ShiftVal;
3452 } else if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3453 NewC.Op0.getOpcode() == ISD::SRL &&
3454 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3455 (MaskVal << ShiftVal != 0) &&
3456 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
3457 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3458 Mask: MaskVal << ShiftVal,
3459 CmpVal: CmpVal << ShiftVal,
3460 ICmpType: SystemZICMP::UnsignedOnly))) {
3461 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3462 MaskVal <<= ShiftVal;
3463 } else {
3464 NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask, Mask: MaskVal, CmpVal,
3465 ICmpType: NewC.ICmpType);
3466 if (!NewCCMask)
3467 return;
3468 }
3469
3470 // Go ahead and make the change.
3471 C.Opcode = SystemZISD::TM;
3472 C.Op0 = NewC.Op0;
3473 if (Mask && Mask->getZExtValue() == MaskVal)
3474 C.Op1 = SDValue(Mask, 0);
3475 else
3476 C.Op1 = DAG.getConstant(Val: MaskVal, DL, VT: C.Op0.getValueType());
3477 C.CCValid = SystemZ::CCMASK_TM;
3478 C.CCMask = NewCCMask;
3479}
3480
3481// Implement i128 comparison in vector registers.
3482static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL,
3483 Comparison &C) {
3484 if (C.Opcode != SystemZISD::ICMP)
3485 return;
3486 if (C.Op0.getValueType() != MVT::i128)
3487 return;
3488
3489 // Recognize vector comparison reductions.
3490 if ((C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3491 C.CCMask == SystemZ::CCMASK_CMP_NE) &&
3492 (isNullConstant(V: C.Op1) || isAllOnesConstant(V: C.Op1))) {
3493 bool CmpEq = C.CCMask == SystemZ::CCMASK_CMP_EQ;
3494 bool CmpNull = isNullConstant(V: C.Op1);
3495 SDValue Src = peekThroughBitcasts(V: C.Op0);
3496 if (Src.hasOneUse() && isBitwiseNot(V: Src)) {
3497 Src = Src.getOperand(i: 0);
3498 CmpNull = !CmpNull;
3499 }
3500 unsigned Opcode = 0;
3501 if (Src.hasOneUse()) {
3502 switch (Src.getOpcode()) {
3503 case SystemZISD::VICMPE: Opcode = SystemZISD::VICMPES; break;
3504 case SystemZISD::VICMPH: Opcode = SystemZISD::VICMPHS; break;
3505 case SystemZISD::VICMPHL: Opcode = SystemZISD::VICMPHLS; break;
3506 case SystemZISD::VFCMPE: Opcode = SystemZISD::VFCMPES; break;
3507 case SystemZISD::VFCMPH: Opcode = SystemZISD::VFCMPHS; break;
3508 case SystemZISD::VFCMPHE: Opcode = SystemZISD::VFCMPHES; break;
3509 default: break;
3510 }
3511 }
3512 if (Opcode) {
3513 C.Opcode = Opcode;
3514 C.Op0 = Src->getOperand(Num: 0);
3515 C.Op1 = Src->getOperand(Num: 1);
3516 C.CCValid = SystemZ::CCMASK_VCMP;
3517 C.CCMask = CmpNull ? SystemZ::CCMASK_VCMP_NONE : SystemZ::CCMASK_VCMP_ALL;
3518 if (!CmpEq)
3519 C.CCMask ^= C.CCValid;
3520 return;
3521 }
3522 }
3523
3524 // Everything below here is not useful if we have native i128 compares.
3525 if (DAG.getSubtarget<SystemZSubtarget>().hasVectorEnhancements3())
3526 return;
3527
3528 // (In-)Equality comparisons can be implemented via VCEQGS.
3529 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3530 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3531 C.Opcode = SystemZISD::VICMPES;
3532 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op0);
3533 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op1);
3534 C.CCValid = SystemZ::CCMASK_VCMP;
3535 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3536 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3537 else
3538 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3539 return;
3540 }
3541
3542 // Normalize other comparisons to GT.
3543 bool Swap = false, Invert = false;
3544 switch (C.CCMask) {
3545 case SystemZ::CCMASK_CMP_GT: break;
3546 case SystemZ::CCMASK_CMP_LT: Swap = true; break;
3547 case SystemZ::CCMASK_CMP_LE: Invert = true; break;
3548 case SystemZ::CCMASK_CMP_GE: Swap = Invert = true; break;
3549 default: llvm_unreachable("Invalid integer condition!");
3550 }
3551 if (Swap)
3552 std::swap(a&: C.Op0, b&: C.Op1);
3553
3554 if (C.ICmpType == SystemZICMP::UnsignedOnly)
3555 C.Opcode = SystemZISD::UCMP128HI;
3556 else
3557 C.Opcode = SystemZISD::SCMP128HI;
3558 C.CCValid = SystemZ::CCMASK_ANY;
3559 C.CCMask = SystemZ::CCMASK_1;
3560
3561 if (Invert)
3562 C.CCMask ^= C.CCValid;
3563}
3564
3565// See whether the comparison argument contains a redundant AND
3566// and remove it if so. This sometimes happens due to the generic
3567// BRCOND expansion.
3568static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
3569 Comparison &C) {
3570 if (C.Op0.getOpcode() != ISD::AND)
3571 return;
3572 auto *Mask = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3573 if (!Mask || Mask->getValueSizeInBits(ResNo: 0) > 64)
3574 return;
3575 KnownBits Known = DAG.computeKnownBits(Op: C.Op0.getOperand(i: 0));
3576 if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
3577 return;
3578
3579 C.Op0 = C.Op0.getOperand(i: 0);
3580}
3581
3582// Return a Comparison that tests the condition-code result of intrinsic
3583// node Call against constant integer CC using comparison code Cond.
3584// Opcode is the opcode of the SystemZISD operation for the intrinsic
3585// and CCValid is the set of possible condition-code results.
3586static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
3587 SDValue Call, unsigned CCValid, uint64_t CC,
3588 ISD::CondCode Cond) {
3589 Comparison C(Call, SDValue(), SDValue());
3590 C.Opcode = Opcode;
3591 C.CCValid = CCValid;
3592 if (Cond == ISD::SETEQ)
3593 // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3.
3594 C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
3595 else if (Cond == ISD::SETNE)
3596 // ...and the inverse of that.
3597 C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
3598 else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
3599 // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
3600 // always true for CC>3.
3601 C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
3602 else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
3603 // ...and the inverse of that.
3604 C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
3605 else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
3606 // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
3607 // always true for CC>3.
3608 C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
3609 else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
3610 // ...and the inverse of that.
3611 C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
3612 else
3613 llvm_unreachable("Unexpected integer comparison type");
3614 C.CCMask &= CCValid;
3615 return C;
3616}
3617
3618// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
3619static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
3620 ISD::CondCode Cond, const SDLoc &DL,
3621 SDValue Chain = SDValue(),
3622 bool IsSignaling = false) {
3623 if (CmpOp1.getOpcode() == ISD::Constant) {
3624 assert(!Chain);
3625 unsigned Opcode, CCValid;
3626 if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
3627 CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(NUses: 1, Value: 0) &&
3628 isIntrinsicWithCCAndChain(Op: CmpOp0, Opcode, CCValid))
3629 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3630 CC: CmpOp1->getAsZExtVal(), Cond);
3631 if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
3632 CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
3633 isIntrinsicWithCC(Op: CmpOp0, Opcode, CCValid))
3634 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3635 CC: CmpOp1->getAsZExtVal(), Cond);
3636 }
3637 Comparison C(CmpOp0, CmpOp1, Chain);
3638 C.CCMask = CCMaskForCondCode(CC: Cond);
3639 if (C.Op0.getValueType().isFloatingPoint()) {
3640 C.CCValid = SystemZ::CCMASK_FCMP;
3641 if (!C.Chain)
3642 C.Opcode = SystemZISD::FCMP;
3643 else if (!IsSignaling)
3644 C.Opcode = SystemZISD::STRICT_FCMP;
3645 else
3646 C.Opcode = SystemZISD::STRICT_FCMPS;
3647 adjustForFNeg(C);
3648 } else {
3649 assert(!C.Chain);
3650 C.CCValid = SystemZ::CCMASK_ICMP;
3651 C.Opcode = SystemZISD::ICMP;
3652 // Choose the type of comparison. Equality and inequality tests can
3653 // use either signed or unsigned comparisons. The choice also doesn't
3654 // matter if both sign bits are known to be clear. In those cases we
3655 // want to give the main isel code the freedom to choose whichever
3656 // form fits best.
3657 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3658 C.CCMask == SystemZ::CCMASK_CMP_NE ||
3659 (DAG.SignBitIsZero(Op: C.Op0) && DAG.SignBitIsZero(Op: C.Op1)))
3660 C.ICmpType = SystemZICMP::Any;
3661 else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
3662 C.ICmpType = SystemZICMP::UnsignedOnly;
3663 else
3664 C.ICmpType = SystemZICMP::SignedOnly;
3665 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
3666 adjustForRedundantAnd(DAG, DL, C);
3667 adjustZeroCmp(DAG, DL, C);
3668 adjustSubwordCmp(DAG, DL, C);
3669 adjustForSubtraction(DAG, DL, C);
3670 adjustForLTGFR(C);
3671 adjustICmpTruncate(DAG, DL, C);
3672 }
3673
3674 if (shouldSwapCmpOperands(C)) {
3675 std::swap(a&: C.Op0, b&: C.Op1);
3676 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3677 }
3678
3679 adjustForTestUnderMask(DAG, DL, C);
3680 adjustICmp128(DAG, DL, C);
3681 adjustForStackGuardCompare(DAG, DL, C);
3682 return C;
3683}
3684
3685// Emit the comparison instruction described by C.
3686static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
3687 if (!C.Op1.getNode()) {
3688 if (C.Opcode == SystemZISD::CMP_STACKGUARD)
3689 return DAG.getNode(Opcode: SystemZISD::CMP_STACKGUARD, DL, VT: MVT::i32, Operand: C.Op0);
3690 SDNode *Node;
3691 switch (C.Op0.getOpcode()) {
3692 case ISD::INTRINSIC_W_CHAIN:
3693 Node = emitIntrinsicWithCCAndChain(DAG, Op: C.Op0, Opcode: C.Opcode);
3694 return SDValue(Node, 0);
3695 case ISD::INTRINSIC_WO_CHAIN:
3696 Node = emitIntrinsicWithCC(DAG, Op: C.Op0, Opcode: C.Opcode);
3697 return SDValue(Node, Node->getNumValues() - 1);
3698 default:
3699 llvm_unreachable("Invalid comparison operands");
3700 }
3701 }
3702 if (C.Opcode == SystemZISD::ICMP)
3703 return DAG.getNode(Opcode: SystemZISD::ICMP, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3704 N3: DAG.getTargetConstant(Val: C.ICmpType, DL, VT: MVT::i32));
3705 if (C.Opcode == SystemZISD::TM) {
3706 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
3707 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
3708 return DAG.getNode(Opcode: SystemZISD::TM, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3709 N3: DAG.getTargetConstant(Val: RegisterOnly, DL, VT: MVT::i32));
3710 }
3711 if (C.Opcode == SystemZISD::VICMPES ||
3712 C.Opcode == SystemZISD::VICMPHS ||
3713 C.Opcode == SystemZISD::VICMPHLS ||
3714 C.Opcode == SystemZISD::VFCMPES ||
3715 C.Opcode == SystemZISD::VFCMPHS ||
3716 C.Opcode == SystemZISD::VFCMPHES) {
3717 EVT IntVT = C.Op0.getValueType().changeVectorElementTypeToInteger();
3718 SDVTList VTs = DAG.getVTList(VT1: IntVT, VT2: MVT::i32);
3719 SDValue Val = DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Op0, N2: C.Op1);
3720 return SDValue(Val.getNode(), 1);
3721 }
3722 if (C.Chain) {
3723 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
3724 return DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Chain, N2: C.Op0, N3: C.Op1);
3725 }
3726 return DAG.getNode(Opcode: C.Opcode, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1);
3727}
3728
3729// Implement a 32-bit *MUL_LOHI operation by extending both operands to
3730// 64 bits. Extend is the extension type to use. Store the high part
3731// in Hi and the low part in Lo.
3732static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
3733 SDValue Op0, SDValue Op1, SDValue &Hi,
3734 SDValue &Lo) {
3735 Op0 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op0);
3736 Op1 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op1);
3737 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
3738 Hi = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Mul,
3739 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3740 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Hi);
3741 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Mul);
3742}
3743
3744// Lower a binary operation that produces two VT results, one in each
3745// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation,
3746// and Opcode performs the GR128 operation. Store the even register result
3747// in Even and the odd register result in Odd.
3748static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
3749 unsigned Opcode, SDValue Op0, SDValue Op1,
3750 SDValue &Even, SDValue &Odd) {
3751 SDValue Result = DAG.getNode(Opcode, DL, VT: MVT::Untyped, N1: Op0, N2: Op1);
3752 bool Is32Bit = is32Bit(VT);
3753 Even = DAG.getTargetExtractSubreg(SRIdx: SystemZ::even128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3754 Odd = DAG.getTargetExtractSubreg(SRIdx: SystemZ::odd128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3755}
3756
3757// Return an i32 value that is 1 if the CC value produced by CCReg is
3758// in the mask CCMask and 0 otherwise. CC is known to have a value
3759// in CCValid, so other values can be ignored.
3760static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
3761 unsigned CCValid, unsigned CCMask) {
3762 SDValue Ops[] = {DAG.getConstant(Val: 1, DL, VT: MVT::i32),
3763 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
3764 DAG.getTargetConstant(Val: CCValid, DL, VT: MVT::i32),
3765 DAG.getTargetConstant(Val: CCMask, DL, VT: MVT::i32), CCReg};
3766 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: MVT::i32, Ops);
3767}
3768
3769// Return the SystemISD vector comparison operation for CC, or 0 if it cannot
3770// be done directly. Mode is CmpMode::Int for integer comparisons, CmpMode::FP
3771// for regular floating-point comparisons, CmpMode::StrictFP for strict (quiet)
3772// floating-point comparisons, and CmpMode::SignalingFP for strict signaling
3773// floating-point comparisons.
3774enum class CmpMode { Int, FP, StrictFP, SignalingFP };
3775static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode) {
3776 switch (CC) {
3777 case ISD::SETOEQ:
3778 case ISD::SETEQ:
3779 switch (Mode) {
3780 case CmpMode::Int: return SystemZISD::VICMPE;
3781 case CmpMode::FP: return SystemZISD::VFCMPE;
3782 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPE;
3783 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPES;
3784 }
3785 llvm_unreachable("Bad mode");
3786
3787 case ISD::SETOGE:
3788 case ISD::SETGE:
3789 switch (Mode) {
3790 case CmpMode::Int: return 0;
3791 case CmpMode::FP: return SystemZISD::VFCMPHE;
3792 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPHE;
3793 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHES;
3794 }
3795 llvm_unreachable("Bad mode");
3796
3797 case ISD::SETOGT:
3798 case ISD::SETGT:
3799 switch (Mode) {
3800 case CmpMode::Int: return SystemZISD::VICMPH;
3801 case CmpMode::FP: return SystemZISD::VFCMPH;
3802 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPH;
3803 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHS;
3804 }
3805 llvm_unreachable("Bad mode");
3806
3807 case ISD::SETUGT:
3808 switch (Mode) {
3809 case CmpMode::Int: return SystemZISD::VICMPHL;
3810 case CmpMode::FP: return 0;
3811 case CmpMode::StrictFP: return 0;
3812 case CmpMode::SignalingFP: return 0;
3813 }
3814 llvm_unreachable("Bad mode");
3815
3816 default:
3817 return 0;
3818 }
3819}
3820
3821// Return the SystemZISD vector comparison operation for CC or its inverse,
3822// or 0 if neither can be done directly. Indicate in Invert whether the
3823// result is for the inverse of CC. Mode is as above.
3824static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode,
3825 bool &Invert) {
3826 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3827 Invert = false;
3828 return Opcode;
3829 }
3830
3831 CC = ISD::getSetCCInverse(Operation: CC, Type: Mode == CmpMode::Int ? MVT::i32 : MVT::f32);
3832 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3833 Invert = true;
3834 return Opcode;
3835 }
3836
3837 return 0;
3838}
3839
3840// Return a v2f64 that contains the extended form of elements Start and Start+1
3841// of v4f32 value Op. If Chain is nonnull, return the strict form.
3842static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
3843 SDValue Op, SDValue Chain) {
3844 int Mask[] = { Start, -1, Start + 1, -1 };
3845 Op = DAG.getVectorShuffle(VT: MVT::v4f32, dl: DL, N1: Op, N2: DAG.getUNDEF(VT: MVT::v4f32), Mask);
3846 if (Chain) {
3847 SDVTList VTs = DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other);
3848 return DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL, VTList: VTs, N1: Chain, N2: Op);
3849 }
3850 return DAG.getNode(Opcode: SystemZISD::VEXTEND, DL, VT: MVT::v2f64, Operand: Op);
3851}
3852
3853// Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
3854// producing a result of type VT. If Chain is nonnull, return the strict form.
3855SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
3856 const SDLoc &DL, EVT VT,
3857 SDValue CmpOp0,
3858 SDValue CmpOp1,
3859 SDValue Chain) const {
3860 // There is no hardware support for v4f32 (unless we have the vector
3861 // enhancements facility 1), so extend the vector into two v2f64s
3862 // and compare those.
3863 if (CmpOp0.getValueType() == MVT::v4f32 &&
3864 !Subtarget.hasVectorEnhancements1()) {
3865 SDValue H0 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp0, Chain);
3866 SDValue L0 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp0, Chain);
3867 SDValue H1 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp1, Chain);
3868 SDValue L1 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp1, Chain);
3869 if (Chain) {
3870 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i64, VT2: MVT::Other);
3871 SDValue HRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: H0, N3: H1);
3872 SDValue LRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: L0, N3: L1);
3873 SDValue Res = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3874 SDValue Chains[6] = { H0.getValue(R: 1), L0.getValue(R: 1),
3875 H1.getValue(R: 1), L1.getValue(R: 1),
3876 HRes.getValue(R: 1), LRes.getValue(R: 1) };
3877 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3878 SDValue Ops[2] = { Res, NewChain };
3879 return DAG.getMergeValues(Ops, dl: DL);
3880 }
3881 SDValue HRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: H0, N2: H1);
3882 SDValue LRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: L0, N2: L1);
3883 return DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3884 }
3885 if (Chain) {
3886 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::Other);
3887 return DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: CmpOp0, N3: CmpOp1);
3888 }
3889 return DAG.getNode(Opcode, DL, VT, N1: CmpOp0, N2: CmpOp1);
3890}
3891
3892// Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
3893// an integer mask of type VT. If Chain is nonnull, we have a strict
3894// floating-point comparison. If in addition IsSignaling is true, we have
3895// a strict signaling floating-point comparison.
3896SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
3897 const SDLoc &DL, EVT VT,
3898 ISD::CondCode CC,
3899 SDValue CmpOp0,
3900 SDValue CmpOp1,
3901 SDValue Chain,
3902 bool IsSignaling) const {
3903 bool IsFP = CmpOp0.getValueType().isFloatingPoint();
3904 assert (!Chain || IsFP);
3905 assert (!IsSignaling || Chain);
3906 CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
3907 Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
3908 bool Invert = false;
3909 SDValue Cmp;
3910 switch (CC) {
3911 // Handle tests for order using (or (ogt y x) (oge x y)).
3912 case ISD::SETUO:
3913 Invert = true;
3914 [[fallthrough]];
3915 case ISD::SETO: {
3916 assert(IsFP && "Unexpected integer comparison");
3917 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3918 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3919 SDValue GE = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGE, Mode),
3920 DL, VT, CmpOp0, CmpOp1, Chain);
3921 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GE);
3922 if (Chain)
3923 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
3924 N1: LT.getValue(R: 1), N2: GE.getValue(R: 1));
3925 break;
3926 }
3927
3928 // Handle <> tests using (or (ogt y x) (ogt x y)).
3929 case ISD::SETUEQ:
3930 Invert = true;
3931 [[fallthrough]];
3932 case ISD::SETONE: {
3933 assert(IsFP && "Unexpected integer comparison");
3934 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3935 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3936 SDValue GT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3937 DL, VT, CmpOp0, CmpOp1, Chain);
3938 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GT);
3939 if (Chain)
3940 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
3941 N1: LT.getValue(R: 1), N2: GT.getValue(R: 1));
3942 break;
3943 }
3944
3945 // Otherwise a single comparison is enough. It doesn't really
3946 // matter whether we try the inversion or the swap first, since
3947 // there are no cases where both work.
3948 default:
3949 // Optimize sign-bit comparisons to signed compares.
3950 if (Mode == CmpMode::Int && (CC == ISD::SETEQ || CC == ISD::SETNE) &&
3951 ISD::isConstantSplatVectorAllZeros(N: CmpOp1.getNode())) {
3952 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
3953 APInt Mask;
3954 if (CmpOp0.getOpcode() == ISD::AND
3955 && ISD::isConstantSplatVector(N: CmpOp0.getOperand(i: 1).getNode(), SplatValue&: Mask)
3956 && Mask == APInt::getSignMask(BitWidth: EltSize)) {
3957 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
3958 CmpOp0 = CmpOp0.getOperand(i: 0);
3959 }
3960 }
3961 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3962 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
3963 else {
3964 CC = ISD::getSetCCSwappedOperands(Operation: CC);
3965 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3966 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3967 else
3968 llvm_unreachable("Unhandled comparison");
3969 }
3970 if (Chain)
3971 Chain = Cmp.getValue(R: 1);
3972 break;
3973 }
3974 if (Invert) {
3975 SDValue Mask =
3976 DAG.getSplatBuildVector(VT, DL, Op: DAG.getAllOnesConstant(DL, VT: MVT::i64));
3977 Cmp = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Cmp, N2: Mask);
3978 }
3979 if (Chain && Chain.getNode() != Cmp.getNode()) {
3980 SDValue Ops[2] = { Cmp, Chain };
3981 Cmp = DAG.getMergeValues(Ops, dl: DL);
3982 }
3983 return Cmp;
3984}
3985
3986SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
3987 SelectionDAG &DAG) const {
3988 SDValue CmpOp0 = Op.getOperand(i: 0);
3989 SDValue CmpOp1 = Op.getOperand(i: 1);
3990 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
3991 SDLoc DL(Op);
3992 EVT VT = Op.getValueType();
3993 if (VT.isVector())
3994 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
3995
3996 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
3997 SDValue CCReg = emitCmp(DAG, DL, C);
3998 return emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
3999}
4000
4001SDValue SystemZTargetLowering::lowerSTRICT_FSETCC(SDValue Op,
4002 SelectionDAG &DAG,
4003 bool IsSignaling) const {
4004 SDValue Chain = Op.getOperand(i: 0);
4005 SDValue CmpOp0 = Op.getOperand(i: 1);
4006 SDValue CmpOp1 = Op.getOperand(i: 2);
4007 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 3))->get();
4008 SDLoc DL(Op);
4009 EVT VT = Op.getNode()->getValueType(ResNo: 0);
4010 if (VT.isVector()) {
4011 SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
4012 Chain, IsSignaling);
4013 return Res.getValue(R: Op.getResNo());
4014 }
4015
4016 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL, Chain, IsSignaling));
4017 SDValue CCReg = emitCmp(DAG, DL, C);
4018 CCReg->setFlags(Op->getFlags());
4019 SDValue Result = emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
4020 SDValue Ops[2] = { Result, CCReg.getValue(R: 1) };
4021 return DAG.getMergeValues(Ops, dl: DL);
4022}
4023
4024SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
4025 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
4026 SDValue CmpOp0 = Op.getOperand(i: 2);
4027 SDValue CmpOp1 = Op.getOperand(i: 3);
4028 SDValue Dest = Op.getOperand(i: 4);
4029 SDLoc DL(Op);
4030
4031 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
4032 SDValue CCReg = emitCmp(DAG, DL, C);
4033 return DAG.getNode(
4034 Opcode: SystemZISD::BR_CCMASK, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
4035 N2: DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
4036 N3: DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), N4: Dest, N5: CCReg);
4037}
4038
4039// Return true if Pos is CmpOp and Neg is the negative of CmpOp,
4040// allowing Pos and Neg to be wider than CmpOp.
4041static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
4042 return (Neg.getOpcode() == ISD::SUB &&
4043 Neg.getOperand(i: 0).getOpcode() == ISD::Constant &&
4044 Neg.getConstantOperandVal(i: 0) == 0 && Neg.getOperand(i: 1) == Pos &&
4045 (Pos == CmpOp || (Pos.getOpcode() == ISD::SIGN_EXTEND &&
4046 Pos.getOperand(i: 0) == CmpOp)));
4047}
4048
4049// Return the absolute or negative absolute of Op; IsNegative decides which.
4050static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
4051 bool IsNegative) {
4052 Op = DAG.getNode(Opcode: ISD::ABS, DL, VT: Op.getValueType(), Operand: Op);
4053 if (IsNegative)
4054 Op = DAG.getNode(Opcode: ISD::SUB, DL, VT: Op.getValueType(),
4055 N1: DAG.getConstant(Val: 0, DL, VT: Op.getValueType()), N2: Op);
4056 return Op;
4057}
4058
4059static SDValue getI128Select(SelectionDAG &DAG, const SDLoc &DL,
4060 Comparison C, SDValue TrueOp, SDValue FalseOp) {
4061 EVT VT = MVT::i128;
4062 unsigned Op;
4063
4064 if (C.CCMask == SystemZ::CCMASK_CMP_NE ||
4065 C.CCMask == SystemZ::CCMASK_CMP_GE ||
4066 C.CCMask == SystemZ::CCMASK_CMP_LE) {
4067 std::swap(a&: TrueOp, b&: FalseOp);
4068 C.CCMask ^= C.CCValid;
4069 }
4070 if (C.CCMask == SystemZ::CCMASK_CMP_LT) {
4071 std::swap(a&: C.Op0, b&: C.Op1);
4072 C.CCMask = SystemZ::CCMASK_CMP_GT;
4073 }
4074 switch (C.CCMask) {
4075 case SystemZ::CCMASK_CMP_EQ:
4076 Op = SystemZISD::VICMPE;
4077 break;
4078 case SystemZ::CCMASK_CMP_GT:
4079 if (C.ICmpType == SystemZICMP::UnsignedOnly)
4080 Op = SystemZISD::VICMPHL;
4081 else
4082 Op = SystemZISD::VICMPH;
4083 break;
4084 default:
4085 llvm_unreachable("Unhandled comparison");
4086 break;
4087 }
4088
4089 SDValue Mask = DAG.getNode(Opcode: Op, DL, VT, N1: C.Op0, N2: C.Op1);
4090 TrueOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: TrueOp, N2: Mask);
4091 FalseOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: FalseOp, N2: DAG.getNOT(DL, Val: Mask, VT));
4092 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: TrueOp, N2: FalseOp);
4093}
4094
4095SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
4096 SelectionDAG &DAG) const {
4097 SDValue CmpOp0 = Op.getOperand(i: 0);
4098 SDValue CmpOp1 = Op.getOperand(i: 1);
4099 SDValue TrueOp = Op.getOperand(i: 2);
4100 SDValue FalseOp = Op.getOperand(i: 3);
4101 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
4102 SDLoc DL(Op);
4103
4104 // SELECT_CC involving f16 will not have the cmp-ops promoted by the
4105 // legalizer, as it will be handled according to the type of the resulting
4106 // value. Extend them here if needed.
4107 if (CmpOp0.getSimpleValueType() == MVT::f16) {
4108 CmpOp0 = DAG.getFPExtendOrRound(Op: CmpOp0, DL: SDLoc(CmpOp0), VT: MVT::f32);
4109 CmpOp1 = DAG.getFPExtendOrRound(Op: CmpOp1, DL: SDLoc(CmpOp1), VT: MVT::f32);
4110 }
4111
4112 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
4113
4114 // Check for absolute and negative-absolute selections, including those
4115 // where the comparison value is sign-extended (for LPGFR and LNGFR).
4116 // This check supplements the one in DAGCombiner.
4117 if (C.Opcode == SystemZISD::ICMP && C.CCMask != SystemZ::CCMASK_CMP_EQ &&
4118 C.CCMask != SystemZ::CCMASK_CMP_NE &&
4119 C.Op1.getOpcode() == ISD::Constant &&
4120 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
4121 C.Op1->getAsZExtVal() == 0) {
4122 if (isAbsolute(CmpOp: C.Op0, Pos: TrueOp, Neg: FalseOp))
4123 return getAbsolute(DAG, DL, Op: TrueOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_LT);
4124 if (isAbsolute(CmpOp: C.Op0, Pos: FalseOp, Neg: TrueOp))
4125 return getAbsolute(DAG, DL, Op: FalseOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_GT);
4126 }
4127
4128 if (Subtarget.hasVectorEnhancements3() &&
4129 C.Opcode == SystemZISD::ICMP &&
4130 C.Op0.getValueType() == MVT::i128 &&
4131 TrueOp.getValueType() == MVT::i128) {
4132 return getI128Select(DAG, DL, C, TrueOp, FalseOp);
4133 }
4134
4135 SDValue CCReg = emitCmp(DAG, DL, C);
4136 SDValue Ops[] = {TrueOp, FalseOp,
4137 DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
4138 DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), CCReg};
4139
4140 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: Op.getValueType(), Ops);
4141}
4142
4143SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
4144 SelectionDAG &DAG) const {
4145 SDLoc DL(Node);
4146 const GlobalValue *GV = Node->getGlobal();
4147 int64_t Offset = Node->getOffset();
4148 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4149 CodeModel::Model CM = DAG.getTarget().getCodeModel();
4150
4151 SDValue Result;
4152 if (Subtarget.isPC32DBLSymbol(GV, CM)) {
4153 if (isInt<32>(x: Offset)) {
4154 // Assign anchors at 1<<12 byte boundaries.
4155 uint64_t Anchor = Offset & ~uint64_t(0xfff);
4156 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor);
4157 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4158
4159 // The offset can be folded into the address if it is aligned to a
4160 // halfword.
4161 Offset -= Anchor;
4162 if (Offset != 0 && (Offset & 1) == 0) {
4163 SDValue Full =
4164 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor + Offset);
4165 Result = DAG.getNode(Opcode: SystemZISD::PCREL_OFFSET, DL, VT: PtrVT, N1: Full, N2: Result);
4166 Offset = 0;
4167 }
4168 } else {
4169 // Conservatively load a constant offset greater than 32 bits into a
4170 // register below.
4171 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT);
4172 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4173 }
4174 } else if (Subtarget.isTargetELF()) {
4175 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_GOT);
4176 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4177 Result = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result,
4178 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4179 } else if (Subtarget.isTargetzOS()) {
4180 Result = getADAEntry(DAG, GV, DL, PtrVT);
4181 } else
4182 llvm_unreachable("Unexpected Subtarget");
4183
4184 // If there was a non-zero offset that we didn't fold, create an explicit
4185 // addition for it.
4186 if (Offset != 0)
4187 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4188 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4189
4190 return Result;
4191}
4192
4193SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
4194 SelectionDAG &DAG,
4195 unsigned Opcode,
4196 SDValue GOTOffset) const {
4197 SDLoc DL(Node);
4198 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4199 SDValue Chain = DAG.getEntryNode();
4200 SDValue Glue;
4201
4202 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4203 CallingConv::GHC)
4204 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4205
4206 // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12.
4207 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(VT: PtrVT);
4208 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R12D, N: GOT, Glue);
4209 Glue = Chain.getValue(R: 1);
4210 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R2D, N: GOTOffset, Glue);
4211 Glue = Chain.getValue(R: 1);
4212
4213 // The first call operand is the chain and the second is the TLS symbol.
4214 SmallVector<SDValue, 8> Ops;
4215 Ops.push_back(Elt: Chain);
4216 Ops.push_back(Elt: DAG.getTargetGlobalAddress(GV: Node->getGlobal(), DL,
4217 VT: Node->getValueType(ResNo: 0),
4218 offset: 0, TargetFlags: 0));
4219
4220 // Add argument registers to the end of the list so that they are
4221 // known live into the call.
4222 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R2D, VT: PtrVT));
4223 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R12D, VT: PtrVT));
4224
4225 // Add a register mask operand representing the call-preserved registers.
4226 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
4227 const uint32_t *Mask =
4228 TRI->getCallPreservedMask(MF: DAG.getMachineFunction(), CallingConv::C);
4229 assert(Mask && "Missing call preserved mask for calling convention");
4230 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
4231
4232 // Glue the call to the argument copies.
4233 Ops.push_back(Elt: Glue);
4234
4235 // Emit the call.
4236 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
4237 Chain = DAG.getNode(Opcode, DL, VTList: NodeTys, Ops);
4238 Glue = Chain.getValue(R: 1);
4239
4240 // Copy the return value from %r2.
4241 return DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::R2D, VT: PtrVT, Glue);
4242}
4243
4244SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
4245 SelectionDAG &DAG) const {
4246 SDValue Chain = DAG.getEntryNode();
4247 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4248
4249 // The high part of the thread pointer is in access register 0.
4250 SDValue TPHi = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A0, VT: MVT::i32);
4251 TPHi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PtrVT, Operand: TPHi);
4252
4253 // The low part of the thread pointer is in access register 1.
4254 SDValue TPLo = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A1, VT: MVT::i32);
4255 TPLo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: PtrVT, Operand: TPLo);
4256
4257 // Merge them into a single 64-bit address.
4258 SDValue TPHiShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: TPHi,
4259 N2: DAG.getConstant(Val: 32, DL, VT: PtrVT));
4260 return DAG.getNode(Opcode: ISD::OR, DL, VT: PtrVT, N1: TPHiShifted, N2: TPLo);
4261}
4262
4263SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
4264 SelectionDAG &DAG) const {
4265 if (DAG.getTarget().useEmulatedTLS())
4266 return LowerToTLSEmulatedModel(GA: Node, DAG);
4267 SDLoc DL(Node);
4268 const GlobalValue *GV = Node->getGlobal();
4269 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4270 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
4271
4272 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4273 CallingConv::GHC)
4274 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4275
4276 SDValue TP = lowerThreadPointer(DL, DAG);
4277
4278 // Get the offset of GA from the thread pointer, based on the TLS model.
4279 SDValue Offset;
4280 switch (model) {
4281 case TLSModel::GeneralDynamic: {
4282 // Load the GOT offset of the tls_index (module ID / per-symbol offset).
4283 SystemZConstantPoolValue *CPV =
4284 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSGD);
4285
4286 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4287 Offset = DAG.getLoad(
4288 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4289 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4290
4291 // Call __tls_get_offset to retrieve the offset.
4292 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_GDCALL, GOTOffset: Offset);
4293 break;
4294 }
4295
4296 case TLSModel::LocalDynamic: {
4297 // Load the GOT offset of the module ID.
4298 SystemZConstantPoolValue *CPV =
4299 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSLDM);
4300
4301 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4302 Offset = DAG.getLoad(
4303 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4304 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4305
4306 // Call __tls_get_offset to retrieve the module base offset.
4307 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_LDCALL, GOTOffset: Offset);
4308
4309 // Note: The SystemZLDCleanupPass will remove redundant computations
4310 // of the module base offset. Count total number of local-dynamic
4311 // accesses to trigger execution of that pass.
4312 SystemZMachineFunctionInfo* MFI =
4313 DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>();
4314 MFI->incNumLocalDynamicTLSAccesses();
4315
4316 // Add the per-symbol offset.
4317 CPV = SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::DTPOFF);
4318
4319 SDValue DTPOffset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4320 DTPOffset = DAG.getLoad(
4321 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: DTPOffset,
4322 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4323
4324 Offset = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Offset, N2: DTPOffset);
4325 break;
4326 }
4327
4328 case TLSModel::InitialExec: {
4329 // Load the offset from the GOT.
4330 Offset = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0,
4331 TargetFlags: SystemZII::MO_INDNTPOFF);
4332 Offset = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Offset);
4333 Offset =
4334 DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4335 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4336 break;
4337 }
4338
4339 case TLSModel::LocalExec: {
4340 // Force the offset into the constant pool and load it from there.
4341 SystemZConstantPoolValue *CPV =
4342 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::NTPOFF);
4343
4344 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4345 Offset = DAG.getLoad(
4346 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4347 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4348 break;
4349 }
4350 }
4351
4352 // Add the base and offset together.
4353 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TP, N2: Offset);
4354}
4355
4356SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
4357 SelectionDAG &DAG) const {
4358 SDLoc DL(Node);
4359 const BlockAddress *BA = Node->getBlockAddress();
4360 int64_t Offset = Node->getOffset();
4361 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4362
4363 SDValue Result = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset);
4364 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4365 return Result;
4366}
4367
4368SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
4369 SelectionDAG &DAG) const {
4370 SDLoc DL(JT);
4371 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4372 SDValue Result = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT);
4373
4374 // Use LARL to load the address of the table.
4375 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4376}
4377
4378SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
4379 SelectionDAG &DAG) const {
4380 SDLoc DL(CP);
4381 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4382
4383 SDValue Result;
4384 if (CP->isMachineConstantPoolEntry())
4385 Result =
4386 DAG.getTargetConstantPool(C: CP->getMachineCPVal(), VT: PtrVT, Align: CP->getAlign());
4387 else
4388 Result = DAG.getTargetConstantPool(C: CP->getConstVal(), VT: PtrVT, Align: CP->getAlign(),
4389 Offset: CP->getOffset());
4390
4391 // Use LARL to load the address of the constant pool entry.
4392 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4393}
4394
4395SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
4396 SelectionDAG &DAG) const {
4397 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4398 MachineFunction &MF = DAG.getMachineFunction();
4399 MachineFrameInfo &MFI = MF.getFrameInfo();
4400 MFI.setFrameAddressIsTaken(true);
4401
4402 SDLoc DL(Op);
4403 unsigned Depth = Op.getConstantOperandVal(i: 0);
4404 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4405
4406 // By definition, the frame address is the address of the back chain. (In
4407 // the case of packed stack without backchain, return the address where the
4408 // backchain would have been stored. This will either be an unused space or
4409 // contain a saved register).
4410 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
4411 SDValue BackChain = DAG.getFrameIndex(FI: BackChainIdx, VT: PtrVT);
4412
4413 if (Depth > 0) {
4414 // FIXME The frontend should detect this case.
4415 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4416 report_fatal_error(reason: "Unsupported stack frame traversal count");
4417
4418 SDValue Offset = DAG.getConstant(Val: TFL->getBackchainOffset(MF), DL, VT: PtrVT);
4419 while (Depth--) {
4420 BackChain = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: BackChain,
4421 PtrInfo: MachinePointerInfo());
4422 BackChain = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: BackChain, N2: Offset);
4423 }
4424 }
4425
4426 return BackChain;
4427}
4428
4429SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
4430 SelectionDAG &DAG) const {
4431 MachineFunction &MF = DAG.getMachineFunction();
4432 MachineFrameInfo &MFI = MF.getFrameInfo();
4433 MFI.setReturnAddressIsTaken(true);
4434
4435 SDLoc DL(Op);
4436 unsigned Depth = Op.getConstantOperandVal(i: 0);
4437 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4438
4439 if (Depth > 0) {
4440 // FIXME The frontend should detect this case.
4441 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4442 report_fatal_error(reason: "Unsupported stack frame traversal count");
4443
4444 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
4445 const auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4446 int Offset = TFL->getReturnAddressOffset(MF);
4447 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FrameAddr,
4448 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4449 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr,
4450 PtrInfo: MachinePointerInfo());
4451 }
4452
4453 // Return R14D (Elf) / R7D (XPLINK), which has the return address. Mark it an
4454 // implicit live-in.
4455 SystemZCallingConventionRegisters *CCR = Subtarget.getSpecialRegisters();
4456 Register LinkReg = MF.addLiveIn(PReg: CCR->getReturnFunctionAddressRegister(),
4457 RC: &SystemZ::GR64BitRegClass);
4458 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: LinkReg, VT: PtrVT);
4459}
4460
4461SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
4462 SelectionDAG &DAG) const {
4463 SDLoc DL(Op);
4464 SDValue In = Op.getOperand(i: 0);
4465 EVT InVT = In.getValueType();
4466 EVT ResVT = Op.getValueType();
4467
4468 // Convert loads directly. This is normally done by DAGCombiner,
4469 // but we need this case for bitcasts that are created during lowering
4470 // and which are then lowered themselves.
4471 if (auto *LoadN = dyn_cast<LoadSDNode>(Val&: In))
4472 if (ISD::isNormalLoad(N: LoadN)) {
4473 SDValue NewLoad = DAG.getLoad(VT: ResVT, dl: DL, Chain: LoadN->getChain(),
4474 Ptr: LoadN->getBasePtr(), MMO: LoadN->getMemOperand());
4475 // Update the chain uses.
4476 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LoadN, 1), To: NewLoad.getValue(R: 1));
4477 return NewLoad;
4478 }
4479
4480 if (InVT == MVT::i32 && ResVT == MVT::f32) {
4481 SDValue In64;
4482 if (Subtarget.hasHighWord()) {
4483 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL,
4484 VT: MVT::i64);
4485 In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4486 VT: MVT::i64, Operand: SDValue(U64, 0), Subreg: In);
4487 } else {
4488 In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: In);
4489 In64 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: In64,
4490 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4491 }
4492 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: In64);
4493 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32,
4494 DL, VT: MVT::f32, Operand: Out64);
4495 }
4496 if (InVT == MVT::f32 && ResVT == MVT::i32) {
4497 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
4498 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4499 VT: MVT::f64, Operand: SDValue(U64, 0), Subreg: In);
4500 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
4501 if (Subtarget.hasHighWord())
4502 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32, DL,
4503 VT: MVT::i32, Operand: Out64);
4504 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Out64,
4505 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4506 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Shift);
4507 }
4508 llvm_unreachable("Unexpected bitcast combination");
4509}
4510
4511SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
4512 SelectionDAG &DAG) const {
4513
4514 if (Subtarget.isTargetXPLINK64())
4515 return lowerVASTART_XPLINK(Op, DAG);
4516 else
4517 return lowerVASTART_ELF(Op, DAG);
4518}
4519
4520SDValue SystemZTargetLowering::lowerVASTART_XPLINK(SDValue Op,
4521 SelectionDAG &DAG) const {
4522 MachineFunction &MF = DAG.getMachineFunction();
4523 SystemZMachineFunctionInfo *FuncInfo =
4524 MF.getInfo<SystemZMachineFunctionInfo>();
4525
4526 SDLoc DL(Op);
4527
4528 // vastart just stores the address of the VarArgsFrameIndex slot into the
4529 // memory location argument.
4530 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4531 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4532 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4533 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FR, Ptr: Op.getOperand(i: 1),
4534 PtrInfo: MachinePointerInfo(SV));
4535}
4536
4537SDValue SystemZTargetLowering::lowerVASTART_ELF(SDValue Op,
4538 SelectionDAG &DAG) const {
4539 MachineFunction &MF = DAG.getMachineFunction();
4540 SystemZMachineFunctionInfo *FuncInfo =
4541 MF.getInfo<SystemZMachineFunctionInfo>();
4542 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4543
4544 SDValue Chain = Op.getOperand(i: 0);
4545 SDValue Addr = Op.getOperand(i: 1);
4546 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4547 SDLoc DL(Op);
4548
4549 // The initial values of each field.
4550 const unsigned NumFields = 4;
4551 SDValue Fields[NumFields] = {
4552 DAG.getConstant(Val: FuncInfo->getVarArgsFirstGPR(), DL, VT: PtrVT),
4553 DAG.getConstant(Val: FuncInfo->getVarArgsFirstFPR(), DL, VT: PtrVT),
4554 DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT),
4555 DAG.getFrameIndex(FI: FuncInfo->getRegSaveFrameIndex(), VT: PtrVT)
4556 };
4557
4558 // Store each field into its respective slot.
4559 SDValue MemOps[NumFields];
4560 unsigned Offset = 0;
4561 for (unsigned I = 0; I < NumFields; ++I) {
4562 SDValue FieldAddr = Addr;
4563 if (Offset != 0)
4564 FieldAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FieldAddr,
4565 N2: DAG.getIntPtrConstant(Val: Offset, DL));
4566 MemOps[I] = DAG.getStore(Chain, dl: DL, Val: Fields[I], Ptr: FieldAddr,
4567 PtrInfo: MachinePointerInfo(SV, Offset));
4568 Offset += 8;
4569 }
4570 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
4571}
4572
4573SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
4574 SelectionDAG &DAG) const {
4575 SDValue Chain = Op.getOperand(i: 0);
4576 SDValue DstPtr = Op.getOperand(i: 1);
4577 SDValue SrcPtr = Op.getOperand(i: 2);
4578 const Value *DstSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 3))->getValue();
4579 const Value *SrcSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
4580 SDLoc DL(Op);
4581
4582 uint32_t Sz =
4583 Subtarget.isTargetXPLINK64() ? DAG.getDataLayout().getPointerSize(AS: 0) : 32;
4584 return DAG.getMemcpy(Chain, dl: DL, Dst: DstPtr, Src: SrcPtr, Size: DAG.getIntPtrConstant(Val: Sz, DL),
4585 DstAlign: Align(8), SrcAlign: Align(8), /*isVolatile*/ isVol: false,
4586 /*AlwaysInline*/ false,
4587 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(DstSV),
4588 SrcPtrInfo: MachinePointerInfo(SrcSV));
4589}
4590
4591SDValue
4592SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
4593 SelectionDAG &DAG) const {
4594 if (Subtarget.isTargetXPLINK64())
4595 return lowerDYNAMIC_STACKALLOC_XPLINK(Op, DAG);
4596 else
4597 return lowerDYNAMIC_STACKALLOC_ELF(Op, DAG);
4598}
4599
4600SDValue
4601SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op,
4602 SelectionDAG &DAG) const {
4603 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4604 MachineFunction &MF = DAG.getMachineFunction();
4605 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4606 SDValue Chain = Op.getOperand(i: 0);
4607 SDValue Size = Op.getOperand(i: 1);
4608 SDValue Align = Op.getOperand(i: 2);
4609 SDLoc DL(Op);
4610
4611 // If user has set the no alignment function attribute, ignore
4612 // alloca alignments.
4613 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4614
4615 uint64_t StackAlign = TFI->getStackAlignment();
4616 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4617 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4618
4619 SDValue NeededSpace = Size;
4620
4621 // Add extra space for alignment if needed.
4622 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4623 if (ExtraAlignSpace)
4624 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: NeededSpace,
4625 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4626
4627 bool IsSigned = false;
4628 bool DoesNotReturn = false;
4629 bool IsReturnValueUsed = false;
4630 EVT VT = Op.getValueType();
4631 SDValue AllocaCall =
4632 makeExternalCall(Chain, DAG, CalleeName: "@@ALCAXP", RetVT: VT, Ops: ArrayRef(NeededSpace),
4633 CallConv: CallingConv::C, IsSigned, DL, DoesNotReturn,
4634 IsReturnValueUsed)
4635 .first;
4636
4637 // Perform a CopyFromReg from %GPR4 (stack pointer register). Chain and Glue
4638 // to end of call in order to ensure it isn't broken up from the call
4639 // sequence.
4640 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
4641 Register SPReg = Regs.getStackPointerRegister();
4642 Chain = AllocaCall.getValue(R: 1);
4643 SDValue Glue = AllocaCall.getValue(R: 2);
4644 SDValue NewSPRegNode = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: PtrVT, Glue);
4645 Chain = NewSPRegNode.getValue(R: 1);
4646
4647 MVT PtrMVT = getPointerMemTy(DL: MF.getDataLayout());
4648 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: PtrMVT);
4649 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrMVT, N1: NewSPRegNode, N2: ArgAdjust);
4650
4651 // Dynamically realign if needed.
4652 if (ExtraAlignSpace) {
4653 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4654 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4655 Result = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Result,
4656 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: PtrVT));
4657 }
4658
4659 SDValue Ops[2] = {Result, Chain};
4660 return DAG.getMergeValues(Ops, dl: DL);
4661}
4662
4663SDValue
4664SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(SDValue Op,
4665 SelectionDAG &DAG) const {
4666 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4667 MachineFunction &MF = DAG.getMachineFunction();
4668 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4669 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
4670
4671 SDValue Chain = Op.getOperand(i: 0);
4672 SDValue Size = Op.getOperand(i: 1);
4673 SDValue Align = Op.getOperand(i: 2);
4674 SDLoc DL(Op);
4675
4676 // If user has set the no alignment function attribute, ignore
4677 // alloca alignments.
4678 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4679
4680 uint64_t StackAlign = TFI->getStackAlignment();
4681 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4682 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4683
4684 Register SPReg = getStackPointerRegisterToSaveRestore();
4685 SDValue NeededSpace = Size;
4686
4687 // Get a reference to the stack pointer.
4688 SDValue OldSP = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: MVT::i64);
4689
4690 // If we need a backchain, save it now.
4691 SDValue Backchain;
4692 if (StoreBackchain)
4693 Backchain = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
4694 PtrInfo: MachinePointerInfo());
4695
4696 // Add extra space for alignment if needed.
4697 if (ExtraAlignSpace)
4698 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NeededSpace,
4699 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4700
4701 // Get the new stack pointer value.
4702 SDValue NewSP;
4703 if (hasInlineStackProbe(MF)) {
4704 NewSP = DAG.getNode(Opcode: SystemZISD::PROBED_ALLOCA, DL,
4705 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N1: Chain, N2: OldSP, N3: NeededSpace);
4706 Chain = NewSP.getValue(R: 1);
4707 }
4708 else {
4709 NewSP = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: OldSP, N2: NeededSpace);
4710 // Copy the new stack pointer back.
4711 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SPReg, N: NewSP);
4712 }
4713
4714 // The allocated data lives above the 160 bytes allocated for the standard
4715 // frame, plus any outgoing stack arguments. We don't know how much that
4716 // amounts to yet, so emit a special ADJDYNALLOC placeholder.
4717 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4718 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NewSP, N2: ArgAdjust);
4719
4720 // Dynamically realign if needed.
4721 if (RequiredAlign > StackAlign) {
4722 Result =
4723 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Result,
4724 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4725 Result =
4726 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, N1: Result,
4727 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: MVT::i64));
4728 }
4729
4730 if (StoreBackchain)
4731 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
4732 PtrInfo: MachinePointerInfo());
4733
4734 SDValue Ops[2] = { Result, Chain };
4735 return DAG.getMergeValues(Ops, dl: DL);
4736}
4737
4738SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
4739 SDValue Op, SelectionDAG &DAG) const {
4740 SDLoc DL(Op);
4741
4742 return DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4743}
4744
4745SDValue SystemZTargetLowering::lowerMULH(SDValue Op,
4746 SelectionDAG &DAG,
4747 unsigned Opcode) const {
4748 EVT VT = Op.getValueType();
4749 SDLoc DL(Op);
4750 SDValue Even, Odd;
4751
4752 // This custom expander is only used on z17 and later for 64-bit types.
4753 assert(!is32Bit(VT));
4754 assert(Subtarget.hasMiscellaneousExtensions2());
4755
4756 // SystemZISD::xMUL_LOHI returns the low result in the odd register and
4757 // the high result in the even register. Return the latter.
4758 lowerGR128Binary(DAG, DL, VT, Opcode,
4759 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even, Odd);
4760 return Even;
4761}
4762
4763SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
4764 SelectionDAG &DAG) const {
4765 EVT VT = Op.getValueType();
4766 SDLoc DL(Op);
4767 SDValue Ops[2];
4768 if (is32Bit(VT))
4769 // Just do a normal 64-bit multiplication and extract the results.
4770 // We define this so that it can be used for constant division.
4771 lowerMUL_LOHI32(DAG, DL, Extend: ISD::SIGN_EXTEND, Op0: Op.getOperand(i: 0),
4772 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4773 else if (Subtarget.hasMiscellaneousExtensions2())
4774 // SystemZISD::SMUL_LOHI returns the low result in the odd register and
4775 // the high result in the even register. ISD::SMUL_LOHI is defined to
4776 // return the low half first, so the results are in reverse order.
4777 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SMUL_LOHI,
4778 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4779 else {
4780 // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:
4781 //
4782 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64)
4783 //
4784 // but using the fact that the upper halves are either all zeros
4785 // or all ones:
4786 //
4787 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)
4788 //
4789 // and grouping the right terms together since they are quicker than the
4790 // multiplication:
4791 //
4792 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
4793 SDValue C63 = DAG.getConstant(Val: 63, DL, VT: MVT::i64);
4794 SDValue LL = Op.getOperand(i: 0);
4795 SDValue RL = Op.getOperand(i: 1);
4796 SDValue LH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LL, N2: C63);
4797 SDValue RH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: RL, N2: C63);
4798 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4799 // the high result in the even register. ISD::SMUL_LOHI is defined to
4800 // return the low half first, so the results are in reverse order.
4801 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4802 Op0: LL, Op1: RL, Even&: Ops[1], Odd&: Ops[0]);
4803 SDValue NegLLTimesRH = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LL, N2: RH);
4804 SDValue NegLHTimesRL = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LH, N2: RL);
4805 SDValue NegSum = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NegLLTimesRH, N2: NegLHTimesRL);
4806 Ops[1] = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Ops[1], N2: NegSum);
4807 }
4808 return DAG.getMergeValues(Ops, dl: DL);
4809}
4810
4811SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
4812 SelectionDAG &DAG) const {
4813 EVT VT = Op.getValueType();
4814 SDLoc DL(Op);
4815 SDValue Ops[2];
4816 if (is32Bit(VT))
4817 // Just do a normal 64-bit multiplication and extract the results.
4818 // We define this so that it can be used for constant division.
4819 lowerMUL_LOHI32(DAG, DL, Extend: ISD::ZERO_EXTEND, Op0: Op.getOperand(i: 0),
4820 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4821 else
4822 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4823 // the high result in the even register. ISD::UMUL_LOHI is defined to
4824 // return the low half first, so the results are in reverse order.
4825 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4826 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4827 return DAG.getMergeValues(Ops, dl: DL);
4828}
4829
4830SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
4831 SelectionDAG &DAG) const {
4832 SDValue Op0 = Op.getOperand(i: 0);
4833 SDValue Op1 = Op.getOperand(i: 1);
4834 EVT VT = Op.getValueType();
4835 SDLoc DL(Op);
4836
4837 // We use DSGF for 32-bit division. This means the first operand must
4838 // always be 64-bit, and the second operand should be 32-bit whenever
4839 // that is possible, to improve performance.
4840 if (is32Bit(VT))
4841 Op0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: Op0);
4842 else if (DAG.ComputeNumSignBits(Op: Op1) > 32)
4843 Op1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Op1);
4844
4845 // DSG(F) returns the remainder in the even register and the
4846 // quotient in the odd register.
4847 SDValue Ops[2];
4848 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SDIVREM, Op0, Op1, Even&: Ops[1], Odd&: Ops[0]);
4849 return DAG.getMergeValues(Ops, dl: DL);
4850}
4851
4852SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
4853 SelectionDAG &DAG) const {
4854 EVT VT = Op.getValueType();
4855 SDLoc DL(Op);
4856
4857 // DL(G) returns the remainder in the even register and the
4858 // quotient in the odd register.
4859 SDValue Ops[2];
4860 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UDIVREM,
4861 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4862 return DAG.getMergeValues(Ops, dl: DL);
4863}
4864
4865SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
4866 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");
4867
4868 // Get the known-zero masks for each operand.
4869 SDValue Ops[] = {Op.getOperand(i: 0), Op.getOperand(i: 1)};
4870 KnownBits Known[2] = {DAG.computeKnownBits(Op: Ops[0]),
4871 DAG.computeKnownBits(Op: Ops[1])};
4872
4873 // See if the upper 32 bits of one operand and the lower 32 bits of the
4874 // other are known zero. They are the low and high operands respectively.
4875 uint64_t Masks[] = { Known[0].Zero.getZExtValue(),
4876 Known[1].Zero.getZExtValue() };
4877 unsigned High, Low;
4878 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
4879 High = 1, Low = 0;
4880 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
4881 High = 0, Low = 1;
4882 else
4883 return Op;
4884
4885 SDValue LowOp = Ops[Low];
4886 SDValue HighOp = Ops[High];
4887
4888 // If the high part is a constant, we're better off using IILH.
4889 if (HighOp.getOpcode() == ISD::Constant)
4890 return Op;
4891
4892 // If the low part is a constant that is outside the range of LHI,
4893 // then we're better off using IILF.
4894 if (LowOp.getOpcode() == ISD::Constant) {
4895 int64_t Value = int32_t(LowOp->getAsZExtVal());
4896 if (!isInt<16>(x: Value))
4897 return Op;
4898 }
4899
4900 // Check whether the high part is an AND that doesn't change the
4901 // high 32 bits and just masks out low bits. We can skip it if so.
4902 if (HighOp.getOpcode() == ISD::AND &&
4903 HighOp.getOperand(i: 1).getOpcode() == ISD::Constant) {
4904 SDValue HighOp0 = HighOp.getOperand(i: 0);
4905 uint64_t Mask = HighOp.getConstantOperandVal(i: 1);
4906 if (DAG.MaskedValueIsZero(Op: HighOp0, Mask: APInt(64, ~(Mask | 0xffffffff))))
4907 HighOp = HighOp0;
4908 }
4909
4910 // Take advantage of the fact that all GR32 operations only change the
4911 // low 32 bits by truncating Low to an i32 and inserting it directly
4912 // using a subreg. The interesting cases are those where the truncation
4913 // can be folded.
4914 SDLoc DL(Op);
4915 SDValue Low32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: LowOp);
4916 return DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_l32, DL,
4917 VT: MVT::i64, Operand: HighOp, Subreg: Low32);
4918}
4919
4920// Lower SADDO/SSUBO/UADDO/USUBO nodes.
4921SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
4922 SelectionDAG &DAG) const {
4923 SDNode *N = Op.getNode();
4924 SDValue LHS = N->getOperand(Num: 0);
4925 SDValue RHS = N->getOperand(Num: 1);
4926 SDLoc DL(N);
4927
4928 if (N->getValueType(ResNo: 0) == MVT::i128) {
4929 unsigned BaseOp = 0;
4930 unsigned FlagOp = 0;
4931 bool IsBorrow = false;
4932 switch (Op.getOpcode()) {
4933 default: llvm_unreachable("Unknown instruction!");
4934 case ISD::UADDO:
4935 BaseOp = ISD::ADD;
4936 FlagOp = SystemZISD::VACC;
4937 break;
4938 case ISD::USUBO:
4939 BaseOp = ISD::SUB;
4940 FlagOp = SystemZISD::VSCBI;
4941 IsBorrow = true;
4942 break;
4943 }
4944 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
4945 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
4946 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
4947 N2: DAG.getValueType(MVT::i1));
4948 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
4949 if (IsBorrow)
4950 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
4951 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
4952 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
4953 }
4954
4955 unsigned BaseOp = 0;
4956 unsigned CCValid = 0;
4957 unsigned CCMask = 0;
4958
4959 switch (Op.getOpcode()) {
4960 default: llvm_unreachable("Unknown instruction!");
4961 case ISD::SADDO:
4962 BaseOp = SystemZISD::SADDO;
4963 CCValid = SystemZ::CCMASK_ARITH;
4964 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4965 break;
4966 case ISD::SSUBO:
4967 BaseOp = SystemZISD::SSUBO;
4968 CCValid = SystemZ::CCMASK_ARITH;
4969 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4970 break;
4971 case ISD::UADDO:
4972 BaseOp = SystemZISD::UADDO;
4973 CCValid = SystemZ::CCMASK_LOGICAL;
4974 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
4975 break;
4976 case ISD::USUBO:
4977 BaseOp = SystemZISD::USUBO;
4978 CCValid = SystemZ::CCMASK_LOGICAL;
4979 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
4980 break;
4981 }
4982
4983 SDVTList VTs = DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::i32);
4984 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS);
4985
4986 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
4987 if (N->getValueType(ResNo: 1) == MVT::i1)
4988 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
4989
4990 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
4991}
4992
4993static bool isAddCarryChain(SDValue Carry) {
4994 while (Carry.getOpcode() == ISD::UADDO_CARRY &&
4995 Carry->getValueType(ResNo: 0) != MVT::i128)
4996 Carry = Carry.getOperand(i: 2);
4997 return Carry.getOpcode() == ISD::UADDO &&
4998 Carry->getValueType(ResNo: 0) != MVT::i128;
4999}
5000
5001static bool isSubBorrowChain(SDValue Carry) {
5002 while (Carry.getOpcode() == ISD::USUBO_CARRY &&
5003 Carry->getValueType(ResNo: 0) != MVT::i128)
5004 Carry = Carry.getOperand(i: 2);
5005 return Carry.getOpcode() == ISD::USUBO &&
5006 Carry->getValueType(ResNo: 0) != MVT::i128;
5007}
5008
5009// Lower UADDO_CARRY/USUBO_CARRY nodes.
5010SDValue SystemZTargetLowering::lowerUADDSUBO_CARRY(SDValue Op,
5011 SelectionDAG &DAG) const {
5012
5013 SDNode *N = Op.getNode();
5014 MVT VT = N->getSimpleValueType(ResNo: 0);
5015
5016 // Let legalize expand this if it isn't a legal type yet.
5017 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
5018 return SDValue();
5019
5020 SDValue LHS = N->getOperand(Num: 0);
5021 SDValue RHS = N->getOperand(Num: 1);
5022 SDValue Carry = Op.getOperand(i: 2);
5023 SDLoc DL(N);
5024
5025 if (VT == MVT::i128) {
5026 unsigned BaseOp = 0;
5027 unsigned FlagOp = 0;
5028 bool IsBorrow = false;
5029 switch (Op.getOpcode()) {
5030 default: llvm_unreachable("Unknown instruction!");
5031 case ISD::UADDO_CARRY:
5032 BaseOp = SystemZISD::VAC;
5033 FlagOp = SystemZISD::VACCC;
5034 break;
5035 case ISD::USUBO_CARRY:
5036 BaseOp = SystemZISD::VSBI;
5037 FlagOp = SystemZISD::VSBCBI;
5038 IsBorrow = true;
5039 break;
5040 }
5041 if (IsBorrow)
5042 Carry = DAG.getNode(Opcode: ISD::XOR, DL, VT: Carry.getValueType(),
5043 N1: Carry, N2: DAG.getConstant(Val: 1, DL, VT: Carry.getValueType()));
5044 Carry = DAG.getZExtOrTrunc(Op: Carry, DL, VT: MVT::i128);
5045 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
5046 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
5047 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
5048 N2: DAG.getValueType(MVT::i1));
5049 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
5050 if (IsBorrow)
5051 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
5052 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
5053 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
5054 }
5055
5056 unsigned BaseOp = 0;
5057 unsigned CCValid = 0;
5058 unsigned CCMask = 0;
5059
5060 switch (Op.getOpcode()) {
5061 default: llvm_unreachable("Unknown instruction!");
5062 case ISD::UADDO_CARRY:
5063 if (!isAddCarryChain(Carry))
5064 return SDValue();
5065
5066 BaseOp = SystemZISD::ADDCARRY;
5067 CCValid = SystemZ::CCMASK_LOGICAL;
5068 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
5069 break;
5070 case ISD::USUBO_CARRY:
5071 if (!isSubBorrowChain(Carry))
5072 return SDValue();
5073
5074 BaseOp = SystemZISD::SUBCARRY;
5075 CCValid = SystemZ::CCMASK_LOGICAL;
5076 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
5077 break;
5078 }
5079
5080 // Set the condition code from the carry flag.
5081 Carry = DAG.getNode(Opcode: SystemZISD::GET_CCMASK, DL, VT: MVT::i32, N1: Carry,
5082 N2: DAG.getConstant(Val: CCValid, DL, VT: MVT::i32),
5083 N3: DAG.getConstant(Val: CCMask, DL, VT: MVT::i32));
5084
5085 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
5086 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS, N3: Carry);
5087
5088 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
5089 if (N->getValueType(ResNo: 1) == MVT::i1)
5090 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
5091
5092 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
5093}
5094
5095SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
5096 SelectionDAG &DAG) const {
5097 EVT VT = Op.getValueType();
5098 SDLoc DL(Op);
5099 Op = Op.getOperand(i: 0);
5100
5101 if (VT.getScalarSizeInBits() == 128) {
5102 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: Op);
5103 Op = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: MVT::v2i64, Operand: Op);
5104 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v2i64, DL,
5105 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i64));
5106 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5107 return Op;
5108 }
5109
5110 // Handle vector types via VPOPCT.
5111 if (VT.isVector()) {
5112 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Op);
5113 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::v16i8, Operand: Op);
5114 switch (VT.getScalarSizeInBits()) {
5115 case 8:
5116 break;
5117 case 16: {
5118 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
5119 SDValue Shift = DAG.getConstant(Val: 8, DL, VT: MVT::i32);
5120 SDValue Tmp = DAG.getNode(Opcode: SystemZISD::VSHL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
5121 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
5122 Op = DAG.getNode(Opcode: SystemZISD::VSRL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
5123 break;
5124 }
5125 case 32: {
5126 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
5127 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5128 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5129 break;
5130 }
5131 case 64: {
5132 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
5133 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5134 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Tmp);
5135 Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v4i32, Operand: Tmp);
5136 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5137 break;
5138 }
5139 default:
5140 llvm_unreachable("Unexpected type");
5141 }
5142 return Op;
5143 }
5144
5145 // Get the known-zero mask for the operand.
5146 KnownBits Known = DAG.computeKnownBits(Op);
5147 unsigned NumSignificantBits = Known.getMaxValue().getActiveBits();
5148 if (NumSignificantBits == 0)
5149 return DAG.getConstant(Val: 0, DL, VT);
5150
5151 // Skip known-zero high parts of the operand.
5152 int64_t OrigBitSize = VT.getSizeInBits();
5153 int64_t BitSize = llvm::bit_ceil(Value: NumSignificantBits);
5154 BitSize = std::min(a: BitSize, b: OrigBitSize);
5155
5156 // The POPCNT instruction counts the number of bits in each byte.
5157 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op);
5158 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::i64, Operand: Op);
5159 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
5160
5161 // Add up per-byte counts in a binary tree. All bits of Op at
5162 // position larger than BitSize remain zero throughout.
5163 for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
5164 SDValue Tmp = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Op, N2: DAG.getConstant(Val: I, DL, VT));
5165 if (BitSize != OrigBitSize)
5166 Tmp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Tmp,
5167 N2: DAG.getConstant(Val: ((uint64_t)1 << BitSize) - 1, DL, VT));
5168 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
5169 }
5170
5171 // Extract overall result from high byte.
5172 if (BitSize > 8)
5173 Op = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Op,
5174 N2: DAG.getConstant(Val: BitSize - 8, DL, VT));
5175
5176 return Op;
5177}
5178
5179SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
5180 SelectionDAG &DAG) const {
5181 SDLoc DL(Op);
5182 AtomicOrdering FenceOrdering =
5183 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
5184 SyncScope::ID FenceSSID =
5185 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
5186
5187 // The only fence that needs an instruction is a sequentially-consistent
5188 // cross-thread fence.
5189 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
5190 FenceSSID == SyncScope::System) {
5191 return SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL, VT: MVT::Other,
5192 Op1: Op.getOperand(i: 0)),
5193 0);
5194 }
5195
5196 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
5197 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
5198}
5199
5200SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
5201 SelectionDAG &DAG) const {
5202 EVT RegVT = Op.getValueType();
5203 if (RegVT.getSizeInBits() == 128)
5204 return lowerATOMIC_LDST_I128(Op, DAG);
5205 return lowerLoadF16(Op, DAG);
5206}
5207
5208SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
5209 SelectionDAG &DAG) const {
5210 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5211 if (Node->getMemoryVT().getSizeInBits() == 128)
5212 return lowerATOMIC_LDST_I128(Op, DAG);
5213 return lowerStoreF16(Op, DAG);
5214}
5215
5216SDValue SystemZTargetLowering::lowerATOMIC_LDST_I128(SDValue Op,
5217 SelectionDAG &DAG) const {
5218 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5219 assert(
5220 (Node->getMemoryVT() == MVT::i128 || Node->getMemoryVT() == MVT::f128) &&
5221 "Only custom lowering i128 or f128.");
5222 // Use same code to handle both legal and non-legal i128 types.
5223 SmallVector<SDValue, 2> Results;
5224 LowerOperationWrapper(N: Node, Results, DAG);
5225 return DAG.getMergeValues(Ops: Results, dl: SDLoc(Op));
5226}
5227
5228// Prepare for a Compare And Swap for a subword operation. This needs to be
5229// done in memory with 4 bytes at natural alignment.
5230static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
5231 SDValue &AlignedAddr, SDValue &BitShift,
5232 SDValue &NegBitShift) {
5233 EVT PtrVT = Addr.getValueType();
5234 EVT WideVT = MVT::i32;
5235
5236 // Get the address of the containing word.
5237 AlignedAddr = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Addr,
5238 N2: DAG.getSignedConstant(Val: -4, DL, VT: PtrVT));
5239
5240 // Get the number of bits that the word must be rotated left in order
5241 // to bring the field to the top bits of a GR32.
5242 BitShift = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: Addr,
5243 N2: DAG.getConstant(Val: 3, DL, VT: PtrVT));
5244 BitShift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: WideVT, Operand: BitShift);
5245
5246 // Get the complementing shift amount, for rotating a field in the top
5247 // bits back to its proper position.
5248 NegBitShift = DAG.getNode(Opcode: ISD::SUB, DL, VT: WideVT,
5249 N1: DAG.getConstant(Val: 0, DL, VT: WideVT), N2: BitShift);
5250
5251}
5252
5253// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first
5254// two into the fullword ATOMIC_LOADW_* operation given by Opcode.
5255SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
5256 SelectionDAG &DAG,
5257 unsigned Opcode) const {
5258 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5259
5260 // 32-bit operations need no special handling.
5261 EVT NarrowVT = Node->getMemoryVT();
5262 EVT WideVT = MVT::i32;
5263 if (NarrowVT == WideVT)
5264 return Op;
5265
5266 int64_t BitSize = NarrowVT.getSizeInBits();
5267 SDValue ChainIn = Node->getChain();
5268 SDValue Addr = Node->getBasePtr();
5269 SDValue Src2 = Node->getVal();
5270 MachineMemOperand *MMO = Node->getMemOperand();
5271 SDLoc DL(Node);
5272
5273 // Convert atomic subtracts of constants into additions.
5274 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
5275 if (auto *Const = dyn_cast<ConstantSDNode>(Val&: Src2)) {
5276 Opcode = SystemZISD::ATOMIC_LOADW_ADD;
5277 Src2 = DAG.getSignedConstant(Val: -Const->getSExtValue(), DL,
5278 VT: Src2.getValueType());
5279 }
5280
5281 SDValue AlignedAddr, BitShift, NegBitShift;
5282 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5283
5284 // Extend the source operand to 32 bits and prepare it for the inner loop.
5285 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
5286 // operations require the source to be shifted in advance. (This shift
5287 // can be folded if the source is constant.) For AND and NAND, the lower
5288 // bits must be set, while for other opcodes they should be left clear.
5289 if (Opcode != SystemZISD::ATOMIC_SWAPW)
5290 Src2 = DAG.getNode(Opcode: ISD::SHL, DL, VT: WideVT, N1: Src2,
5291 N2: DAG.getConstant(Val: 32 - BitSize, DL, VT: WideVT));
5292 if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
5293 Opcode == SystemZISD::ATOMIC_LOADW_NAND)
5294 Src2 = DAG.getNode(Opcode: ISD::OR, DL, VT: WideVT, N1: Src2,
5295 N2: DAG.getConstant(Val: uint32_t(-1) >> BitSize, DL, VT: WideVT));
5296
5297 // Construct the ATOMIC_LOADW_* node.
5298 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::Other);
5299 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
5300 DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5301 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops,
5302 MemVT: NarrowVT, MMO);
5303
5304 // Rotate the result of the final CS so that the field is in the lower
5305 // bits of a GR32, then truncate it.
5306 SDValue ResultShift = DAG.getNode(Opcode: ISD::ADD, DL, VT: WideVT, N1: BitShift,
5307 N2: DAG.getConstant(Val: BitSize, DL, VT: WideVT));
5308 SDValue Result = DAG.getNode(Opcode: ISD::ROTL, DL, VT: WideVT, N1: AtomicOp, N2: ResultShift);
5309
5310 SDValue RetOps[2] = { Result, AtomicOp.getValue(R: 1) };
5311 return DAG.getMergeValues(Ops: RetOps, dl: DL);
5312}
5313
5314// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations into
5315// ATOMIC_LOADW_SUBs and convert 32- and 64-bit operations into additions.
5316SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
5317 SelectionDAG &DAG) const {
5318 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5319 EVT MemVT = Node->getMemoryVT();
5320 if (MemVT == MVT::i32 || MemVT == MVT::i64) {
5321 // A full-width operation: negate and use LAA(G).
5322 assert(Op.getValueType() == MemVT && "Mismatched VTs");
5323 assert(Subtarget.hasInterlockedAccess1() &&
5324 "Should have been expanded by AtomicExpand pass.");
5325 SDValue Src2 = Node->getVal();
5326 SDLoc DL(Src2);
5327 SDValue NegSrc2 =
5328 DAG.getNode(Opcode: ISD::SUB, DL, VT: MemVT, N1: DAG.getConstant(Val: 0, DL, VT: MemVT), N2: Src2);
5329 return DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT,
5330 Chain: Node->getChain(), Ptr: Node->getBasePtr(), Val: NegSrc2,
5331 MMO: Node->getMemOperand());
5332 }
5333
5334 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_SUB);
5335}
5336
5337// Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
5338SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
5339 SelectionDAG &DAG) const {
5340 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5341 SDValue ChainIn = Node->getOperand(Num: 0);
5342 SDValue Addr = Node->getOperand(Num: 1);
5343 SDValue CmpVal = Node->getOperand(Num: 2);
5344 SDValue SwapVal = Node->getOperand(Num: 3);
5345 MachineMemOperand *MMO = Node->getMemOperand();
5346 SDLoc DL(Node);
5347
5348 if (Node->getMemoryVT() == MVT::i128) {
5349 // Use same code to handle both legal and non-legal i128 types.
5350 SmallVector<SDValue, 3> Results;
5351 LowerOperationWrapper(N: Node, Results, DAG);
5352 return DAG.getMergeValues(Ops: Results, dl: DL);
5353 }
5354
5355 // We have native support for 32-bit and 64-bit compare and swap, but we
5356 // still need to expand extracting the "success" result from the CC.
5357 EVT NarrowVT = Node->getMemoryVT();
5358 EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
5359 if (NarrowVT == WideVT) {
5360 SDVTList Tys = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5361 SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
5362 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP,
5363 dl: DL, VTList: Tys, Ops, MemVT: NarrowVT, MMO);
5364 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5365 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
5366
5367 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: AtomicOp.getValue(R: 0));
5368 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5369 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5370 return SDValue();
5371 }
5372
5373 // Convert 8-bit and 16-bit compare and swap to a loop, implemented
5374 // via a fullword ATOMIC_CMP_SWAPW operation.
5375 int64_t BitSize = NarrowVT.getSizeInBits();
5376
5377 SDValue AlignedAddr, BitShift, NegBitShift;
5378 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5379
5380 // Construct the ATOMIC_CMP_SWAPW node.
5381 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5382 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
5383 NegBitShift, DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5384 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAPW, dl: DL,
5385 VTList, Ops, MemVT: NarrowVT, MMO);
5386 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5387 CCValid: SystemZ::CCMASK_ICMP, CCMask: SystemZ::CCMASK_CMP_EQ);
5388
5389 // emitAtomicCmpSwapW() will zero extend the result (original value).
5390 SDValue OrigVal = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: WideVT, N1: AtomicOp.getValue(R: 0),
5391 N2: DAG.getValueType(NarrowVT));
5392 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: OrigVal);
5393 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5394 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5395 return SDValue();
5396}
5397
5398MachineMemOperand::Flags
5399SystemZTargetLowering::getTargetMMOFlags(const Instruction &I) const {
5400 // Because of how we convert atomic_load and atomic_store to normal loads and
5401 // stores in the DAG, we need to ensure that the MMOs are marked volatile
5402 // since DAGCombine hasn't been updated to account for atomic, but non
5403 // volatile loads. (See D57601)
5404 if (auto *SI = dyn_cast<StoreInst>(Val: &I))
5405 if (SI->isAtomic())
5406 return MachineMemOperand::MOVolatile;
5407 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
5408 if (LI->isAtomic())
5409 return MachineMemOperand::MOVolatile;
5410 if (auto *AI = dyn_cast<AtomicRMWInst>(Val: &I))
5411 if (AI->isAtomic())
5412 return MachineMemOperand::MOVolatile;
5413 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
5414 if (AI->isAtomic())
5415 return MachineMemOperand::MOVolatile;
5416 return MachineMemOperand::MONone;
5417}
5418
5419SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
5420 SelectionDAG &DAG) const {
5421 MachineFunction &MF = DAG.getMachineFunction();
5422 auto *Regs = Subtarget.getSpecialRegisters();
5423 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5424 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5425 "in GHC calling convention");
5426 return DAG.getCopyFromReg(Chain: Op.getOperand(i: 0), dl: SDLoc(Op),
5427 Reg: Regs->getStackPointerRegister(), VT: Op.getValueType());
5428}
5429
5430SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
5431 SelectionDAG &DAG) const {
5432 MachineFunction &MF = DAG.getMachineFunction();
5433 auto *Regs = Subtarget.getSpecialRegisters();
5434 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
5435
5436 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5437 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5438 "in GHC calling convention");
5439
5440 SDValue Chain = Op.getOperand(i: 0);
5441 SDValue NewSP = Op.getOperand(i: 1);
5442 SDValue Backchain;
5443 SDLoc DL(Op);
5444
5445 if (StoreBackchain) {
5446 SDValue OldSP = DAG.getCopyFromReg(
5447 Chain, dl: DL, Reg: Regs->getStackPointerRegister(), VT: MVT::i64);
5448 Backchain = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
5449 PtrInfo: MachinePointerInfo());
5450 }
5451
5452 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Regs->getStackPointerRegister(), N: NewSP);
5453
5454 if (StoreBackchain)
5455 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
5456 PtrInfo: MachinePointerInfo());
5457
5458 return Chain;
5459}
5460
5461SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
5462 SelectionDAG &DAG) const {
5463 bool IsData = Op.getConstantOperandVal(i: 4);
5464 if (!IsData)
5465 // Just preserve the chain.
5466 return Op.getOperand(i: 0);
5467
5468 SDLoc DL(Op);
5469 bool IsWrite = Op.getConstantOperandVal(i: 2);
5470 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
5471 auto *Node = cast<MemIntrinsicSDNode>(Val: Op.getNode());
5472 SDValue Ops[] = {Op.getOperand(i: 0), DAG.getTargetConstant(Val: Code, DL, VT: MVT::i32),
5473 Op.getOperand(i: 1)};
5474 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::PREFETCH, dl: DL,
5475 VTList: Node->getVTList(), Ops,
5476 MemVT: Node->getMemoryVT(), MMO: Node->getMemOperand());
5477}
5478
5479SDValue
5480SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
5481 SelectionDAG &DAG) const {
5482 unsigned Opcode, CCValid;
5483 if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) {
5484 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
5485 SDNode *Node = emitIntrinsicWithCCAndChain(DAG, Op, Opcode);
5486 SDValue CC = getCCResult(DAG, CCReg: SDValue(Node, 0));
5487 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Op.getNode(), 0), To: CC);
5488 return SDValue();
5489 }
5490
5491 return SDValue();
5492}
5493
5494SDValue
5495SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
5496 SelectionDAG &DAG) const {
5497 unsigned Opcode, CCValid;
5498 if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
5499 SDNode *Node = emitIntrinsicWithCC(DAG, Op, Opcode);
5500 if (Op->getNumValues() == 1)
5501 return getCCResult(DAG, CCReg: SDValue(Node, 0));
5502 assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");
5503 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op), VTList: Op->getVTList(),
5504 N1: SDValue(Node, 0), N2: getCCResult(DAG, CCReg: SDValue(Node, 1)));
5505 }
5506
5507 unsigned Id = Op.getConstantOperandVal(i: 0);
5508 switch (Id) {
5509 case Intrinsic::thread_pointer:
5510 return lowerThreadPointer(DL: SDLoc(Op), DAG);
5511
5512 case Intrinsic::s390_vpdi:
5513 return DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL: SDLoc(Op), VT: Op.getValueType(),
5514 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5515
5516 case Intrinsic::s390_vperm:
5517 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL: SDLoc(Op), VT: Op.getValueType(),
5518 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5519
5520 case Intrinsic::s390_vuphb:
5521 case Intrinsic::s390_vuphh:
5522 case Intrinsic::s390_vuphf:
5523 case Intrinsic::s390_vuphg:
5524 return DAG.getNode(Opcode: SystemZISD::UNPACK_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5525 Operand: Op.getOperand(i: 1));
5526
5527 case Intrinsic::s390_vuplhb:
5528 case Intrinsic::s390_vuplhh:
5529 case Intrinsic::s390_vuplhf:
5530 case Intrinsic::s390_vuplhg:
5531 return DAG.getNode(Opcode: SystemZISD::UNPACKL_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5532 Operand: Op.getOperand(i: 1));
5533
5534 case Intrinsic::s390_vuplb:
5535 case Intrinsic::s390_vuplhw:
5536 case Intrinsic::s390_vuplf:
5537 case Intrinsic::s390_vuplg:
5538 return DAG.getNode(Opcode: SystemZISD::UNPACK_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5539 Operand: Op.getOperand(i: 1));
5540
5541 case Intrinsic::s390_vupllb:
5542 case Intrinsic::s390_vupllh:
5543 case Intrinsic::s390_vupllf:
5544 case Intrinsic::s390_vupllg:
5545 return DAG.getNode(Opcode: SystemZISD::UNPACKL_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5546 Operand: Op.getOperand(i: 1));
5547
5548 case Intrinsic::s390_vsumb:
5549 case Intrinsic::s390_vsumh:
5550 case Intrinsic::s390_vsumgh:
5551 case Intrinsic::s390_vsumgf:
5552 case Intrinsic::s390_vsumqf:
5553 case Intrinsic::s390_vsumqg:
5554 return DAG.getNode(Opcode: SystemZISD::VSUM, DL: SDLoc(Op), VT: Op.getValueType(),
5555 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5556
5557 case Intrinsic::s390_vaq:
5558 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5559 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5560 case Intrinsic::s390_vaccb:
5561 case Intrinsic::s390_vacch:
5562 case Intrinsic::s390_vaccf:
5563 case Intrinsic::s390_vaccg:
5564 case Intrinsic::s390_vaccq:
5565 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(Op), VT: Op.getValueType(),
5566 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5567 case Intrinsic::s390_vacq:
5568 return DAG.getNode(Opcode: SystemZISD::VAC, DL: SDLoc(Op), VT: Op.getValueType(),
5569 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5570 case Intrinsic::s390_vacccq:
5571 return DAG.getNode(Opcode: SystemZISD::VACCC, DL: SDLoc(Op), VT: Op.getValueType(),
5572 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5573
5574 case Intrinsic::s390_vsq:
5575 return DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(Op), VT: Op.getValueType(),
5576 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5577 case Intrinsic::s390_vscbib:
5578 case Intrinsic::s390_vscbih:
5579 case Intrinsic::s390_vscbif:
5580 case Intrinsic::s390_vscbig:
5581 case Intrinsic::s390_vscbiq:
5582 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5583 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5584 case Intrinsic::s390_vsbiq:
5585 return DAG.getNode(Opcode: SystemZISD::VSBI, DL: SDLoc(Op), VT: Op.getValueType(),
5586 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5587 case Intrinsic::s390_vsbcbiq:
5588 return DAG.getNode(Opcode: SystemZISD::VSBCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5589 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5590
5591 case Intrinsic::s390_vmhb:
5592 case Intrinsic::s390_vmhh:
5593 case Intrinsic::s390_vmhf:
5594 case Intrinsic::s390_vmhg:
5595 case Intrinsic::s390_vmhq:
5596 return DAG.getNode(Opcode: ISD::MULHS, DL: SDLoc(Op), VT: Op.getValueType(),
5597 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5598 case Intrinsic::s390_vmlhb:
5599 case Intrinsic::s390_vmlhh:
5600 case Intrinsic::s390_vmlhf:
5601 case Intrinsic::s390_vmlhg:
5602 case Intrinsic::s390_vmlhq:
5603 return DAG.getNode(Opcode: ISD::MULHU, DL: SDLoc(Op), VT: Op.getValueType(),
5604 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5605
5606 case Intrinsic::s390_vmahb:
5607 case Intrinsic::s390_vmahh:
5608 case Intrinsic::s390_vmahf:
5609 case Intrinsic::s390_vmahg:
5610 case Intrinsic::s390_vmahq:
5611 return DAG.getNode(Opcode: SystemZISD::VMAH, DL: SDLoc(Op), VT: Op.getValueType(),
5612 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5613 case Intrinsic::s390_vmalhb:
5614 case Intrinsic::s390_vmalhh:
5615 case Intrinsic::s390_vmalhf:
5616 case Intrinsic::s390_vmalhg:
5617 case Intrinsic::s390_vmalhq:
5618 return DAG.getNode(Opcode: SystemZISD::VMALH, DL: SDLoc(Op), VT: Op.getValueType(),
5619 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5620
5621 case Intrinsic::s390_vmeb:
5622 case Intrinsic::s390_vmeh:
5623 case Intrinsic::s390_vmef:
5624 case Intrinsic::s390_vmeg:
5625 return DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5626 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5627 case Intrinsic::s390_vmleb:
5628 case Intrinsic::s390_vmleh:
5629 case Intrinsic::s390_vmlef:
5630 case Intrinsic::s390_vmleg:
5631 return DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5632 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5633 case Intrinsic::s390_vmob:
5634 case Intrinsic::s390_vmoh:
5635 case Intrinsic::s390_vmof:
5636 case Intrinsic::s390_vmog:
5637 return DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5638 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5639 case Intrinsic::s390_vmlob:
5640 case Intrinsic::s390_vmloh:
5641 case Intrinsic::s390_vmlof:
5642 case Intrinsic::s390_vmlog:
5643 return DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5644 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5645
5646 case Intrinsic::s390_vmaeb:
5647 case Intrinsic::s390_vmaeh:
5648 case Intrinsic::s390_vmaef:
5649 case Intrinsic::s390_vmaeg:
5650 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5651 N1: DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5652 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5653 N2: Op.getOperand(i: 3));
5654 case Intrinsic::s390_vmaleb:
5655 case Intrinsic::s390_vmaleh:
5656 case Intrinsic::s390_vmalef:
5657 case Intrinsic::s390_vmaleg:
5658 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5659 N1: DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5660 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5661 N2: Op.getOperand(i: 3));
5662 case Intrinsic::s390_vmaob:
5663 case Intrinsic::s390_vmaoh:
5664 case Intrinsic::s390_vmaof:
5665 case Intrinsic::s390_vmaog:
5666 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5667 N1: DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5668 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5669 N2: Op.getOperand(i: 3));
5670 case Intrinsic::s390_vmalob:
5671 case Intrinsic::s390_vmaloh:
5672 case Intrinsic::s390_vmalof:
5673 case Intrinsic::s390_vmalog:
5674 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5675 N1: DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5676 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5677 N2: Op.getOperand(i: 3));
5678 }
5679
5680 return SDValue();
5681}
5682
5683namespace {
5684// Says that SystemZISD operation Opcode can be used to perform the equivalent
5685// of a VPERM with permute vector Bytes. If Opcode takes three operands,
5686// Operand is the constant third operand, otherwise it is the number of
5687// bytes in each element of the result.
5688struct Permute {
5689 unsigned Opcode;
5690 unsigned Operand;
5691 unsigned char Bytes[SystemZ::VectorBytes];
5692};
5693}
5694
5695static const Permute PermuteForms[] = {
5696 // VMRHG
5697 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 8,
5698 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
5699 // VMRHF
5700 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 4,
5701 .Bytes: { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
5702 // VMRHH
5703 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 2,
5704 .Bytes: { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
5705 // VMRHB
5706 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 1,
5707 .Bytes: { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
5708 // VMRLG
5709 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 8,
5710 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
5711 // VMRLF
5712 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 4,
5713 .Bytes: { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
5714 // VMRLH
5715 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 2,
5716 .Bytes: { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
5717 // VMRLB
5718 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 1,
5719 .Bytes: { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
5720 // VPKG
5721 { .Opcode: SystemZISD::PACK, .Operand: 4,
5722 .Bytes: { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
5723 // VPKF
5724 { .Opcode: SystemZISD::PACK, .Operand: 2,
5725 .Bytes: { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
5726 // VPKH
5727 { .Opcode: SystemZISD::PACK, .Operand: 1,
5728 .Bytes: { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
5729 // VPDI V1, V2, 4 (low half of V1, high half of V2)
5730 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 4,
5731 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
5732 // VPDI V1, V2, 1 (high half of V1, low half of V2)
5733 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 1,
5734 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
5735};
5736
5737// Called after matching a vector shuffle against a particular pattern.
5738// Both the original shuffle and the pattern have two vector operands.
5739// OpNos[0] is the operand of the original shuffle that should be used for
5740// operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
5741// OpNos[1] is the same for operand 1 of the pattern. Resolve these -1s and
5742// set OpNo0 and OpNo1 to the shuffle operands that should actually be used
5743// for operands 0 and 1 of the pattern.
5744static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
5745 if (OpNos[0] < 0) {
5746 if (OpNos[1] < 0)
5747 return false;
5748 OpNo0 = OpNo1 = OpNos[1];
5749 } else if (OpNos[1] < 0) {
5750 OpNo0 = OpNo1 = OpNos[0];
5751 } else {
5752 OpNo0 = OpNos[0];
5753 OpNo1 = OpNos[1];
5754 }
5755 return true;
5756}
5757
5758// Bytes is a VPERM-like permute vector, except that -1 is used for
5759// undefined bytes. Return true if the VPERM can be implemented using P.
5760// When returning true set OpNo0 to the VPERM operand that should be
5761// used for operand 0 of P and likewise OpNo1 for operand 1 of P.
5762//
5763// For example, if swapping the VPERM operands allows P to match, OpNo0
5764// will be 1 and OpNo1 will be 0. If instead Bytes only refers to one
5765// operand, but rewriting it to use two duplicated operands allows it to
5766// match P, then OpNo0 and OpNo1 will be the same.
5767static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
5768 unsigned &OpNo0, unsigned &OpNo1) {
5769 int OpNos[] = { -1, -1 };
5770 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5771 int Elt = Bytes[I];
5772 if (Elt >= 0) {
5773 // Make sure that the two permute vectors use the same suboperand
5774 // byte number. Only the operand numbers (the high bits) are
5775 // allowed to differ.
5776 if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
5777 return false;
5778 int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
5779 int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
5780 // Make sure that the operand mappings are consistent with previous
5781 // elements.
5782 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5783 return false;
5784 OpNos[ModelOpNo] = RealOpNo;
5785 }
5786 }
5787 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5788}
5789
5790// As above, but search for a matching permute.
5791static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
5792 unsigned &OpNo0, unsigned &OpNo1) {
5793 for (auto &P : PermuteForms)
5794 if (matchPermute(Bytes, P, OpNo0, OpNo1))
5795 return &P;
5796 return nullptr;
5797}
5798
5799// Bytes is a VPERM-like permute vector, except that -1 is used for
5800// undefined bytes. This permute is an operand of an outer permute.
5801// See whether redistributing the -1 bytes gives a shuffle that can be
5802// implemented using P. If so, set Transform to a VPERM-like permute vector
5803// that, when applied to the result of P, gives the original permute in Bytes.
5804static bool matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5805 const Permute &P,
5806 SmallVectorImpl<int> &Transform) {
5807 unsigned To = 0;
5808 for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
5809 int Elt = Bytes[From];
5810 if (Elt < 0)
5811 // Byte number From of the result is undefined.
5812 Transform[From] = -1;
5813 else {
5814 while (P.Bytes[To] != Elt) {
5815 To += 1;
5816 if (To == SystemZ::VectorBytes)
5817 return false;
5818 }
5819 Transform[From] = To;
5820 }
5821 }
5822 return true;
5823}
5824
5825// As above, but search for a matching permute.
5826static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5827 SmallVectorImpl<int> &Transform) {
5828 for (auto &P : PermuteForms)
5829 if (matchDoublePermute(Bytes, P, Transform))
5830 return &P;
5831 return nullptr;
5832}
5833
5834// Convert the mask of the given shuffle op into a byte-level mask,
5835// as if it had type vNi8.
5836static bool getVPermMask(SDValue ShuffleOp,
5837 SmallVectorImpl<int> &Bytes) {
5838 EVT VT = ShuffleOp.getValueType();
5839 unsigned NumElements = VT.getVectorNumElements();
5840 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5841
5842 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: ShuffleOp)) {
5843 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5844 for (unsigned I = 0; I < NumElements; ++I) {
5845 int Index = VSN->getMaskElt(Idx: I);
5846 if (Index >= 0)
5847 for (unsigned J = 0; J < BytesPerElement; ++J)
5848 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5849 }
5850 return true;
5851 }
5852 if (SystemZISD::SPLAT == ShuffleOp.getOpcode() &&
5853 isa<ConstantSDNode>(Val: ShuffleOp.getOperand(i: 1))) {
5854 unsigned Index = ShuffleOp.getConstantOperandVal(i: 1);
5855 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5856 for (unsigned I = 0; I < NumElements; ++I)
5857 for (unsigned J = 0; J < BytesPerElement; ++J)
5858 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5859 return true;
5860 }
5861 return false;
5862}
5863
5864// Bytes is a VPERM-like permute vector, except that -1 is used for
5865// undefined bytes. See whether bytes [Start, Start + BytesPerElement) of
5866// the result come from a contiguous sequence of bytes from one input.
5867// Set Base to the selector for the first byte if so.
5868static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
5869 unsigned BytesPerElement, int &Base) {
5870 Base = -1;
5871 for (unsigned I = 0; I < BytesPerElement; ++I) {
5872 if (Bytes[Start + I] >= 0) {
5873 unsigned Elem = Bytes[Start + I];
5874 if (Base < 0) {
5875 Base = Elem - I;
5876 // Make sure the bytes would come from one input operand.
5877 if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
5878 return false;
5879 } else if (unsigned(Base) != Elem - I)
5880 return false;
5881 }
5882 }
5883 return true;
5884}
5885
5886// Bytes is a VPERM-like permute vector, except that -1 is used for
5887// undefined bytes. Return true if it can be performed using VSLDB.
5888// When returning true, set StartIndex to the shift amount and OpNo0
5889// and OpNo1 to the VPERM operands that should be used as the first
5890// and second shift operand respectively.
5891static bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes,
5892 unsigned &StartIndex, unsigned &OpNo0,
5893 unsigned &OpNo1) {
5894 int OpNos[] = { -1, -1 };
5895 int Shift = -1;
5896 for (unsigned I = 0; I < 16; ++I) {
5897 int Index = Bytes[I];
5898 if (Index >= 0) {
5899 int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
5900 int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
5901 int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
5902 if (Shift < 0)
5903 Shift = ExpectedShift;
5904 else if (Shift != ExpectedShift)
5905 return false;
5906 // Make sure that the operand mappings are consistent with previous
5907 // elements.
5908 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5909 return false;
5910 OpNos[ModelOpNo] = RealOpNo;
5911 }
5912 }
5913 StartIndex = Shift;
5914 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5915}
5916
5917// Create a node that performs P on operands Op0 and Op1, casting the
5918// operands to the appropriate type. The type of the result is determined by P.
5919static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5920 const Permute &P, SDValue Op0, SDValue Op1) {
5921 // VPDI (PERMUTE_DWORDS) always operates on v2i64s. The input
5922 // elements of a PACK are twice as wide as the outputs.
5923 unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
5924 P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
5925 P.Operand);
5926 // Cast both operands to the appropriate type.
5927 MVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBytes * 8),
5928 NumElements: SystemZ::VectorBytes / InBytes);
5929 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op0);
5930 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op1);
5931 SDValue Op;
5932 if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
5933 SDValue Op2 = DAG.getTargetConstant(Val: P.Operand, DL, VT: MVT::i32);
5934 Op = DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL, VT: InVT, N1: Op0, N2: Op1, N3: Op2);
5935 } else if (P.Opcode == SystemZISD::PACK) {
5936 MVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: P.Operand * 8),
5937 NumElements: SystemZ::VectorBytes / P.Operand);
5938 Op = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT: OutVT, N1: Op0, N2: Op1);
5939 } else {
5940 Op = DAG.getNode(Opcode: P.Opcode, DL, VT: InVT, N1: Op0, N2: Op1);
5941 }
5942 return Op;
5943}
5944
5945static bool isZeroVector(SDValue N) {
5946 if (N->getOpcode() == ISD::BITCAST)
5947 N = N->getOperand(Num: 0);
5948 if (N->getOpcode() == ISD::SPLAT_VECTOR)
5949 if (auto *Op = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0)))
5950 return Op->getZExtValue() == 0;
5951 return ISD::isBuildVectorAllZeros(N: N.getNode());
5952}
5953
5954// Return the index of the zero/undef vector, or UINT32_MAX if not found.
5955static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num) {
5956 for (unsigned I = 0; I < Num ; I++)
5957 if (isZeroVector(N: Ops[I]))
5958 return I;
5959 return UINT32_MAX;
5960}
5961
5962// Bytes is a VPERM-like permute vector, except that -1 is used for
5963// undefined bytes. Implement it on operands Ops[0] and Ops[1] using
5964// VSLDB or VPERM.
5965static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5966 SDValue *Ops,
5967 const SmallVectorImpl<int> &Bytes) {
5968 for (unsigned I = 0; I < 2; ++I)
5969 Ops[I] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Ops[I]);
5970
5971 // First see whether VSLDB can be used.
5972 unsigned StartIndex, OpNo0, OpNo1;
5973 if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
5974 return DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE, DL, VT: MVT::v16i8, N1: Ops[OpNo0],
5975 N2: Ops[OpNo1],
5976 N3: DAG.getTargetConstant(Val: StartIndex, DL, VT: MVT::i32));
5977
5978 // Fall back on VPERM. Construct an SDNode for the permute vector. Try to
5979 // eliminate a zero vector by reusing any zero index in the permute vector.
5980 unsigned ZeroVecIdx = findZeroVectorIdx(Ops: &Ops[0], Num: 2);
5981 if (ZeroVecIdx != UINT32_MAX) {
5982 bool MaskFirst = true;
5983 int ZeroIdx = -1;
5984 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5985 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5986 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
5987 if (OpNo == ZeroVecIdx && I == 0) {
5988 // If the first byte is zero, use mask as first operand.
5989 ZeroIdx = 0;
5990 break;
5991 }
5992 if (OpNo != ZeroVecIdx && Byte == 0) {
5993 // If mask contains a zero, use it by placing that vector first.
5994 ZeroIdx = I + SystemZ::VectorBytes;
5995 MaskFirst = false;
5996 break;
5997 }
5998 }
5999 if (ZeroIdx != -1) {
6000 SDValue IndexNodes[SystemZ::VectorBytes];
6001 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
6002 if (Bytes[I] >= 0) {
6003 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6004 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
6005 if (OpNo == ZeroVecIdx)
6006 IndexNodes[I] = DAG.getConstant(Val: ZeroIdx, DL, VT: MVT::i32);
6007 else {
6008 unsigned BIdx = MaskFirst ? Byte + SystemZ::VectorBytes : Byte;
6009 IndexNodes[I] = DAG.getConstant(Val: BIdx, DL, VT: MVT::i32);
6010 }
6011 } else
6012 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
6013 }
6014 SDValue Mask = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
6015 SDValue Src = ZeroVecIdx == 0 ? Ops[1] : Ops[0];
6016 if (MaskFirst)
6017 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Mask, N2: Src,
6018 N3: Mask);
6019 else
6020 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Src, N2: Mask,
6021 N3: Mask);
6022 }
6023 }
6024
6025 SDValue IndexNodes[SystemZ::VectorBytes];
6026 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6027 if (Bytes[I] >= 0)
6028 IndexNodes[I] = DAG.getConstant(Val: Bytes[I], DL, VT: MVT::i32);
6029 else
6030 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
6031 SDValue Op2 = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
6032 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Ops[0],
6033 N2: (!Ops[1].isUndef() ? Ops[1] : Ops[0]), N3: Op2);
6034}
6035
6036namespace {
6037// Describes a general N-operand vector shuffle.
6038struct GeneralShuffle {
6039 GeneralShuffle(EVT vt)
6040 : VT(vt), UnpackFromEltSize(UINT_MAX), UnpackLow(false) {}
6041 void addUndef();
6042 bool add(SDValue, unsigned);
6043 SDValue getNode(SelectionDAG &, const SDLoc &);
6044 void tryPrepareForUnpack();
6045 bool unpackWasPrepared() { return UnpackFromEltSize <= 4; }
6046 SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
6047
6048 // The operands of the shuffle.
6049 SmallVector<SDValue, SystemZ::VectorBytes> Ops;
6050
6051 // Index I is -1 if byte I of the result is undefined. Otherwise the
6052 // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
6053 // Bytes[I] / SystemZ::VectorBytes.
6054 SmallVector<int, SystemZ::VectorBytes> Bytes;
6055
6056 // The type of the shuffle result.
6057 EVT VT;
6058
6059 // Holds a value of 1, 2 or 4 if a final unpack has been prepared for.
6060 unsigned UnpackFromEltSize;
6061 // True if the final unpack uses the low half.
6062 bool UnpackLow;
6063};
6064} // namespace
6065
6066// Add an extra undefined element to the shuffle.
6067void GeneralShuffle::addUndef() {
6068 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
6069 for (unsigned I = 0; I < BytesPerElement; ++I)
6070 Bytes.push_back(Elt: -1);
6071}
6072
6073// Add an extra element to the shuffle, taking it from element Elem of Op.
6074// A null Op indicates a vector input whose value will be calculated later;
6075// there is at most one such input per shuffle and it always has the same
6076// type as the result. Aborts and returns false if the source vector elements
6077// of an EXTRACT_VECTOR_ELT are smaller than the destination elements. Per
6078// LLVM they become implicitly extended, but this is rare and not optimized.
6079bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
6080 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
6081
6082 // The source vector can have wider elements than the result,
6083 // either through an explicit TRUNCATE or because of type legalization.
6084 // We want the least significant part.
6085 EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
6086 unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
6087
6088 // Return false if the source elements are smaller than their destination
6089 // elements.
6090 if (FromBytesPerElement < BytesPerElement)
6091 return false;
6092
6093 unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
6094 (FromBytesPerElement - BytesPerElement));
6095
6096 // Look through things like shuffles and bitcasts.
6097 while (Op.getNode()) {
6098 if (Op.getOpcode() == ISD::BITCAST)
6099 Op = Op.getOperand(i: 0);
6100 else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
6101 // See whether the bytes we need come from a contiguous part of one
6102 // operand.
6103 SmallVector<int, SystemZ::VectorBytes> OpBytes;
6104 if (!getVPermMask(ShuffleOp: Op, Bytes&: OpBytes))
6105 break;
6106 int NewByte;
6107 if (!getShuffleInput(Bytes: OpBytes, Start: Byte, BytesPerElement, Base&: NewByte))
6108 break;
6109 if (NewByte < 0) {
6110 addUndef();
6111 return true;
6112 }
6113 Op = Op.getOperand(i: unsigned(NewByte) / SystemZ::VectorBytes);
6114 Byte = unsigned(NewByte) % SystemZ::VectorBytes;
6115 } else if (Op.isUndef()) {
6116 addUndef();
6117 return true;
6118 } else
6119 break;
6120 }
6121
6122 // Make sure that the source of the extraction is in Ops.
6123 unsigned OpNo = 0;
6124 for (; OpNo < Ops.size(); ++OpNo)
6125 if (Ops[OpNo] == Op)
6126 break;
6127 if (OpNo == Ops.size())
6128 Ops.push_back(Elt: Op);
6129
6130 // Add the element to Bytes.
6131 unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
6132 for (unsigned I = 0; I < BytesPerElement; ++I)
6133 Bytes.push_back(Elt: Base + I);
6134
6135 return true;
6136}
6137
6138// Return SDNodes for the completed shuffle.
6139SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
6140 assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
6141
6142 if (Ops.size() == 0)
6143 return DAG.getUNDEF(VT);
6144
6145 // Use a single unpack if possible as the last operation.
6146 tryPrepareForUnpack();
6147
6148 // Make sure that there are at least two shuffle operands.
6149 if (Ops.size() == 1)
6150 Ops.push_back(Elt: DAG.getUNDEF(VT: MVT::v16i8));
6151
6152 // Create a tree of shuffles, deferring root node until after the loop.
6153 // Try to redistribute the undefined elements of non-root nodes so that
6154 // the non-root shuffles match something like a pack or merge, then adjust
6155 // the parent node's permute vector to compensate for the new order.
6156 // Among other things, this copes with vectors like <2 x i16> that were
6157 // padded with undefined elements during type legalization.
6158 //
6159 // In the best case this redistribution will lead to the whole tree
6160 // using packs and merges. It should rarely be a loss in other cases.
6161 unsigned Stride = 1;
6162 for (; Stride * 2 < Ops.size(); Stride *= 2) {
6163 for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
6164 SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
6165
6166 // Create a mask for just these two operands.
6167 SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes);
6168 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6169 unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
6170 unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
6171 if (OpNo == I)
6172 NewBytes[J] = Byte;
6173 else if (OpNo == I + Stride)
6174 NewBytes[J] = SystemZ::VectorBytes + Byte;
6175 else
6176 NewBytes[J] = -1;
6177 }
6178 // See if it would be better to reorganize NewMask to avoid using VPERM.
6179 SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes);
6180 if (const Permute *P = matchDoublePermute(Bytes: NewBytes, Transform&: NewBytesMap)) {
6181 Ops[I] = getPermuteNode(DAG, DL, P: *P, Op0: SubOps[0], Op1: SubOps[1]);
6182 // Applying NewBytesMap to Ops[I] gets back to NewBytes.
6183 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6184 if (NewBytes[J] >= 0) {
6185 assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
6186 "Invalid double permute");
6187 Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
6188 } else
6189 assert(NewBytesMap[J] < 0 && "Invalid double permute");
6190 }
6191 } else {
6192 // Just use NewBytes on the operands.
6193 Ops[I] = getGeneralPermuteNode(DAG, DL, Ops: SubOps, Bytes: NewBytes);
6194 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
6195 if (NewBytes[J] >= 0)
6196 Bytes[J] = I * SystemZ::VectorBytes + J;
6197 }
6198 }
6199 }
6200
6201 // Now we just have 2 inputs. Put the second operand in Ops[1].
6202 if (Stride > 1) {
6203 Ops[1] = Ops[Stride];
6204 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6205 if (Bytes[I] >= int(SystemZ::VectorBytes))
6206 Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
6207 }
6208
6209 // Look for an instruction that can do the permute without resorting
6210 // to VPERM.
6211 unsigned OpNo0, OpNo1;
6212 SDValue Op;
6213 if (unpackWasPrepared() && Ops[1].isUndef())
6214 Op = Ops[0];
6215 else if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
6216 Op = getPermuteNode(DAG, DL, P: *P, Op0: Ops[OpNo0], Op1: Ops[OpNo1]);
6217 else
6218 Op = getGeneralPermuteNode(DAG, DL, Ops: &Ops[0], Bytes);
6219
6220 Op = insertUnpackIfPrepared(DAG, DL, Op);
6221
6222 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6223}
6224
6225#ifndef NDEBUG
6226static void dumpBytes(const SmallVectorImpl<int> &Bytes, std::string Msg) {
6227 dbgs() << Msg.c_str() << " { ";
6228 for (unsigned I = 0; I < Bytes.size(); I++)
6229 dbgs() << Bytes[I] << " ";
6230 dbgs() << "}\n";
6231}
6232#endif
6233
6234// If the Bytes vector matches an unpack operation, prepare to do the unpack
6235// after all else by removing the zero vector and the effect of the unpack on
6236// Bytes.
6237void GeneralShuffle::tryPrepareForUnpack() {
6238 uint32_t ZeroVecOpNo = findZeroVectorIdx(Ops: &Ops[0], Num: Ops.size());
6239 if (ZeroVecOpNo == UINT32_MAX || Ops.size() == 1)
6240 return;
6241
6242 // Only do this if removing the zero vector reduces the depth, otherwise
6243 // the critical path will increase with the final unpack.
6244 if (Ops.size() > 2 &&
6245 Log2_32_Ceil(Value: Ops.size()) == Log2_32_Ceil(Value: Ops.size() - 1))
6246 return;
6247
6248 // Find an unpack that would allow removing the zero vector from Ops.
6249 UnpackFromEltSize = 1;
6250 for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
6251 bool MatchUnpack = true;
6252 SmallVector<int, SystemZ::VectorBytes> SrcBytes;
6253 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes; Elt++) {
6254 unsigned ToEltSize = UnpackFromEltSize * 2;
6255 bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
6256 if (!IsZextByte)
6257 SrcBytes.push_back(Elt: Bytes[Elt]);
6258 if (Bytes[Elt] != -1) {
6259 unsigned OpNo = unsigned(Bytes[Elt]) / SystemZ::VectorBytes;
6260 if (IsZextByte != (OpNo == ZeroVecOpNo)) {
6261 MatchUnpack = false;
6262 break;
6263 }
6264 }
6265 }
6266 if (MatchUnpack) {
6267 if (Ops.size() == 2) {
6268 // Don't use unpack if a single source operand needs rearrangement.
6269 bool CanUseUnpackLow = true, CanUseUnpackHigh = true;
6270 for (unsigned i = 0; i < SystemZ::VectorBytes / 2; i++) {
6271 if (SrcBytes[i] == -1)
6272 continue;
6273 if (SrcBytes[i] % 16 != int(i))
6274 CanUseUnpackHigh = false;
6275 if (SrcBytes[i] % 16 != int(i + SystemZ::VectorBytes / 2))
6276 CanUseUnpackLow = false;
6277 if (!CanUseUnpackLow && !CanUseUnpackHigh) {
6278 UnpackFromEltSize = UINT_MAX;
6279 return;
6280 }
6281 }
6282 if (!CanUseUnpackHigh)
6283 UnpackLow = true;
6284 }
6285 break;
6286 }
6287 }
6288 if (UnpackFromEltSize > 4)
6289 return;
6290
6291 LLVM_DEBUG(dbgs() << "Preparing for final unpack of element size "
6292 << UnpackFromEltSize << ". Zero vector is Op#" << ZeroVecOpNo
6293 << ".\n";
6294 dumpBytes(Bytes, "Original Bytes vector:"););
6295
6296 // Apply the unpack in reverse to the Bytes array.
6297 unsigned B = 0;
6298 if (UnpackLow) {
6299 while (B < SystemZ::VectorBytes / 2)
6300 Bytes[B++] = -1;
6301 }
6302 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes;) {
6303 Elt += UnpackFromEltSize;
6304 for (unsigned i = 0; i < UnpackFromEltSize; i++, Elt++, B++)
6305 Bytes[B] = Bytes[Elt];
6306 }
6307 if (!UnpackLow) {
6308 while (B < SystemZ::VectorBytes)
6309 Bytes[B++] = -1;
6310 }
6311
6312 // Remove the zero vector from Ops
6313 Ops.erase(CI: &Ops[ZeroVecOpNo]);
6314 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6315 if (Bytes[I] >= 0) {
6316 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6317 if (OpNo > ZeroVecOpNo)
6318 Bytes[I] -= SystemZ::VectorBytes;
6319 }
6320
6321 LLVM_DEBUG(dumpBytes(Bytes, "Resulting Bytes vector, zero vector removed:");
6322 dbgs() << "\n";);
6323}
6324
6325SDValue GeneralShuffle::insertUnpackIfPrepared(SelectionDAG &DAG,
6326 const SDLoc &DL,
6327 SDValue Op) {
6328 if (!unpackWasPrepared())
6329 return Op;
6330 unsigned InBits = UnpackFromEltSize * 8;
6331 EVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBits),
6332 NumElements: SystemZ::VectorBits / InBits);
6333 SDValue PackedOp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op);
6334 unsigned OutBits = InBits * 2;
6335 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: OutBits),
6336 NumElements: SystemZ::VectorBits / OutBits);
6337 return DAG.getNode(Opcode: UnpackLow ? SystemZISD::UNPACKL_LOW
6338 : SystemZISD::UNPACKL_HIGH,
6339 DL, VT: OutVT, Operand: PackedOp);
6340}
6341
6342// Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
6343static bool isScalarToVector(SDValue Op) {
6344 for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
6345 if (!Op.getOperand(i: I).isUndef())
6346 return false;
6347 return true;
6348}
6349
6350// Return a vector of type VT that contains Value in the first element.
6351// The other elements don't matter.
6352static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6353 SDValue Value) {
6354 // If we have a constant, replicate it to all elements and let the
6355 // BUILD_VECTOR lowering take care of it.
6356 if (Value.getOpcode() == ISD::Constant ||
6357 Value.getOpcode() == ISD::ConstantFP) {
6358 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value);
6359 return DAG.getBuildVector(VT, DL, Ops);
6360 }
6361 if (Value.isUndef())
6362 return DAG.getUNDEF(VT);
6363 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Value);
6364}
6365
6366// Return a vector of type VT in which Op0 is in element 0 and Op1 is in
6367// element 1. Used for cases in which replication is cheap.
6368static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6369 SDValue Op0, SDValue Op1) {
6370 if (Op0.isUndef()) {
6371 if (Op1.isUndef())
6372 return DAG.getUNDEF(VT);
6373 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op1);
6374 }
6375 if (Op1.isUndef())
6376 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0);
6377 return DAG.getNode(Opcode: SystemZISD::MERGE_HIGH, DL, VT,
6378 N1: buildScalarToVector(DAG, DL, VT, Value: Op0),
6379 N2: buildScalarToVector(DAG, DL, VT, Value: Op1));
6380}
6381
6382// Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
6383// vector for them.
6384static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
6385 SDValue Op1) {
6386 if (Op0.isUndef() && Op1.isUndef())
6387 return DAG.getUNDEF(VT: MVT::v2i64);
6388 // If one of the two inputs is undefined then replicate the other one,
6389 // in order to avoid using another register unnecessarily.
6390 if (Op0.isUndef())
6391 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6392 else if (Op1.isUndef())
6393 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6394 else {
6395 Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6396 Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6397 }
6398 return DAG.getNode(Opcode: SystemZISD::JOIN_DWORDS, DL, VT: MVT::v2i64, N1: Op0, N2: Op1);
6399}
6400
6401// If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
6402// better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
6403// the non-EXTRACT_VECTOR_ELT elements. See if the given BUILD_VECTOR
6404// would benefit from this representation and return it if so.
6405static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
6406 BuildVectorSDNode *BVN) {
6407 EVT VT = BVN->getValueType(ResNo: 0);
6408 unsigned NumElements = VT.getVectorNumElements();
6409
6410 // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
6411 // on byte vectors. If there are non-EXTRACT_VECTOR_ELT elements that still
6412 // need a BUILD_VECTOR, add an additional placeholder operand for that
6413 // BUILD_VECTOR and store its operands in ResidueOps.
6414 GeneralShuffle GS(VT);
6415 SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
6416 bool FoundOne = false;
6417 for (unsigned I = 0; I < NumElements; ++I) {
6418 SDValue Op = BVN->getOperand(Num: I);
6419 if (Op.getOpcode() == ISD::TRUNCATE)
6420 Op = Op.getOperand(i: 0);
6421 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6422 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
6423 unsigned Elem = Op.getConstantOperandVal(i: 1);
6424 if (!GS.add(Op: Op.getOperand(i: 0), Elem))
6425 return SDValue();
6426 FoundOne = true;
6427 } else if (Op.isUndef()) {
6428 GS.addUndef();
6429 } else {
6430 if (!GS.add(Op: SDValue(), Elem: ResidueOps.size()))
6431 return SDValue();
6432 ResidueOps.push_back(Elt: BVN->getOperand(Num: I));
6433 }
6434 }
6435
6436 // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
6437 if (!FoundOne)
6438 return SDValue();
6439
6440 // Create the BUILD_VECTOR for the remaining elements, if any.
6441 if (!ResidueOps.empty()) {
6442 while (ResidueOps.size() < NumElements)
6443 ResidueOps.push_back(Elt: DAG.getUNDEF(VT: ResidueOps[0].getValueType()));
6444 for (auto &Op : GS.Ops) {
6445 if (!Op.getNode()) {
6446 Op = DAG.getBuildVector(VT, DL: SDLoc(BVN), Ops: ResidueOps);
6447 break;
6448 }
6449 }
6450 }
6451 return GS.getNode(DAG, DL: SDLoc(BVN));
6452}
6453
6454bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
6455 if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Val&: Op)->isUnindexed())
6456 return true;
6457 if (auto *AL = dyn_cast<AtomicSDNode>(Val&: Op))
6458 if (AL->getOpcode() == ISD::ATOMIC_LOAD)
6459 return true;
6460 if (Subtarget.hasVectorEnhancements2() && Op.getOpcode() == SystemZISD::LRV)
6461 return true;
6462 return false;
6463}
6464
6465static SDValue mergeHighParts(SelectionDAG &DAG, const SDLoc &DL,
6466 unsigned MergedBits, EVT VT, SDValue Op0,
6467 SDValue Op1) {
6468 MVT IntVecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: MergedBits),
6469 NumElements: SystemZ::VectorBits / MergedBits);
6470 assert(VT.getSizeInBits() == 128 && IntVecVT.getSizeInBits() == 128 &&
6471 "Handling full vectors only.");
6472 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0);
6473 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op1);
6474 SDValue Op = DAG.getNode(Opcode: SystemZISD::MERGE_HIGH, DL, VT: IntVecVT, N1: Op0, N2: Op1);
6475 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6476}
6477
6478static SDValue buildFPVecFromScalars4(SelectionDAG &DAG, const SDLoc &DL,
6479 EVT VT, SmallVectorImpl<SDValue> &Elems,
6480 unsigned Pos) {
6481 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Op0: Elems[Pos + 0], Op1: Elems[Pos + 1]);
6482 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Op0: Elems[Pos + 2], Op1: Elems[Pos + 3]);
6483 // Avoid unnecessary undefs by reusing the other operand.
6484 if (Op01.isUndef()) {
6485 if (Op23.isUndef())
6486 return Op01;
6487 Op01 = Op23;
6488 } else if (Op23.isUndef())
6489 Op23 = Op01;
6490 // Merging identical replications is a no-op.
6491 if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
6492 return Op01;
6493 unsigned MergedBits = VT.getSimpleVT().getScalarSizeInBits() * 2;
6494 return mergeHighParts(DAG, DL, MergedBits, VT, Op0: Op01, Op1: Op23);
6495}
6496
6497// Combine GPR scalar values Elems into a vector of type VT.
6498SDValue
6499SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6500 SmallVectorImpl<SDValue> &Elems) const {
6501 // See whether there is a single replicated value.
6502 SDValue Single;
6503 unsigned int NumElements = Elems.size();
6504 unsigned int Count = 0;
6505 for (auto Elem : Elems) {
6506 if (!Elem.isUndef()) {
6507 if (!Single.getNode())
6508 Single = Elem;
6509 else if (Elem != Single) {
6510 Single = SDValue();
6511 break;
6512 }
6513 Count += 1;
6514 }
6515 }
6516 // There are three cases here:
6517 //
6518 // - if the only defined element is a loaded one, the best sequence
6519 // is a replicating load.
6520 //
6521 // - otherwise, if the only defined element is an i64 value, we will
6522 // end up with the same VLVGP sequence regardless of whether we short-cut
6523 // for replication or fall through to the later code.
6524 //
6525 // - otherwise, if the only defined element is an i32 or smaller value,
6526 // we would need 2 instructions to replicate it: VLVGP followed by VREPx.
6527 // This is only a win if the single defined element is used more than once.
6528 // In other cases we're better off using a single VLVGx.
6529 if (Single.getNode() && (Count > 1 || isVectorElementLoad(Op: Single)))
6530 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Single);
6531
6532 // If all elements are loads, use VLREP/VLEs (below).
6533 bool AllLoads = true;
6534 for (auto Elem : Elems)
6535 if (!isVectorElementLoad(Op: Elem)) {
6536 AllLoads = false;
6537 break;
6538 }
6539
6540 // The best way of building a v2i64 from two i64s is to use VLVGP.
6541 if (VT == MVT::v2i64 && !AllLoads)
6542 return joinDwords(DAG, DL, Op0: Elems[0], Op1: Elems[1]);
6543
6544 // Use a 64-bit merge high to combine two doubles.
6545 if (VT == MVT::v2f64 && !AllLoads)
6546 return buildMergeScalars(DAG, DL, VT, Op0: Elems[0], Op1: Elems[1]);
6547
6548 // Build v4f32 values directly from the FPRs:
6549 //
6550 // <Axxx> <Bxxx> <Cxxxx> <Dxxx>
6551 // V V VMRHF
6552 // <ABxx> <CDxx>
6553 // V VMRHG
6554 // <ABCD>
6555 if (VT == MVT::v4f32 && !AllLoads)
6556 return buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 0);
6557
6558 // Same for v8f16.
6559 if (VT == MVT::v8f16 && !AllLoads) {
6560 SDValue Op0123 = buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 0);
6561 SDValue Op4567 = buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 4);
6562 // Avoid unnecessary undefs by reusing the other operand.
6563 if (Op0123.isUndef())
6564 Op0123 = Op4567;
6565 else if (Op4567.isUndef())
6566 Op4567 = Op0123;
6567 // Merging identical replications is a no-op.
6568 if (Op0123.getOpcode() == SystemZISD::REPLICATE && Op0123 == Op4567)
6569 return Op0123;
6570 return mergeHighParts(DAG, DL, MergedBits: 64, VT, Op0: Op0123, Op1: Op4567);
6571 }
6572
6573 // Collect the constant terms.
6574 SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
6575 SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false);
6576
6577 unsigned NumConstants = 0;
6578 for (unsigned I = 0; I < NumElements; ++I) {
6579 SDValue Elem = Elems[I];
6580 if (Elem.getOpcode() == ISD::Constant ||
6581 Elem.getOpcode() == ISD::ConstantFP) {
6582 NumConstants += 1;
6583 Constants[I] = Elem;
6584 Done[I] = true;
6585 }
6586 }
6587 // If there was at least one constant, fill in the other elements of
6588 // Constants with undefs to get a full vector constant and use that
6589 // as the starting point.
6590 SDValue Result;
6591 SDValue ReplicatedVal;
6592 if (NumConstants > 0) {
6593 for (unsigned I = 0; I < NumElements; ++I)
6594 if (!Constants[I].getNode())
6595 Constants[I] = DAG.getUNDEF(VT: Elems[I].getValueType());
6596 Result = DAG.getBuildVector(VT, DL, Ops: Constants);
6597 } else {
6598 // Otherwise try to use VLREP or VLVGP to start the sequence in order to
6599 // avoid a false dependency on any previous contents of the vector
6600 // register.
6601
6602 // Use a VLREP if at least one element is a load. Make sure to replicate
6603 // the load with the most elements having its value.
6604 std::map<const SDNode*, unsigned> UseCounts;
6605 SDNode *LoadMaxUses = nullptr;
6606 for (unsigned I = 0; I < NumElements; ++I)
6607 if (isVectorElementLoad(Op: Elems[I])) {
6608 SDNode *Ld = Elems[I].getNode();
6609 unsigned Count = ++UseCounts[Ld];
6610 if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < Count)
6611 LoadMaxUses = Ld;
6612 }
6613 if (LoadMaxUses != nullptr) {
6614 ReplicatedVal = SDValue(LoadMaxUses, 0);
6615 Result = DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: ReplicatedVal);
6616 } else {
6617 // Try to use VLVGP.
6618 unsigned I1 = NumElements / 2 - 1;
6619 unsigned I2 = NumElements - 1;
6620 bool Def1 = !Elems[I1].isUndef();
6621 bool Def2 = !Elems[I2].isUndef();
6622 if (Def1 || Def2) {
6623 SDValue Elem1 = Elems[Def1 ? I1 : I2];
6624 SDValue Elem2 = Elems[Def2 ? I2 : I1];
6625 Result = DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
6626 Operand: joinDwords(DAG, DL, Op0: Elem1, Op1: Elem2));
6627 Done[I1] = true;
6628 Done[I2] = true;
6629 } else
6630 Result = DAG.getUNDEF(VT);
6631 }
6632 }
6633
6634 // Use VLVGx to insert the other elements.
6635 for (unsigned I = 0; I < NumElements; ++I)
6636 if (!Done[I] && !Elems[I].isUndef() && Elems[I] != ReplicatedVal)
6637 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: Result, N2: Elems[I],
6638 N3: DAG.getConstant(Val: I, DL, VT: MVT::i32));
6639 return Result;
6640}
6641
6642SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
6643 SelectionDAG &DAG) const {
6644 auto *BVN = cast<BuildVectorSDNode>(Val: Op.getNode());
6645 SDLoc DL(Op);
6646 EVT VT = Op.getValueType();
6647
6648 if (BVN->isConstant()) {
6649 if (SystemZVectorConstantInfo(BVN).isVectorConstantLegal(Subtarget))
6650 return Op;
6651
6652 // Fall back to loading it from memory.
6653 return SDValue();
6654 }
6655
6656 // See if we should use shuffles to construct the vector from other vectors.
6657 if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
6658 return Res;
6659
6660 // Detect SCALAR_TO_VECTOR conversions.
6661 if (isOperationLegal(Op: ISD::SCALAR_TO_VECTOR, VT) && isScalarToVector(Op))
6662 return buildScalarToVector(DAG, DL, VT, Value: Op.getOperand(i: 0));
6663
6664 // Otherwise use buildVector to build the vector up from GPRs.
6665 unsigned NumElements = Op.getNumOperands();
6666 SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
6667 for (unsigned I = 0; I < NumElements; ++I)
6668 Ops[I] = Op.getOperand(i: I);
6669 return buildVector(DAG, DL, VT, Elems&: Ops);
6670}
6671
6672SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6673 SelectionDAG &DAG) const {
6674 auto *VSN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
6675 SDLoc DL(Op);
6676 EVT VT = Op.getValueType();
6677 unsigned NumElements = VT.getVectorNumElements();
6678
6679 if (VSN->isSplat()) {
6680 SDValue Op0 = Op.getOperand(i: 0);
6681 unsigned Index = VSN->getSplatIndex();
6682 assert(Index < VT.getVectorNumElements() &&
6683 "Splat index should be defined and in first operand");
6684 // See whether the value we're splatting is directly available as a scalar.
6685 if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6686 Op0.getOpcode() == ISD::BUILD_VECTOR)
6687 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0.getOperand(i: Index));
6688 // Otherwise keep it as a vector-to-vector operation.
6689 return DAG.getNode(Opcode: SystemZISD::SPLAT, DL, VT, N1: Op.getOperand(i: 0),
6690 N2: DAG.getTargetConstant(Val: Index, DL, VT: MVT::i32));
6691 }
6692
6693 GeneralShuffle GS(VT);
6694 for (unsigned I = 0; I < NumElements; ++I) {
6695 int Elt = VSN->getMaskElt(Idx: I);
6696 if (Elt < 0)
6697 GS.addUndef();
6698 else if (!GS.add(Op: Op.getOperand(i: unsigned(Elt) / NumElements),
6699 Elem: unsigned(Elt) % NumElements))
6700 return SDValue();
6701 }
6702 return GS.getNode(DAG, DL: SDLoc(VSN));
6703}
6704
6705SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
6706 SelectionDAG &DAG) const {
6707 SDLoc DL(Op);
6708 // Just insert the scalar into element 0 of an undefined vector.
6709 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL,
6710 VT: Op.getValueType(), N1: DAG.getUNDEF(VT: Op.getValueType()),
6711 N2: Op.getOperand(i: 0), N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
6712}
6713
6714// Shift the lower 2 bytes of Op to the left in order to insert into the
6715// upper 2 bytes of the FP register.
6716static SDValue convertToF16(SDValue Op, SelectionDAG &DAG) {
6717 assert(Op.getSimpleValueType() == MVT::i64 &&
6718 "Expexted to convert i64 to f16.");
6719 SDLoc DL(Op);
6720 SDValue Shft = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: Op,
6721 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i64));
6722 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Shft);
6723 SDValue F16Val =
6724 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f16, Operand: BCast);
6725 return F16Val;
6726}
6727
6728// Extract Op into GPR and shift the 2 f16 bytes to the right.
6729static SDValue convertFromF16(SDValue Op, SDLoc DL, SelectionDAG &DAG) {
6730 assert(Op.getSimpleValueType() == MVT::f16 &&
6731 "Expected to convert f16 to i64.");
6732 SDNode *U32 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
6733 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f64,
6734 Operand: SDValue(U32, 0), Subreg: Op);
6735 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
6736 SDValue Shft = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: BCast,
6737 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i32));
6738 return Shft;
6739}
6740
6741SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
6742 SelectionDAG &DAG) const {
6743 // Handle insertions of floating-point values.
6744 SDLoc DL(Op);
6745 SDValue Op0 = Op.getOperand(i: 0);
6746 SDValue Op1 = Op.getOperand(i: 1);
6747 SDValue Op2 = Op.getOperand(i: 2);
6748 EVT VT = Op.getValueType();
6749
6750 // Insertions into constant indices of a v2f64 can be done using VPDI.
6751 // However, if the inserted value is a bitcast or a constant then it's
6752 // better to use GPRs, as below.
6753 if (VT == MVT::v2f64 &&
6754 Op1.getOpcode() != ISD::BITCAST &&
6755 Op1.getOpcode() != ISD::ConstantFP &&
6756 Op2.getOpcode() == ISD::Constant) {
6757 uint64_t Index = Op2->getAsZExtVal();
6758 unsigned Mask = VT.getVectorNumElements() - 1;
6759 if (Index <= Mask)
6760 return Op;
6761 }
6762
6763 // Otherwise bitcast to the equivalent integer form and insert via a GPR.
6764 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits());
6765 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VT.getVectorNumElements());
6766 SDValue IntOp1 =
6767 VT == MVT::v8f16
6768 ? DAG.getZExtOrTrunc(Op: convertFromF16(Op: Op1, DL, DAG), DL, VT: MVT::i32)
6769 : DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Op1);
6770 SDValue Res =
6771 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: IntVecVT,
6772 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0), N2: IntOp1, N3: Op2);
6773 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
6774}
6775
6776SDValue
6777SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
6778 SelectionDAG &DAG) const {
6779 // Handle extractions of floating-point values.
6780 SDLoc DL(Op);
6781 SDValue Op0 = Op.getOperand(i: 0);
6782 SDValue Op1 = Op.getOperand(i: 1);
6783 EVT VT = Op.getValueType();
6784 EVT VecVT = Op0.getValueType();
6785
6786 // Extractions of constant indices can be done directly.
6787 if (auto *CIndexN = dyn_cast<ConstantSDNode>(Val&: Op1)) {
6788 uint64_t Index = CIndexN->getZExtValue();
6789 unsigned Mask = VecVT.getVectorNumElements() - 1;
6790 if (Index <= Mask)
6791 return Op;
6792 }
6793
6794 // Otherwise bitcast to the equivalent integer form and extract via a GPR.
6795 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getSizeInBits());
6796 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VecVT.getVectorNumElements());
6797 MVT ExtrVT = IntVT == MVT::i16 ? MVT::i32 : IntVT;
6798 SDValue Extr = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ExtrVT,
6799 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0), N2: Op1);
6800 if (VT == MVT::f16)
6801 return convertToF16(Op: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Extr), DAG);
6802 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Extr);
6803}
6804
6805SDValue SystemZTargetLowering::
6806lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6807 SDValue PackedOp = Op.getOperand(i: 0);
6808 EVT OutVT = Op.getValueType();
6809 EVT InVT = PackedOp.getValueType();
6810 unsigned ToBits = OutVT.getScalarSizeInBits();
6811 unsigned FromBits = InVT.getScalarSizeInBits();
6812 unsigned StartOffset = 0;
6813
6814 // If the input is a VECTOR_SHUFFLE, there are a number of important
6815 // cases where we can directly implement the sign-extension of the
6816 // original input lanes of the shuffle.
6817 if (PackedOp.getOpcode() == ISD::VECTOR_SHUFFLE) {
6818 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: PackedOp.getNode());
6819 ArrayRef<int> ShuffleMask = SVN->getMask();
6820 int OutNumElts = OutVT.getVectorNumElements();
6821
6822 // Recognize the special case where the sign-extension can be done
6823 // by the VSEG instruction. Handled via the default expander.
6824 if (ToBits == 64 && OutNumElts == 2) {
6825 int NumElem = ToBits / FromBits;
6826 if (ShuffleMask[0] == NumElem - 1 && ShuffleMask[1] == 2 * NumElem - 1)
6827 return SDValue();
6828 }
6829
6830 // Recognize the special case where we can fold the shuffle by
6831 // replacing some of the UNPACK_HIGH with UNPACK_LOW.
6832 int StartOffsetCandidate = -1;
6833 for (int Elt = 0; Elt < OutNumElts; Elt++) {
6834 if (ShuffleMask[Elt] == -1)
6835 continue;
6836 if (ShuffleMask[Elt] % OutNumElts == Elt) {
6837 if (StartOffsetCandidate == -1)
6838 StartOffsetCandidate = ShuffleMask[Elt] - Elt;
6839 if (StartOffsetCandidate == ShuffleMask[Elt] - Elt)
6840 continue;
6841 }
6842 StartOffsetCandidate = -1;
6843 break;
6844 }
6845 if (StartOffsetCandidate != -1) {
6846 StartOffset = StartOffsetCandidate;
6847 PackedOp = PackedOp.getOperand(i: 0);
6848 }
6849 }
6850
6851 do {
6852 FromBits *= 2;
6853 unsigned OutNumElts = SystemZ::VectorBits / FromBits;
6854 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: FromBits), NumElements: OutNumElts);
6855 unsigned Opcode = SystemZISD::UNPACK_HIGH;
6856 if (StartOffset >= OutNumElts) {
6857 Opcode = SystemZISD::UNPACK_LOW;
6858 StartOffset -= OutNumElts;
6859 }
6860 PackedOp = DAG.getNode(Opcode, DL: SDLoc(PackedOp), VT: OutVT, Operand: PackedOp);
6861 } while (FromBits != ToBits);
6862 return PackedOp;
6863}
6864
6865// Lower a ZERO_EXTEND_VECTOR_INREG to a vector shuffle with a zero vector.
6866SDValue SystemZTargetLowering::
6867lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6868 SDValue PackedOp = Op.getOperand(i: 0);
6869 SDLoc DL(Op);
6870 EVT OutVT = Op.getValueType();
6871 EVT InVT = PackedOp.getValueType();
6872 unsigned InNumElts = InVT.getVectorNumElements();
6873 unsigned OutNumElts = OutVT.getVectorNumElements();
6874 unsigned NumInPerOut = InNumElts / OutNumElts;
6875
6876 SDValue ZeroVec =
6877 DAG.getSplatVector(VT: InVT, DL, Op: DAG.getConstant(Val: 0, DL, VT: InVT.getScalarType()));
6878
6879 SmallVector<int, 16> Mask(InNumElts);
6880 unsigned ZeroVecElt = InNumElts;
6881 for (unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
6882 unsigned MaskElt = PackedElt * NumInPerOut;
6883 unsigned End = MaskElt + NumInPerOut - 1;
6884 for (; MaskElt < End; MaskElt++)
6885 Mask[MaskElt] = ZeroVecElt++;
6886 Mask[MaskElt] = PackedElt;
6887 }
6888 SDValue Shuf = DAG.getVectorShuffle(VT: InVT, dl: DL, N1: PackedOp, N2: ZeroVec, Mask);
6889 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: OutVT, Operand: Shuf);
6890}
6891
6892SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
6893 unsigned ByScalar) const {
6894 // Look for cases where a vector shift can use the *_BY_SCALAR form.
6895 SDValue Op0 = Op.getOperand(i: 0);
6896 SDValue Op1 = Op.getOperand(i: 1);
6897 SDLoc DL(Op);
6898 EVT VT = Op.getValueType();
6899 unsigned ElemBitSize = VT.getScalarSizeInBits();
6900
6901 // See whether the shift vector is a splat represented as BUILD_VECTOR.
6902 if (auto *BVN = dyn_cast<BuildVectorSDNode>(Val&: Op1)) {
6903 APInt SplatBits, SplatUndef;
6904 unsigned SplatBitSize;
6905 bool HasAnyUndefs;
6906 // Check for constant splats. Use ElemBitSize as the minimum element
6907 // width and reject splats that need wider elements.
6908 if (BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6909 MinSplatBits: ElemBitSize, isBigEndian: true) &&
6910 SplatBitSize == ElemBitSize) {
6911 SDValue Shift = DAG.getConstant(Val: SplatBits.getZExtValue() & 0xfff,
6912 DL, VT: MVT::i32);
6913 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6914 }
6915 // Check for variable splats.
6916 BitVector UndefElements;
6917 SDValue Splat = BVN->getSplatValue(UndefElements: &UndefElements);
6918 if (Splat) {
6919 // Since i32 is the smallest legal type, we either need a no-op
6920 // or a truncation.
6921 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Splat);
6922 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6923 }
6924 }
6925
6926 // See whether the shift vector is a splat represented as SHUFFLE_VECTOR,
6927 // and the shift amount is directly available in a GPR.
6928 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: Op1)) {
6929 if (VSN->isSplat()) {
6930 SDValue VSNOp0 = VSN->getOperand(Num: 0);
6931 unsigned Index = VSN->getSplatIndex();
6932 assert(Index < VT.getVectorNumElements() &&
6933 "Splat index should be defined and in first operand");
6934 if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6935 VSNOp0.getOpcode() == ISD::BUILD_VECTOR) {
6936 // Since i32 is the smallest legal type, we either need a no-op
6937 // or a truncation.
6938 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32,
6939 Operand: VSNOp0.getOperand(i: Index));
6940 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6941 }
6942 }
6943 }
6944
6945 // Otherwise just treat the current form as legal.
6946 return Op;
6947}
6948
6949SDValue SystemZTargetLowering::lowerFSHL(SDValue Op, SelectionDAG &DAG) const {
6950 SDLoc DL(Op);
6951
6952 // i128 FSHL with a constant amount that is a multiple of 8 can be
6953 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
6954 // facility, FSHL with a constant amount less than 8 can be implemented
6955 // via SHL_DOUBLE_BIT, and FSHL with other constant amounts by a
6956 // combination of the two.
6957 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
6958 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
6959 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
6960 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
6961 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
6962 if (ShiftAmt > 120) {
6963 // For N in 121..128, fshl N == fshr (128 - N), and for 1 <= N < 8
6964 // SHR_DOUBLE_BIT emits fewer instructions.
6965 SDValue Val =
6966 DAG.getNode(Opcode: SystemZISD::SHR_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Op0, N2: Op1,
6967 N3: DAG.getTargetConstant(Val: 128 - ShiftAmt, DL, VT: MVT::i32));
6968 return DAG.getBitcast(VT: MVT::i128, V: Val);
6969 }
6970 SmallVector<int, 16> Mask(16);
6971 for (unsigned Elt = 0; Elt < 16; Elt++)
6972 Mask[Elt] = (ShiftAmt >> 3) + Elt;
6973 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
6974 if ((ShiftAmt & 7) == 0)
6975 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
6976 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op1, N2: Op1, Mask);
6977 SDValue Val =
6978 DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf1, N2: Shuf2,
6979 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
6980 return DAG.getBitcast(VT: MVT::i128, V: Val);
6981 }
6982 }
6983
6984 return SDValue();
6985}
6986
6987SDValue SystemZTargetLowering::lowerFSHR(SDValue Op, SelectionDAG &DAG) const {
6988 SDLoc DL(Op);
6989
6990 // i128 FSHR with a constant amount that is a multiple of 8 can be
6991 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
6992 // facility, FSHR with a constant amount less than 8 can be implemented
6993 // via SHR_DOUBLE_BIT, and FSHR with other constant amounts by a
6994 // combination of the two.
6995 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
6996 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
6997 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
6998 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
6999 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
7000 if (ShiftAmt > 120) {
7001 // For N in 121..128, fshr N == fshl (128 - N), and for 1 <= N < 8
7002 // SHL_DOUBLE_BIT emits fewer instructions.
7003 SDValue Val =
7004 DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Op0, N2: Op1,
7005 N3: DAG.getTargetConstant(Val: 128 - ShiftAmt, DL, VT: MVT::i32));
7006 return DAG.getBitcast(VT: MVT::i128, V: Val);
7007 }
7008 SmallVector<int, 16> Mask(16);
7009 for (unsigned Elt = 0; Elt < 16; Elt++)
7010 Mask[Elt] = 16 - (ShiftAmt >> 3) + Elt;
7011 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
7012 if ((ShiftAmt & 7) == 0)
7013 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
7014 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op0, Mask);
7015 SDValue Val =
7016 DAG.getNode(Opcode: SystemZISD::SHR_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf2, N2: Shuf1,
7017 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
7018 return DAG.getBitcast(VT: MVT::i128, V: Val);
7019 }
7020 }
7021
7022 return SDValue();
7023}
7024
7025static SDValue lowerAddrSpaceCast(SDValue Op, SelectionDAG &DAG) {
7026 SDLoc DL(Op);
7027 SDValue Src = Op.getOperand(i: 0);
7028 MVT DstVT = Op.getSimpleValueType();
7029
7030 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Val: Op.getNode());
7031 unsigned SrcAS = N->getSrcAddressSpace();
7032
7033 assert(SrcAS != N->getDestAddressSpace() &&
7034 "addrspacecast must be between different address spaces");
7035
7036 // addrspacecast [0 <- 1] : Assinging a ptr32 value to a 64-bit pointer.
7037 // addrspacecast [1 <- 0] : Assigining a 64-bit pointer to a ptr32 value.
7038 if (SrcAS == SYSTEMZAS::PTR32 && DstVT == MVT::i64) {
7039 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Src,
7040 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
7041 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
7042 } else if (DstVT == MVT::i32) {
7043 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Src);
7044 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Op,
7045 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
7046 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
7047 } else {
7048 report_fatal_error(reason: "Bad address space in addrspacecast");
7049 }
7050 return Op;
7051}
7052
7053SDValue SystemZTargetLowering::lowerFP_EXTEND(SDValue Op,
7054 SelectionDAG &DAG) const {
7055 SDValue In = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0);
7056 if (In.getSimpleValueType() != MVT::f16)
7057 return Op; // Legal
7058 return SDValue(); // Let legalizer emit the libcall.
7059}
7060
7061SDValue SystemZTargetLowering::useLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
7062 MVT VT, SDValue Arg, SDLoc DL,
7063 SDValue Chain, bool IsStrict) const {
7064 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
7065 MakeLibCallOptions CallOptions;
7066 SDValue Result;
7067 std::tie(args&: Result, args&: Chain) =
7068 makeLibCall(DAG, LC, RetVT: VT, Ops: Arg, CallOptions, dl: DL, Chain);
7069 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: DL) : Result;
7070}
7071
7072SDValue SystemZTargetLowering::lower_FP_TO_INT(SDValue Op,
7073 SelectionDAG &DAG) const {
7074 bool IsSigned = (Op->getOpcode() == ISD::FP_TO_SINT ||
7075 Op->getOpcode() == ISD::STRICT_FP_TO_SINT);
7076 bool IsStrict = Op->isStrictFPOpcode();
7077 SDLoc DL(Op);
7078 MVT VT = Op.getSimpleValueType();
7079 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
7080 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
7081 EVT InVT = InOp.getValueType();
7082
7083 // FP to unsigned is not directly supported on z10. Promoting an i32
7084 // result to (signed) i64 doesn't generate an inexact condition (fp
7085 // exception) for values that are outside the i32 range but in the i64
7086 // range, so use the default expansion.
7087 if (!Subtarget.hasFPExtension() && !IsSigned)
7088 // Expand i32/i64. F16 values will be recognized to fit and extended.
7089 return SDValue();
7090
7091 // Conversion from f16 is done via f32.
7092 if (InOp.getSimpleValueType() == MVT::f16) {
7093 SmallVector<SDValue, 2> Results;
7094 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
7095 return DAG.getMergeValues(Ops: Results, dl: DL);
7096 }
7097
7098 if (VT == MVT::i128) {
7099 RTLIB::Libcall LC =
7100 IsSigned ? RTLIB::getFPTOSINT(OpVT: InVT, RetVT: VT) : RTLIB::getFPTOUINT(OpVT: InVT, RetVT: VT);
7101 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
7102 }
7103
7104 return Op; // Legal
7105}
7106
7107SDValue SystemZTargetLowering::lower_INT_TO_FP(SDValue Op,
7108 SelectionDAG &DAG) const {
7109 bool IsSigned = (Op->getOpcode() == ISD::SINT_TO_FP ||
7110 Op->getOpcode() == ISD::STRICT_SINT_TO_FP);
7111 bool IsStrict = Op->isStrictFPOpcode();
7112 SDLoc DL(Op);
7113 MVT VT = Op.getSimpleValueType();
7114 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
7115 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
7116 EVT InVT = InOp.getValueType();
7117
7118 // Conversion to f16 is done via f32.
7119 if (VT == MVT::f16) {
7120 SmallVector<SDValue, 2> Results;
7121 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
7122 return DAG.getMergeValues(Ops: Results, dl: DL);
7123 }
7124
7125 // Unsigned to fp is not directly supported on z10.
7126 if (!Subtarget.hasFPExtension() && !IsSigned)
7127 return SDValue(); // Expand i64.
7128
7129 if (InVT == MVT::i128) {
7130 RTLIB::Libcall LC =
7131 IsSigned ? RTLIB::getSINTTOFP(OpVT: InVT, RetVT: VT) : RTLIB::getUINTTOFP(OpVT: InVT, RetVT: VT);
7132 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
7133 }
7134
7135 return Op; // Legal
7136}
7137
7138// Lower an f16 LOAD in case of no vector support.
7139SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
7140 SelectionDAG &DAG) const {
7141 EVT RegVT = Op.getValueType();
7142 assert(RegVT == MVT::f16 && "Expected to lower an f16 load.");
7143 (void)RegVT;
7144
7145 // Load as integer.
7146 SDLoc DL(Op);
7147 SDValue NewLd;
7148 if (auto *AtomicLd = dyn_cast<AtomicSDNode>(Val: Op.getNode())) {
7149 assert(EVT(RegVT) == AtomicLd->getMemoryVT() && "Unhandled f16 load");
7150 NewLd = DAG.getAtomicLoad(ExtType: ISD::EXTLOAD, dl: DL, MemVT: MVT::i16, VT: MVT::i64,
7151 Chain: AtomicLd->getChain(), Ptr: AtomicLd->getBasePtr(),
7152 MMO: AtomicLd->getMemOperand());
7153 } else {
7154 LoadSDNode *Ld = cast<LoadSDNode>(Val: Op.getNode());
7155 assert(EVT(RegVT) == Ld->getMemoryVT() && "Unhandled f16 load");
7156 NewLd = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i64, Chain: Ld->getChain(),
7157 Ptr: Ld->getBasePtr(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i16,
7158 Alignment: Ld->getBaseAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
7159 }
7160 SDValue F16Val = convertToF16(Op: NewLd, DAG);
7161 return DAG.getMergeValues(Ops: {F16Val, NewLd.getValue(R: 1)}, dl: DL);
7162}
7163
7164// Lower an f16 STORE in case of no vector support.
7165SDValue SystemZTargetLowering::lowerStoreF16(SDValue Op,
7166 SelectionDAG &DAG) const {
7167 SDLoc DL(Op);
7168 SDValue Shft = convertFromF16(Op: Op->getOperand(Num: 1), DL, DAG);
7169
7170 if (auto *AtomicSt = dyn_cast<AtomicSDNode>(Val: Op.getNode()))
7171 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MVT::i16, Chain: AtomicSt->getChain(),
7172 Ptr: Shft, Val: AtomicSt->getBasePtr(),
7173 MMO: AtomicSt->getMemOperand());
7174
7175 StoreSDNode *St = cast<StoreSDNode>(Val: Op.getNode());
7176 return DAG.getTruncStore(Chain: St->getChain(), dl: DL, Val: Shft, Ptr: St->getBasePtr(), SVT: MVT::i16,
7177 MMO: St->getMemOperand());
7178}
7179
7180SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op,
7181 SelectionDAG &DAG) const {
7182 SDLoc DL(Op);
7183 MVT ResultVT = Op.getSimpleValueType();
7184 SDValue Arg = Op.getOperand(i: 0);
7185 unsigned Check = Op.getConstantOperandVal(i: 1);
7186
7187 unsigned TDCMask = 0;
7188 if (Check & fcSNan)
7189 TDCMask |= SystemZ::TDCMASK_SNAN_PLUS | SystemZ::TDCMASK_SNAN_MINUS;
7190 if (Check & fcQNan)
7191 TDCMask |= SystemZ::TDCMASK_QNAN_PLUS | SystemZ::TDCMASK_QNAN_MINUS;
7192 if (Check & fcPosInf)
7193 TDCMask |= SystemZ::TDCMASK_INFINITY_PLUS;
7194 if (Check & fcNegInf)
7195 TDCMask |= SystemZ::TDCMASK_INFINITY_MINUS;
7196 if (Check & fcPosNormal)
7197 TDCMask |= SystemZ::TDCMASK_NORMAL_PLUS;
7198 if (Check & fcNegNormal)
7199 TDCMask |= SystemZ::TDCMASK_NORMAL_MINUS;
7200 if (Check & fcPosSubnormal)
7201 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_PLUS;
7202 if (Check & fcNegSubnormal)
7203 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_MINUS;
7204 if (Check & fcPosZero)
7205 TDCMask |= SystemZ::TDCMASK_ZERO_PLUS;
7206 if (Check & fcNegZero)
7207 TDCMask |= SystemZ::TDCMASK_ZERO_MINUS;
7208 SDValue TDCMaskV = DAG.getConstant(Val: TDCMask, DL, VT: MVT::i64);
7209
7210 SDValue Intr = DAG.getNode(Opcode: SystemZISD::TDC, DL, VT: ResultVT, N1: Arg, N2: TDCMaskV);
7211 return getCCResult(DAG, CCReg: Intr);
7212}
7213
7214SDValue SystemZTargetLowering::lowerREADCYCLECOUNTER(SDValue Op,
7215 SelectionDAG &DAG) const {
7216 SDLoc DL(Op);
7217 SDValue Chain = Op.getOperand(i: 0);
7218
7219 // STCKF only supports a memory operand, so we have to use a temporary.
7220 SDValue StackPtr = DAG.CreateStackTemporary(VT: MVT::i64);
7221 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
7222 MachinePointerInfo MPI =
7223 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7224
7225 // Use STCFK to store the TOD clock into the temporary.
7226 SDValue StoreOps[] = {Chain, StackPtr};
7227 Chain = DAG.getMemIntrinsicNode(
7228 Opcode: SystemZISD::STCKF, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps, MemVT: MVT::i64,
7229 PtrInfo: MPI, Alignment: MaybeAlign(), Flags: MachineMemOperand::MOStore);
7230
7231 // And read it back from there.
7232 return DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: StackPtr, PtrInfo: MPI);
7233}
7234
7235SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
7236 SelectionDAG &DAG) const {
7237 switch (Op.getOpcode()) {
7238 case ISD::FRAMEADDR:
7239 return lowerFRAMEADDR(Op, DAG);
7240 case ISD::RETURNADDR:
7241 return lowerRETURNADDR(Op, DAG);
7242 case ISD::BR_CC:
7243 return lowerBR_CC(Op, DAG);
7244 case ISD::SELECT_CC:
7245 return lowerSELECT_CC(Op, DAG);
7246 case ISD::SETCC:
7247 return lowerSETCC(Op, DAG);
7248 case ISD::STRICT_FSETCC:
7249 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: false);
7250 case ISD::STRICT_FSETCCS:
7251 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: true);
7252 case ISD::GlobalAddress:
7253 return lowerGlobalAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7254 case ISD::GlobalTLSAddress:
7255 return lowerGlobalTLSAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7256 case ISD::BlockAddress:
7257 return lowerBlockAddress(Node: cast<BlockAddressSDNode>(Val&: Op), DAG);
7258 case ISD::JumpTable:
7259 return lowerJumpTable(JT: cast<JumpTableSDNode>(Val&: Op), DAG);
7260 case ISD::ConstantPool:
7261 return lowerConstantPool(CP: cast<ConstantPoolSDNode>(Val&: Op), DAG);
7262 case ISD::BITCAST:
7263 return lowerBITCAST(Op, DAG);
7264 case ISD::VASTART:
7265 return lowerVASTART(Op, DAG);
7266 case ISD::VACOPY:
7267 return lowerVACOPY(Op, DAG);
7268 case ISD::DYNAMIC_STACKALLOC:
7269 return lowerDYNAMIC_STACKALLOC(Op, DAG);
7270 case ISD::GET_DYNAMIC_AREA_OFFSET:
7271 return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
7272 case ISD::MULHS:
7273 return lowerMULH(Op, DAG, Opcode: SystemZISD::SMUL_LOHI);
7274 case ISD::MULHU:
7275 return lowerMULH(Op, DAG, Opcode: SystemZISD::UMUL_LOHI);
7276 case ISD::SMUL_LOHI:
7277 return lowerSMUL_LOHI(Op, DAG);
7278 case ISD::UMUL_LOHI:
7279 return lowerUMUL_LOHI(Op, DAG);
7280 case ISD::SDIVREM:
7281 return lowerSDIVREM(Op, DAG);
7282 case ISD::UDIVREM:
7283 return lowerUDIVREM(Op, DAG);
7284 case ISD::SADDO:
7285 case ISD::SSUBO:
7286 case ISD::UADDO:
7287 case ISD::USUBO:
7288 return lowerXALUO(Op, DAG);
7289 case ISD::UADDO_CARRY:
7290 case ISD::USUBO_CARRY:
7291 return lowerUADDSUBO_CARRY(Op, DAG);
7292 case ISD::OR:
7293 return lowerOR(Op, DAG);
7294 case ISD::CTPOP:
7295 return lowerCTPOP(Op, DAG);
7296 case ISD::VECREDUCE_ADD:
7297 return lowerVECREDUCE_ADD(Op, DAG);
7298 case ISD::ATOMIC_FENCE:
7299 return lowerATOMIC_FENCE(Op, DAG);
7300 case ISD::ATOMIC_SWAP:
7301 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_SWAPW);
7302 case ISD::ATOMIC_STORE:
7303 return lowerATOMIC_STORE(Op, DAG);
7304 case ISD::ATOMIC_LOAD:
7305 return lowerATOMIC_LOAD(Op, DAG);
7306 case ISD::ATOMIC_LOAD_ADD:
7307 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_ADD);
7308 case ISD::ATOMIC_LOAD_SUB:
7309 return lowerATOMIC_LOAD_SUB(Op, DAG);
7310 case ISD::ATOMIC_LOAD_AND:
7311 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_AND);
7312 case ISD::ATOMIC_LOAD_OR:
7313 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_OR);
7314 case ISD::ATOMIC_LOAD_XOR:
7315 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_XOR);
7316 case ISD::ATOMIC_LOAD_NAND:
7317 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_NAND);
7318 case ISD::ATOMIC_LOAD_MIN:
7319 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MIN);
7320 case ISD::ATOMIC_LOAD_MAX:
7321 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MAX);
7322 case ISD::ATOMIC_LOAD_UMIN:
7323 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMIN);
7324 case ISD::ATOMIC_LOAD_UMAX:
7325 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMAX);
7326 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
7327 return lowerATOMIC_CMP_SWAP(Op, DAG);
7328 case ISD::STACKSAVE:
7329 return lowerSTACKSAVE(Op, DAG);
7330 case ISD::STACKRESTORE:
7331 return lowerSTACKRESTORE(Op, DAG);
7332 case ISD::PREFETCH:
7333 return lowerPREFETCH(Op, DAG);
7334 case ISD::INTRINSIC_W_CHAIN:
7335 return lowerINTRINSIC_W_CHAIN(Op, DAG);
7336 case ISD::INTRINSIC_WO_CHAIN:
7337 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
7338 case ISD::BUILD_VECTOR:
7339 return lowerBUILD_VECTOR(Op, DAG);
7340 case ISD::VECTOR_SHUFFLE:
7341 return lowerVECTOR_SHUFFLE(Op, DAG);
7342 case ISD::SCALAR_TO_VECTOR:
7343 return lowerSCALAR_TO_VECTOR(Op, DAG);
7344 case ISD::INSERT_VECTOR_ELT:
7345 return lowerINSERT_VECTOR_ELT(Op, DAG);
7346 case ISD::EXTRACT_VECTOR_ELT:
7347 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7348 case ISD::SIGN_EXTEND_VECTOR_INREG:
7349 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
7350 case ISD::ZERO_EXTEND_VECTOR_INREG:
7351 return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
7352 case ISD::SHL:
7353 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSHL_BY_SCALAR);
7354 case ISD::SRL:
7355 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRL_BY_SCALAR);
7356 case ISD::SRA:
7357 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRA_BY_SCALAR);
7358 case ISD::ADDRSPACECAST:
7359 return lowerAddrSpaceCast(Op, DAG);
7360 case ISD::ROTL:
7361 return lowerShift(Op, DAG, ByScalar: SystemZISD::VROTL_BY_SCALAR);
7362 case ISD::FSHL:
7363 return lowerFSHL(Op, DAG);
7364 case ISD::FSHR:
7365 return lowerFSHR(Op, DAG);
7366 case ISD::FP_EXTEND:
7367 case ISD::STRICT_FP_EXTEND:
7368 return lowerFP_EXTEND(Op, DAG);
7369 case ISD::FP_TO_UINT:
7370 case ISD::FP_TO_SINT:
7371 case ISD::STRICT_FP_TO_UINT:
7372 case ISD::STRICT_FP_TO_SINT:
7373 return lower_FP_TO_INT(Op, DAG);
7374 case ISD::UINT_TO_FP:
7375 case ISD::SINT_TO_FP:
7376 case ISD::STRICT_UINT_TO_FP:
7377 case ISD::STRICT_SINT_TO_FP:
7378 return lower_INT_TO_FP(Op, DAG);
7379 case ISD::LOAD:
7380 return lowerLoadF16(Op, DAG);
7381 case ISD::STORE:
7382 return lowerStoreF16(Op, DAG);
7383 case ISD::IS_FPCLASS:
7384 return lowerIS_FPCLASS(Op, DAG);
7385 case ISD::GET_ROUNDING:
7386 return lowerGET_ROUNDING(Op, DAG);
7387 case ISD::READCYCLECOUNTER:
7388 return lowerREADCYCLECOUNTER(Op, DAG);
7389 case ISD::EH_SJLJ_SETJMP:
7390 case ISD::EH_SJLJ_LONGJMP:
7391 // These operations are legal on our platform, but we cannot actually
7392 // set the operation action to Legal as common code would treat this
7393 // as equivalent to Expand. Instead, we keep the operation action to
7394 // Custom and just leave them unchanged here.
7395 return Op;
7396
7397 default:
7398 llvm_unreachable("Unexpected node to lower");
7399 }
7400}
7401
7402static SDValue expandBitCastI128ToF128(SelectionDAG &DAG, SDValue Src,
7403 const SDLoc &SL) {
7404 // If i128 is legal, just use a normal bitcast.
7405 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7406 return DAG.getBitcast(VT: MVT::f128, V: Src);
7407
7408 // Otherwise, f128 must live in FP128, so do a partwise move.
7409 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7410 &SystemZ::FP128BitRegClass);
7411
7412 SDValue Hi, Lo;
7413 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Src, DL: SL, LoVT: MVT::i64, HiVT: MVT::i64);
7414
7415 Hi = DAG.getBitcast(VT: MVT::f64, V: Hi);
7416 Lo = DAG.getBitcast(VT: MVT::f64, V: Lo);
7417
7418 SDNode *Pair = DAG.getMachineNode(
7419 Opcode: SystemZ::REG_SEQUENCE, dl: SL, VT: MVT::f128,
7420 Ops: {DAG.getTargetConstant(Val: SystemZ::FP128BitRegClassID, DL: SL, VT: MVT::i32), Lo,
7421 DAG.getTargetConstant(Val: SystemZ::subreg_l64, DL: SL, VT: MVT::i32), Hi,
7422 DAG.getTargetConstant(Val: SystemZ::subreg_h64, DL: SL, VT: MVT::i32)});
7423 return SDValue(Pair, 0);
7424}
7425
7426static SDValue expandBitCastF128ToI128(SelectionDAG &DAG, SDValue Src,
7427 const SDLoc &SL) {
7428 // If i128 is legal, just use a normal bitcast.
7429 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7430 return DAG.getBitcast(VT: MVT::i128, V: Src);
7431
7432 // Otherwise, f128 must live in FP128, so do a partwise move.
7433 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7434 &SystemZ::FP128BitRegClass);
7435
7436 SDValue LoFP =
7437 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64, DL: SL, VT: MVT::f64, Operand: Src);
7438 SDValue HiFP =
7439 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64, DL: SL, VT: MVT::f64, Operand: Src);
7440 SDValue Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: LoFP);
7441 SDValue Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: HiFP);
7442
7443 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i128, N1: Lo, N2: Hi);
7444}
7445
7446// Lower operations with invalid operand or result types.
7447void
7448SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
7449 SmallVectorImpl<SDValue> &Results,
7450 SelectionDAG &DAG) const {
7451 switch (N->getOpcode()) {
7452 case ISD::ATOMIC_LOAD: {
7453 SDLoc DL(N);
7454 SDVTList Tys = DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::Other);
7455 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
7456 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7457 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_LOAD_128,
7458 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7459
7460 SDValue Lowered = lowerGR128ToI128(DAG, In: Res);
7461 if (N->getValueType(ResNo: 0) == MVT::f128)
7462 Lowered = expandBitCastI128ToF128(DAG, Src: Lowered, SL: DL);
7463 Results.push_back(Elt: Lowered);
7464 Results.push_back(Elt: Res.getValue(R: 1));
7465 break;
7466 }
7467 case ISD::ATOMIC_STORE: {
7468 SDLoc DL(N);
7469 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
7470 SDValue Val = N->getOperand(Num: 1);
7471 if (Val.getValueType() == MVT::f128)
7472 Val = expandBitCastF128ToI128(DAG, Src: Val, SL: DL);
7473 Val = lowerI128ToGR128(DAG, In: Val);
7474
7475 SDValue Ops[] = {N->getOperand(Num: 0), Val, N->getOperand(Num: 2)};
7476 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7477 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_STORE_128,
7478 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7479 // We have to enforce sequential consistency by performing a
7480 // serialization operation after the store.
7481 if (cast<AtomicSDNode>(Val: N)->getSuccessOrdering() ==
7482 AtomicOrdering::SequentiallyConsistent)
7483 Res = SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL,
7484 VT: MVT::Other, Op1: Res), 0);
7485 Results.push_back(Elt: Res);
7486 break;
7487 }
7488 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
7489 SDLoc DL(N);
7490 SDVTList Tys = DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::i32, VT3: MVT::Other);
7491 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1),
7492 lowerI128ToGR128(DAG, In: N->getOperand(Num: 2)),
7493 lowerI128ToGR128(DAG, In: N->getOperand(Num: 3)) };
7494 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7495 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP_128,
7496 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7497 SDValue Success = emitSETCC(DAG, DL, CCReg: Res.getValue(R: 1),
7498 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
7499 Success = DAG.getZExtOrTrunc(Op: Success, DL, VT: N->getValueType(ResNo: 1));
7500 Results.push_back(Elt: lowerGR128ToI128(DAG, In: Res));
7501 Results.push_back(Elt: Success);
7502 Results.push_back(Elt: Res.getValue(R: 2));
7503 break;
7504 }
7505 case ISD::BITCAST: {
7506 if (useSoftFloat())
7507 return;
7508 SDLoc DL(N);
7509 SDValue Src = N->getOperand(Num: 0);
7510 EVT SrcVT = Src.getValueType();
7511 EVT ResVT = N->getValueType(ResNo: 0);
7512 if (ResVT == MVT::i128 && SrcVT == MVT::f128)
7513 Results.push_back(Elt: expandBitCastF128ToI128(DAG, Src, SL: DL));
7514 else if (SrcVT == MVT::i16 && ResVT == MVT::f16) {
7515 if (Subtarget.hasVector()) {
7516 SDValue In32 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Src);
7517 Results.push_back(Elt: SDValue(
7518 DAG.getMachineNode(Opcode: SystemZ::LEFR_16, dl: DL, VT: MVT::f16, Op1: In32), 0));
7519 } else {
7520 SDValue In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Src);
7521 Results.push_back(Elt: convertToF16(Op: In64, DAG));
7522 }
7523 } else if (SrcVT == MVT::f16 && ResVT == MVT::i16) {
7524 SDValue ExtractedVal =
7525 Subtarget.hasVector()
7526 ? SDValue(DAG.getMachineNode(Opcode: SystemZ::LFER_16, dl: DL, VT: MVT::i32, Op1: Src),
7527 0)
7528 : convertFromF16(Op: Src, DL, DAG);
7529 Results.push_back(Elt: DAG.getZExtOrTrunc(Op: ExtractedVal, DL, VT: ResVT));
7530 }
7531 break;
7532 }
7533 case ISD::UINT_TO_FP:
7534 case ISD::SINT_TO_FP:
7535 case ISD::STRICT_UINT_TO_FP:
7536 case ISD::STRICT_SINT_TO_FP: {
7537 if (useSoftFloat())
7538 return;
7539 bool IsStrict = N->isStrictFPOpcode();
7540 SDLoc DL(N);
7541 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7542 EVT ResVT = N->getValueType(ResNo: 0);
7543 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7544 if (ResVT == MVT::f16) {
7545 if (!IsStrict) {
7546 SDValue OpF32 = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::f32, Operand: InOp);
7547 Results.push_back(Elt: DAG.getFPExtendOrRound(Op: OpF32, DL, VT: MVT::f16));
7548 } else {
7549 SDValue OpF32 =
7550 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
7551 Ops: {Chain, InOp});
7552 SDValue F16Res;
7553 std::tie(args&: F16Res, args&: Chain) = DAG.getStrictFPExtendOrRound(
7554 Op: OpF32, Chain: OpF32.getValue(R: 1), DL, VT: MVT::f16);
7555 Results.push_back(Elt: F16Res);
7556 Results.push_back(Elt: Chain);
7557 }
7558 }
7559 break;
7560 }
7561 case ISD::FP_TO_UINT:
7562 case ISD::FP_TO_SINT:
7563 case ISD::STRICT_FP_TO_UINT:
7564 case ISD::STRICT_FP_TO_SINT: {
7565 if (useSoftFloat())
7566 return;
7567 bool IsStrict = N->isStrictFPOpcode();
7568 SDLoc DL(N);
7569 EVT ResVT = N->getValueType(ResNo: 0);
7570 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7571 EVT InVT = InOp->getValueType(ResNo: 0);
7572 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7573 if (InVT == MVT::f16) {
7574 if (!IsStrict) {
7575 SDValue InF32 = DAG.getFPExtendOrRound(Op: InOp, DL, VT: MVT::f32);
7576 Results.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: InF32));
7577 } else {
7578 SDValue InF32;
7579 std::tie(args&: InF32, args&: Chain) =
7580 DAG.getStrictFPExtendOrRound(Op: InOp, Chain, DL, VT: MVT::f32);
7581 SDValue OpF32 =
7582 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: ResVT, VT2: MVT::Other),
7583 Ops: {Chain, InF32});
7584 Results.push_back(Elt: OpF32);
7585 Results.push_back(Elt: OpF32.getValue(R: 1));
7586 }
7587 }
7588 break;
7589 }
7590 default:
7591 llvm_unreachable("Unexpected node to lower");
7592 }
7593}
7594
7595void
7596SystemZTargetLowering::ReplaceNodeResults(SDNode *N,
7597 SmallVectorImpl<SDValue> &Results,
7598 SelectionDAG &DAG) const {
7599 return LowerOperationWrapper(N, Results, DAG);
7600}
7601
7602// Return true if VT is a vector whose elements are a whole number of bytes
7603// in width. Also check for presence of vector support.
7604bool SystemZTargetLowering::canTreatAsByteVector(EVT VT) const {
7605 if (!Subtarget.hasVector())
7606 return false;
7607
7608 return VT.isVector() && VT.getScalarSizeInBits() % 8 == 0 && VT.isSimple();
7609}
7610
7611// Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT
7612// producing a result of type ResVT. Op is a possibly bitcast version
7613// of the input vector and Index is the index (based on type VecVT) that
7614// should be extracted. Return the new extraction if a simplification
7615// was possible or if Force is true.
7616SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
7617 EVT VecVT, SDValue Op,
7618 unsigned Index,
7619 DAGCombinerInfo &DCI,
7620 bool Force) const {
7621 SelectionDAG &DAG = DCI.DAG;
7622
7623 // The number of bytes being extracted.
7624 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7625
7626 for (;;) {
7627 unsigned Opcode = Op.getOpcode();
7628 if (Opcode == ISD::BITCAST)
7629 // Look through bitcasts.
7630 Op = Op.getOperand(i: 0);
7631 else if ((Opcode == ISD::VECTOR_SHUFFLE || Opcode == SystemZISD::SPLAT) &&
7632 canTreatAsByteVector(VT: Op.getValueType())) {
7633 // Get a VPERM-like permute mask and see whether the bytes covered
7634 // by the extracted element are a contiguous sequence from one
7635 // source operand.
7636 SmallVector<int, SystemZ::VectorBytes> Bytes;
7637 if (!getVPermMask(ShuffleOp: Op, Bytes))
7638 break;
7639 int First;
7640 if (!getShuffleInput(Bytes, Start: Index * BytesPerElement,
7641 BytesPerElement, Base&: First))
7642 break;
7643 if (First < 0)
7644 return DAG.getUNDEF(VT: ResVT);
7645 // Make sure the contiguous sequence starts at a multiple of the
7646 // original element size.
7647 unsigned Byte = unsigned(First) % Bytes.size();
7648 if (Byte % BytesPerElement != 0)
7649 break;
7650 // We can get the extracted value directly from an input.
7651 Index = Byte / BytesPerElement;
7652 Op = Op.getOperand(i: unsigned(First) / Bytes.size());
7653 Force = true;
7654 } else if (Opcode == ISD::BUILD_VECTOR &&
7655 canTreatAsByteVector(VT: Op.getValueType())) {
7656 // We can only optimize this case if the BUILD_VECTOR elements are
7657 // at least as wide as the extracted value.
7658 EVT OpVT = Op.getValueType();
7659 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7660 if (OpBytesPerElement < BytesPerElement)
7661 break;
7662 // Make sure that the least-significant bit of the extracted value
7663 // is the least significant bit of an input.
7664 unsigned End = (Index + 1) * BytesPerElement;
7665 if (End % OpBytesPerElement != 0)
7666 break;
7667 // We're extracting the low part of one operand of the BUILD_VECTOR.
7668 Op = Op.getOperand(i: End / OpBytesPerElement - 1);
7669 EVT ResIntVT = MVT::getIntegerVT(BitWidth: ResVT.getSizeInBits());
7670 if (!isTypeLegal(VT: ResIntVT))
7671 break;
7672 if (!Op.getValueType().isInteger()) {
7673 EVT OpIntVT = MVT::getIntegerVT(BitWidth: Op.getValueSizeInBits());
7674 if (!isTypeLegal(VT: OpIntVT))
7675 break;
7676 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: OpIntVT, Operand: Op);
7677 DCI.AddToWorklist(N: Op.getNode());
7678 }
7679 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResIntVT, Operand: Op);
7680 if (ResIntVT != ResVT) {
7681 DCI.AddToWorklist(N: Op.getNode());
7682 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ResVT, Operand: Op);
7683 }
7684 return Op;
7685 } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
7686 Opcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
7687 Opcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
7688 canTreatAsByteVector(VT: Op.getValueType()) &&
7689 canTreatAsByteVector(VT: Op.getOperand(i: 0).getValueType())) {
7690 // Make sure that only the unextended bits are significant.
7691 EVT ExtVT = Op.getValueType();
7692 EVT OpVT = Op.getOperand(i: 0).getValueType();
7693 unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize();
7694 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7695 unsigned Byte = Index * BytesPerElement;
7696 unsigned SubByte = Byte % ExtBytesPerElement;
7697 unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
7698 if (SubByte < MinSubByte ||
7699 SubByte + BytesPerElement > ExtBytesPerElement)
7700 break;
7701 // Get the byte offset of the unextended element
7702 Byte = Byte / ExtBytesPerElement * OpBytesPerElement;
7703 // ...then add the byte offset relative to that element.
7704 Byte += SubByte - MinSubByte;
7705 if (Byte % BytesPerElement != 0)
7706 break;
7707 Op = Op.getOperand(i: 0);
7708 Index = Byte / BytesPerElement;
7709 Force = true;
7710 } else
7711 break;
7712 }
7713 if (Force) {
7714 if (Op.getValueType() != VecVT) {
7715 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VecVT, Operand: Op);
7716 DCI.AddToWorklist(N: Op.getNode());
7717 }
7718 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ResVT, N1: Op,
7719 N2: DAG.getConstant(Val: Index, DL, VT: MVT::i32));
7720 }
7721 return SDValue();
7722}
7723
7724// Optimize vector operations in scalar value Op on the basis that Op
7725// is truncated to TruncVT.
7726SDValue SystemZTargetLowering::combineTruncateExtract(
7727 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
7728 // If we have (trunc (extract_vector_elt X, Y)), try to turn it into
7729 // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements
7730 // of type TruncVT.
7731 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7732 TruncVT.getSizeInBits() % 8 == 0) {
7733 SDValue Vec = Op.getOperand(i: 0);
7734 EVT VecVT = Vec.getValueType();
7735 if (canTreatAsByteVector(VT: VecVT)) {
7736 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
7737 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7738 unsigned TruncBytes = TruncVT.getStoreSize();
7739 if (BytesPerElement % TruncBytes == 0) {
7740 // Calculate the value of Y' in the above description. We are
7741 // splitting the original elements into Scale equal-sized pieces
7742 // and for truncation purposes want the last (least-significant)
7743 // of these pieces for IndexN. This is easiest to do by calculating
7744 // the start index of the following element and then subtracting 1.
7745 unsigned Scale = BytesPerElement / TruncBytes;
7746 unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
7747
7748 // Defer the creation of the bitcast from X to combineExtract,
7749 // which might be able to optimize the extraction.
7750 VecVT = EVT::getVectorVT(Context&: *DCI.DAG.getContext(),
7751 VT: MVT::getIntegerVT(BitWidth: TruncBytes * 8),
7752 NumElements: VecVT.getStoreSize() / TruncBytes);
7753 EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
7754 return combineExtract(DL, ResVT, VecVT, Op: Vec, Index: NewIndex, DCI, Force: true);
7755 }
7756 }
7757 }
7758 }
7759 return SDValue();
7760}
7761
7762SDValue SystemZTargetLowering::combineZERO_EXTEND(
7763 SDNode *N, DAGCombinerInfo &DCI) const {
7764 // Convert (zext (select_ccmask C1, C2)) into (select_ccmask C1', C2')
7765 SelectionDAG &DAG = DCI.DAG;
7766 SDValue N0 = N->getOperand(Num: 0);
7767 EVT VT = N->getValueType(ResNo: 0);
7768 if (N0.getOpcode() == SystemZISD::SELECT_CCMASK) {
7769 auto *TrueOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0));
7770 auto *FalseOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7771 if (TrueOp && FalseOp) {
7772 SDLoc DL(N0);
7773 SDValue Ops[] = { DAG.getConstant(Val: TrueOp->getZExtValue(), DL, VT),
7774 DAG.getConstant(Val: FalseOp->getZExtValue(), DL, VT),
7775 N0.getOperand(i: 2), N0.getOperand(i: 3), N0.getOperand(i: 4) };
7776 SDValue NewSelect = DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT, Ops);
7777 // If N0 has multiple uses, change other uses as well.
7778 if (!N0.hasOneUse()) {
7779 SDValue TruncSelect =
7780 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N0.getValueType(), Operand: NewSelect);
7781 DCI.CombineTo(N: N0.getNode(), Res: TruncSelect);
7782 }
7783 return NewSelect;
7784 }
7785 }
7786 // Convert (zext (xor (trunc X), C)) into (xor (trunc X), C') if the size
7787 // of the result is smaller than the size of X and all the truncated bits
7788 // of X are already zero.
7789 if (N0.getOpcode() == ISD::XOR &&
7790 N0.hasOneUse() && N0.getOperand(i: 0).hasOneUse() &&
7791 N0.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
7792 N0.getOperand(i: 1).getOpcode() == ISD::Constant) {
7793 SDValue X = N0.getOperand(i: 0).getOperand(i: 0);
7794 if (VT.isScalarInteger() && VT.getSizeInBits() < X.getValueSizeInBits()) {
7795 KnownBits Known = DAG.computeKnownBits(Op: X);
7796 APInt TruncatedBits = APInt::getBitsSet(numBits: X.getValueSizeInBits(),
7797 loBit: N0.getValueSizeInBits(),
7798 hiBit: VT.getSizeInBits());
7799 if (TruncatedBits.isSubsetOf(RHS: Known.Zero)) {
7800 X = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(X), VT, Operand: X);
7801 APInt Mask = N0.getConstantOperandAPInt(i: 1).zext(width: VT.getSizeInBits());
7802 return DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N0), VT,
7803 N1: X, N2: DAG.getConstant(Val: Mask, DL: SDLoc(N0), VT));
7804 }
7805 }
7806 }
7807 // Recognize patterns for VECTOR SUBTRACT COMPUTE BORROW INDICATION
7808 // and VECTOR ADD COMPUTE CARRY for i128:
7809 // (zext (setcc_uge X Y)) --> (VSCBI X Y)
7810 // (zext (setcc_ule Y X)) --> (VSCBI X Y)
7811 // (zext (setcc_ult (add X Y) X/Y) -> (VACC X Y)
7812 // (zext (setcc_ugt X/Y (add X Y)) -> (VACC X Y)
7813 // For vector types, these patterns are recognized in the .td file.
7814 if (N0.getOpcode() == ISD::SETCC && isTypeLegal(VT) && VT == MVT::i128 &&
7815 N0.getOperand(i: 0).getValueType() == VT) {
7816 SDValue Op0 = N0.getOperand(i: 0);
7817 SDValue Op1 = N0.getOperand(i: 1);
7818 const ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
7819 switch (CC) {
7820 case ISD::SETULE:
7821 std::swap(a&: Op0, b&: Op1);
7822 [[fallthrough]];
7823 case ISD::SETUGE:
7824 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(N0), VT, N1: Op0, N2: Op1);
7825 case ISD::SETUGT:
7826 std::swap(a&: Op0, b&: Op1);
7827 [[fallthrough]];
7828 case ISD::SETULT:
7829 if (Op0->hasOneUse() && Op0->getOpcode() == ISD::ADD &&
7830 (Op0->getOperand(Num: 0) == Op1 || Op0->getOperand(Num: 1) == Op1))
7831 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(N0), VT, N1: Op0->getOperand(Num: 0),
7832 N2: Op0->getOperand(Num: 1));
7833 break;
7834 default:
7835 break;
7836 }
7837 }
7838
7839 return SDValue();
7840}
7841
7842SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
7843 SDNode *N, DAGCombinerInfo &DCI) const {
7844 // Convert (sext_in_reg (setcc LHS, RHS, COND), i1)
7845 // and (sext_in_reg (any_extend (setcc LHS, RHS, COND)), i1)
7846 // into (select_cc LHS, RHS, -1, 0, COND)
7847 SelectionDAG &DAG = DCI.DAG;
7848 SDValue N0 = N->getOperand(Num: 0);
7849 EVT VT = N->getValueType(ResNo: 0);
7850 EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
7851 if (N0.hasOneUse() && N0.getOpcode() == ISD::ANY_EXTEND)
7852 N0 = N0.getOperand(i: 0);
7853 if (EVT == MVT::i1 && N0.hasOneUse() && N0.getOpcode() == ISD::SETCC) {
7854 SDLoc DL(N0);
7855 SDValue Ops[] = { N0.getOperand(i: 0), N0.getOperand(i: 1),
7856 DAG.getAllOnesConstant(DL, VT),
7857 DAG.getConstant(Val: 0, DL, VT), N0.getOperand(i: 2) };
7858 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT, Ops);
7859 }
7860 return SDValue();
7861}
7862
7863SDValue SystemZTargetLowering::combineSIGN_EXTEND(
7864 SDNode *N, DAGCombinerInfo &DCI) const {
7865 // Convert (sext (ashr (shl X, C1), C2)) to
7866 // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as
7867 // cheap as narrower ones.
7868 SelectionDAG &DAG = DCI.DAG;
7869 SDValue N0 = N->getOperand(Num: 0);
7870 EVT VT = N->getValueType(ResNo: 0);
7871 if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) {
7872 auto *SraAmt = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7873 SDValue Inner = N0.getOperand(i: 0);
7874 if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) {
7875 if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Val: Inner.getOperand(i: 1))) {
7876 unsigned Extra = (VT.getSizeInBits() - N0.getValueSizeInBits());
7877 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
7878 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
7879 EVT ShiftVT = N0.getOperand(i: 1).getValueType();
7880 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(Inner), VT,
7881 Operand: Inner.getOperand(i: 0));
7882 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(Inner), VT, N1: Ext,
7883 N2: DAG.getConstant(Val: NewShlAmt, DL: SDLoc(Inner),
7884 VT: ShiftVT));
7885 return DAG.getNode(Opcode: ISD::SRA, DL: SDLoc(N0), VT, N1: Shl,
7886 N2: DAG.getConstant(Val: NewSraAmt, DL: SDLoc(N0), VT: ShiftVT));
7887 }
7888 }
7889 }
7890
7891 return SDValue();
7892}
7893
7894SDValue SystemZTargetLowering::combineMERGE(
7895 SDNode *N, DAGCombinerInfo &DCI) const {
7896 SelectionDAG &DAG = DCI.DAG;
7897 unsigned Opcode = N->getOpcode();
7898 SDValue Op0 = N->getOperand(Num: 0);
7899 SDValue Op1 = N->getOperand(Num: 1);
7900 if (Op0.getOpcode() == ISD::BITCAST)
7901 Op0 = Op0.getOperand(i: 0);
7902 if (ISD::isBuildVectorAllZeros(N: Op0.getNode())) {
7903 // (z_merge_* 0, 0) -> 0. This is mostly useful for using VLLEZF
7904 // for v4f32.
7905 if (Op1 == N->getOperand(Num: 0))
7906 return Op1;
7907 // (z_merge_? 0, X) -> (z_unpackl_? 0, X).
7908 EVT VT = Op1.getValueType();
7909 unsigned ElemBytes = VT.getVectorElementType().getStoreSize();
7910 if (ElemBytes <= 4) {
7911 Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
7912 SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
7913 EVT InVT = VT.changeVectorElementTypeToInteger();
7914 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ElemBytes * 16),
7915 NumElements: SystemZ::VectorBytes / ElemBytes / 2);
7916 if (VT != InVT) {
7917 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: InVT, Operand: Op1);
7918 DCI.AddToWorklist(N: Op1.getNode());
7919 }
7920 SDValue Op = DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: Op1);
7921 DCI.AddToWorklist(N: Op.getNode());
7922 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT, Operand: Op);
7923 }
7924 }
7925 return SDValue();
7926}
7927
7928static bool isI128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
7929 SDNode *&HiPart) {
7930 LoPart = HiPart = nullptr;
7931
7932 // Scan through all users.
7933 for (SDUse &Use : LD->uses()) {
7934 // Skip the uses of the chain.
7935 if (Use.getResNo() != 0)
7936 continue;
7937
7938 // Verify every user is a TRUNCATE to i64 of the low or high half.
7939 SDNode *User = Use.getUser();
7940 bool IsLoPart = true;
7941 if (User->getOpcode() == ISD::SRL &&
7942 User->getOperand(Num: 1).getOpcode() == ISD::Constant &&
7943 User->getConstantOperandVal(Num: 1) == 64 && User->hasOneUse()) {
7944 User = *User->user_begin();
7945 IsLoPart = false;
7946 }
7947 if (User->getOpcode() != ISD::TRUNCATE || User->getValueType(ResNo: 0) != MVT::i64)
7948 return false;
7949
7950 if (IsLoPart) {
7951 if (LoPart)
7952 return false;
7953 LoPart = User;
7954 } else {
7955 if (HiPart)
7956 return false;
7957 HiPart = User;
7958 }
7959 }
7960 return true;
7961}
7962
7963static bool isF128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
7964 SDNode *&HiPart) {
7965 LoPart = HiPart = nullptr;
7966
7967 // Scan through all users.
7968 for (SDUse &Use : LD->uses()) {
7969 // Skip the uses of the chain.
7970 if (Use.getResNo() != 0)
7971 continue;
7972
7973 // Verify every user is an EXTRACT_SUBREG of the low or high half.
7974 SDNode *User = Use.getUser();
7975 if (!User->hasOneUse() || !User->isMachineOpcode() ||
7976 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
7977 return false;
7978
7979 switch (User->getConstantOperandVal(Num: 1)) {
7980 case SystemZ::subreg_l64:
7981 if (LoPart)
7982 return false;
7983 LoPart = User;
7984 break;
7985 case SystemZ::subreg_h64:
7986 if (HiPart)
7987 return false;
7988 HiPart = User;
7989 break;
7990 default:
7991 return false;
7992 }
7993 }
7994 return true;
7995}
7996
7997SDValue SystemZTargetLowering::combineLOAD(
7998 SDNode *N, DAGCombinerInfo &DCI) const {
7999 SelectionDAG &DAG = DCI.DAG;
8000 EVT LdVT = N->getValueType(ResNo: 0);
8001 if (auto *LN = dyn_cast<LoadSDNode>(Val: N)) {
8002 if (LN->getAddressSpace() == SYSTEMZAS::PTR32) {
8003 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8004 MVT LoadNodeVT = LN->getBasePtr().getSimpleValueType();
8005 if (PtrVT != LoadNodeVT) {
8006 SDLoc DL(LN);
8007 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(
8008 dl: DL, VT: PtrVT, Ptr: LN->getBasePtr(), SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
8009 return DAG.getExtLoad(ExtType: LN->getExtensionType(), dl: DL, VT: LN->getValueType(ResNo: 0),
8010 Chain: LN->getChain(), Ptr: AddrSpaceCast, MemVT: LN->getMemoryVT(),
8011 MMO: LN->getMemOperand());
8012 }
8013 }
8014 }
8015 SDLoc DL(N);
8016
8017 // Replace a 128-bit load that is used solely to move its value into GPRs
8018 // by separate loads of both halves.
8019 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
8020 if (LD->isSimple() && ISD::isNormalLoad(N: LD)) {
8021 SDNode *LoPart, *HiPart;
8022 if ((LdVT == MVT::i128 && isI128MovedToParts(LD, LoPart, HiPart)) ||
8023 (LdVT == MVT::f128 && isF128MovedToParts(LD, LoPart, HiPart))) {
8024 // Rewrite each extraction as an independent load.
8025 SmallVector<SDValue, 2> ArgChains;
8026 if (HiPart) {
8027 SDValue EltLoad = DAG.getLoad(
8028 VT: HiPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(), Ptr: LD->getBasePtr(),
8029 PtrInfo: LD->getPointerInfo(), Alignment: LD->getBaseAlign(),
8030 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
8031
8032 DCI.CombineTo(N: HiPart, Res: EltLoad, AddTo: true);
8033 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
8034 }
8035 if (LoPart) {
8036 SDValue EltLoad = DAG.getLoad(
8037 VT: LoPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(),
8038 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: LD->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
8039 PtrInfo: LD->getPointerInfo().getWithOffset(O: 8), Alignment: LD->getBaseAlign(),
8040 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
8041
8042 DCI.CombineTo(N: LoPart, Res: EltLoad, AddTo: true);
8043 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
8044 }
8045
8046 // Collect all chains via TokenFactor.
8047 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ArgChains);
8048 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: Chain);
8049 DCI.AddToWorklist(N: Chain.getNode());
8050 return SDValue(N, 0);
8051 }
8052 }
8053
8054 if (LdVT.isVector() || LdVT.isInteger())
8055 return SDValue();
8056 // Transform a scalar load that is REPLICATEd as well as having other
8057 // use(s) to the form where the other use(s) use the first element of the
8058 // REPLICATE instead of the load. Otherwise instruction selection will not
8059 // produce a VLREP. Avoid extracting to a GPR, so only do this for floating
8060 // point loads.
8061
8062 SDValue Replicate;
8063 SmallVector<SDNode*, 8> OtherUses;
8064 for (SDUse &Use : N->uses()) {
8065 if (Use.getUser()->getOpcode() == SystemZISD::REPLICATE) {
8066 if (Replicate)
8067 return SDValue(); // Should never happen
8068 Replicate = SDValue(Use.getUser(), 0);
8069 } else if (Use.getResNo() == 0)
8070 OtherUses.push_back(Elt: Use.getUser());
8071 }
8072 if (!Replicate || OtherUses.empty())
8073 return SDValue();
8074
8075 SDValue Extract0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: LdVT,
8076 N1: Replicate, N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
8077 // Update uses of the loaded Value while preserving old chains.
8078 for (SDNode *U : OtherUses) {
8079 SmallVector<SDValue, 8> Ops;
8080 for (SDValue Op : U->ops())
8081 Ops.push_back(Elt: (Op.getNode() == N && Op.getResNo() == 0) ? Extract0 : Op);
8082 DAG.UpdateNodeOperands(N: U, Ops);
8083 }
8084 return SDValue(N, 0);
8085}
8086
8087bool SystemZTargetLowering::canLoadStoreByteSwapped(EVT VT) const {
8088 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64)
8089 return true;
8090 if (Subtarget.hasVectorEnhancements2())
8091 if (VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v2i64 || VT == MVT::i128)
8092 return true;
8093 return false;
8094}
8095
8096static bool isVectorElementSwap(ArrayRef<int> M, EVT VT) {
8097 if (!VT.isVector() || !VT.isSimple() ||
8098 VT.getSizeInBits() != 128 ||
8099 VT.getScalarSizeInBits() % 8 != 0)
8100 return false;
8101
8102 unsigned NumElts = VT.getVectorNumElements();
8103 for (unsigned i = 0; i < NumElts; ++i) {
8104 if (M[i] < 0) continue; // ignore UNDEF indices
8105 if ((unsigned) M[i] != NumElts - 1 - i)
8106 return false;
8107 }
8108
8109 return true;
8110}
8111
8112static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG) {
8113 for (auto *U : StoredVal->users()) {
8114 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: U)) {
8115 EVT CurrMemVT = ST->getMemoryVT().getScalarType();
8116 if (CurrMemVT.isRound() && CurrMemVT.getStoreSize() <= 16)
8117 continue;
8118 } else if (isa<BuildVectorSDNode>(Val: U)) {
8119 SDValue BuildVector = SDValue(U, 0);
8120 if (DAG.isSplatValue(V: BuildVector, AllowUndefs: true/*AllowUndefs*/) &&
8121 isOnlyUsedByStores(StoredVal: BuildVector, DAG))
8122 continue;
8123 }
8124 return false;
8125 }
8126 return true;
8127}
8128
8129static bool isI128MovedFromParts(SDValue Val, SDValue &LoPart,
8130 SDValue &HiPart) {
8131 if (Val.getOpcode() != ISD::OR || !Val.getNode()->hasOneUse())
8132 return false;
8133
8134 SDValue Op0 = Val.getOperand(i: 0);
8135 SDValue Op1 = Val.getOperand(i: 1);
8136
8137 if (Op0.getOpcode() == ISD::SHL)
8138 std::swap(a&: Op0, b&: Op1);
8139 if (Op1.getOpcode() != ISD::SHL || !Op1.getNode()->hasOneUse() ||
8140 Op1.getOperand(i: 1).getOpcode() != ISD::Constant ||
8141 Op1.getConstantOperandVal(i: 1) != 64)
8142 return false;
8143 Op1 = Op1.getOperand(i: 0);
8144
8145 if (Op0.getOpcode() != ISD::ZERO_EXTEND || !Op0.getNode()->hasOneUse() ||
8146 Op0.getOperand(i: 0).getValueType() != MVT::i64)
8147 return false;
8148 if (Op1.getOpcode() != ISD::ANY_EXTEND || !Op1.getNode()->hasOneUse() ||
8149 Op1.getOperand(i: 0).getValueType() != MVT::i64)
8150 return false;
8151
8152 LoPart = Op0.getOperand(i: 0);
8153 HiPart = Op1.getOperand(i: 0);
8154 return true;
8155}
8156
8157static bool isF128MovedFromParts(SDValue Val, SDValue &LoPart,
8158 SDValue &HiPart) {
8159 if (!Val.getNode()->hasOneUse() || !Val.isMachineOpcode() ||
8160 Val.getMachineOpcode() != TargetOpcode::REG_SEQUENCE)
8161 return false;
8162
8163 if (Val->getNumOperands() != 5 ||
8164 Val->getOperand(Num: 0)->getAsZExtVal() != SystemZ::FP128BitRegClassID ||
8165 Val->getOperand(Num: 2)->getAsZExtVal() != SystemZ::subreg_l64 ||
8166 Val->getOperand(Num: 4)->getAsZExtVal() != SystemZ::subreg_h64)
8167 return false;
8168
8169 LoPart = Val->getOperand(Num: 1);
8170 HiPart = Val->getOperand(Num: 3);
8171 return true;
8172}
8173
8174SDValue SystemZTargetLowering::combineSTORE(
8175 SDNode *N, DAGCombinerInfo &DCI) const {
8176 SelectionDAG &DAG = DCI.DAG;
8177 auto *SN = cast<StoreSDNode>(Val: N);
8178 auto &Op1 = N->getOperand(Num: 1);
8179 EVT MemVT = SN->getMemoryVT();
8180
8181 if (SN->getAddressSpace() == SYSTEMZAS::PTR32) {
8182 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8183 MVT StoreNodeVT = SN->getBasePtr().getSimpleValueType();
8184 if (PtrVT != StoreNodeVT) {
8185 SDLoc DL(SN);
8186 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(dl: DL, VT: PtrVT, Ptr: SN->getBasePtr(),
8187 SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
8188 return DAG.getStore(Chain: SN->getChain(), dl: DL, Val: SN->getValue(), Ptr: AddrSpaceCast,
8189 PtrInfo: SN->getPointerInfo(), Alignment: SN->getBaseAlign(),
8190 MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
8191 }
8192 }
8193
8194 // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better
8195 // for the extraction to be done on a vMiN value, so that we can use VSTE.
8196 // If X has wider elements then convert it to:
8197 // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z).
8198 if (MemVT.isInteger() && SN->isTruncatingStore()) {
8199 if (SDValue Value =
8200 combineTruncateExtract(DL: SDLoc(N), TruncVT: MemVT, Op: SN->getValue(), DCI)) {
8201 DCI.AddToWorklist(N: Value.getNode());
8202
8203 // Rewrite the store with the new form of stored value.
8204 return DAG.getTruncStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: Value,
8205 Ptr: SN->getBasePtr(), SVT: SN->getMemoryVT(),
8206 MMO: SN->getMemOperand());
8207 }
8208 }
8209
8210 // combine STORE (LOAD_STACK_GUARD) into MOV_STACKGUARD_DAG
8211 if (Op1->isMachineOpcode() &&
8212 (Op1->getMachineOpcode() == SystemZ::LOAD_STACK_GUARD)) {
8213 // Obtain the frame index the store was targeting.
8214 int FI = cast<FrameIndexSDNode>(Val: SN->getOperand(Num: 2))->getIndex();
8215 // Prepare operands of the MOV_STACKGUARD ISD Node - Chain and FrameIndex.
8216 SDValue Ops[] = {SN->getChain(), DAG.getTargetFrameIndex(FI, VT: MVT::i64)};
8217 return DAG.getNode(Opcode: SystemZISD::MOV_STACKGUARD, DL: SDLoc(SN), VT: MVT::Other, Ops);
8218 }
8219
8220 // Combine STORE (BSWAP) into STRVH/STRV/STRVG/VSTBR
8221 if (!SN->isTruncatingStore() &&
8222 Op1.getOpcode() == ISD::BSWAP &&
8223 Op1.getNode()->hasOneUse() &&
8224 canLoadStoreByteSwapped(VT: Op1.getValueType())) {
8225
8226 SDValue BSwapOp = Op1.getOperand(i: 0);
8227
8228 if (BSwapOp.getValueType() == MVT::i16)
8229 BSwapOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: MVT::i32, Operand: BSwapOp);
8230
8231 SDValue Ops[] = {
8232 N->getOperand(Num: 0), BSwapOp, N->getOperand(Num: 2)
8233 };
8234
8235 return
8236 DAG.getMemIntrinsicNode(Opcode: SystemZISD::STRV, dl: SDLoc(N), VTList: DAG.getVTList(VT: MVT::Other),
8237 Ops, MemVT, MMO: SN->getMemOperand());
8238 }
8239 // Combine STORE (element-swap) into VSTER
8240 if (!SN->isTruncatingStore() &&
8241 Op1.getOpcode() == ISD::VECTOR_SHUFFLE &&
8242 Op1.getNode()->hasOneUse() &&
8243 Subtarget.hasVectorEnhancements2()) {
8244 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op1.getNode());
8245 ArrayRef<int> ShuffleMask = SVN->getMask();
8246 if (isVectorElementSwap(M: ShuffleMask, VT: Op1.getValueType())) {
8247 SDValue Ops[] = {
8248 N->getOperand(Num: 0), Op1.getOperand(i: 0), N->getOperand(Num: 2)
8249 };
8250
8251 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::VSTER, dl: SDLoc(N),
8252 VTList: DAG.getVTList(VT: MVT::Other),
8253 Ops, MemVT, MMO: SN->getMemOperand());
8254 }
8255 }
8256
8257 // Combine STORE (READCYCLECOUNTER) into STCKF.
8258 if (!SN->isTruncatingStore() &&
8259 Op1.getOpcode() == ISD::READCYCLECOUNTER &&
8260 Op1.hasOneUse() &&
8261 N->getOperand(Num: 0).reachesChainWithoutSideEffects(Dest: SDValue(Op1.getNode(), 1))) {
8262 SDValue Ops[] = { Op1.getOperand(i: 0), N->getOperand(Num: 2) };
8263 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::STCKF, dl: SDLoc(N),
8264 VTList: DAG.getVTList(VT: MVT::Other),
8265 Ops, MemVT, MMO: SN->getMemOperand());
8266 }
8267
8268 // Transform a store of a 128-bit value moved from parts into two stores.
8269 if (SN->isSimple() && ISD::isNormalStore(N: SN)) {
8270 SDValue LoPart, HiPart;
8271 if ((MemVT == MVT::i128 && isI128MovedFromParts(Val: Op1, LoPart, HiPart)) ||
8272 (MemVT == MVT::f128 && isF128MovedFromParts(Val: Op1, LoPart, HiPart))) {
8273 SDLoc DL(SN);
8274 SDValue Chain0 = DAG.getStore(
8275 Chain: SN->getChain(), dl: DL, Val: HiPart, Ptr: SN->getBasePtr(), PtrInfo: SN->getPointerInfo(),
8276 Alignment: SN->getBaseAlign(), MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
8277 SDValue Chain1 = DAG.getStore(
8278 Chain: SN->getChain(), dl: DL, Val: LoPart,
8279 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: SN->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
8280 PtrInfo: SN->getPointerInfo().getWithOffset(O: 8), Alignment: SN->getBaseAlign(),
8281 MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
8282
8283 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Chain0, N2: Chain1);
8284 }
8285 }
8286
8287 // Replicate a reg or immediate with VREP instead of scalar multiply or
8288 // immediate load. It seems best to do this during the first DAGCombine as
8289 // it is straight-forward to handle the zero-extend node in the initial
8290 // DAG, and also not worry about the keeping the new MemVT legal (e.g. when
8291 // extracting an i16 element from a v16i8 vector).
8292 if (Subtarget.hasVector() && DCI.Level == BeforeLegalizeTypes &&
8293 isOnlyUsedByStores(StoredVal: Op1, DAG)) {
8294 SDValue Word = SDValue();
8295 EVT WordVT;
8296
8297 // Find a replicated immediate and return it if found in Word and its
8298 // type in WordVT.
8299 auto FindReplicatedImm = [&](ConstantSDNode *C, unsigned TotBytes) {
8300 // Some constants are better handled with a scalar store.
8301 if (C->getAPIntValue().getBitWidth() > 64 || C->isAllOnes() ||
8302 isInt<16>(x: C->getSExtValue()) || MemVT.getStoreSize() <= 2)
8303 return;
8304
8305 APInt Val = C->getAPIntValue();
8306 // Truncate Val in case of a truncating store.
8307 if (!llvm::isUIntN(N: TotBytes * 8, x: Val.getZExtValue())) {
8308 assert(SN->isTruncatingStore() &&
8309 "Non-truncating store and immediate value does not fit?");
8310 Val = Val.trunc(width: TotBytes * 8);
8311 }
8312
8313 SystemZVectorConstantInfo VCI(APInt(TotBytes * 8, Val.getZExtValue()));
8314 if (VCI.isVectorConstantLegal(Subtarget) &&
8315 VCI.Opcode == SystemZISD::REPLICATE) {
8316 Word = DAG.getConstant(Val: VCI.OpVals[0], DL: SDLoc(SN), VT: MVT::i32);
8317 WordVT = VCI.VecVT.getScalarType();
8318 }
8319 };
8320
8321 // Find a replicated register and return it if found in Word and its type
8322 // in WordVT.
8323 auto FindReplicatedReg = [&](SDValue MulOp) {
8324 EVT MulVT = MulOp.getValueType();
8325 if (MulOp->getOpcode() == ISD::MUL &&
8326 (MulVT == MVT::i16 || MulVT == MVT::i32 || MulVT == MVT::i64)) {
8327 // Find a zero extended value and its type.
8328 SDValue LHS = MulOp->getOperand(Num: 0);
8329 if (LHS->getOpcode() == ISD::ZERO_EXTEND)
8330 WordVT = LHS->getOperand(Num: 0).getValueType();
8331 else if (LHS->getOpcode() == ISD::AssertZext)
8332 WordVT = cast<VTSDNode>(Val: LHS->getOperand(Num: 1))->getVT();
8333 else
8334 return;
8335 // Find a replicating constant, e.g. 0x00010001.
8336 if (auto *C = dyn_cast<ConstantSDNode>(Val: MulOp->getOperand(Num: 1))) {
8337 SystemZVectorConstantInfo VCI(
8338 APInt(MulVT.getSizeInBits(), C->getZExtValue()));
8339 if (VCI.isVectorConstantLegal(Subtarget) &&
8340 VCI.Opcode == SystemZISD::REPLICATE && VCI.OpVals[0] == 1 &&
8341 WordVT == VCI.VecVT.getScalarType())
8342 Word = DAG.getZExtOrTrunc(Op: LHS->getOperand(Num: 0), DL: SDLoc(SN), VT: WordVT);
8343 }
8344 }
8345 };
8346
8347 if (isa<BuildVectorSDNode>(Val: Op1) &&
8348 DAG.isSplatValue(V: Op1, AllowUndefs: true/*AllowUndefs*/)) {
8349 SDValue SplatVal = Op1->getOperand(Num: 0);
8350 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatVal))
8351 FindReplicatedImm(C, SplatVal.getValueType().getStoreSize());
8352 else
8353 FindReplicatedReg(SplatVal);
8354 } else {
8355 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op1))
8356 FindReplicatedImm(C, MemVT.getStoreSize());
8357 else
8358 FindReplicatedReg(Op1);
8359 }
8360
8361 if (Word != SDValue()) {
8362 assert(MemVT.getSizeInBits() % WordVT.getSizeInBits() == 0 &&
8363 "Bad type handling");
8364 unsigned NumElts = MemVT.getSizeInBits() / WordVT.getSizeInBits();
8365 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WordVT, NumElements: NumElts);
8366 SDValue SplatVal = DAG.getSplatVector(VT: SplatVT, DL: SDLoc(SN), Op: Word);
8367 return DAG.getStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: SplatVal,
8368 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
8369 }
8370 }
8371
8372 return SDValue();
8373}
8374
8375SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
8376 SDNode *N, DAGCombinerInfo &DCI) const {
8377 SelectionDAG &DAG = DCI.DAG;
8378 // Combine element-swap (LOAD) into VLER
8379 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8380 N->getOperand(Num: 0).hasOneUse() &&
8381 Subtarget.hasVectorEnhancements2()) {
8382 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
8383 ArrayRef<int> ShuffleMask = SVN->getMask();
8384 if (isVectorElementSwap(M: ShuffleMask, VT: N->getValueType(ResNo: 0))) {
8385 SDValue Load = N->getOperand(Num: 0);
8386 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8387
8388 // Create the element-swapping load.
8389 SDValue Ops[] = {
8390 LD->getChain(), // Chain
8391 LD->getBasePtr() // Ptr
8392 };
8393 SDValue ESLoad =
8394 DAG.getMemIntrinsicNode(Opcode: SystemZISD::VLER, dl: SDLoc(N),
8395 VTList: DAG.getVTList(VT1: LD->getValueType(ResNo: 0), VT2: MVT::Other),
8396 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8397
8398 // First, combine the VECTOR_SHUFFLE away. This makes the value produced
8399 // by the load dead.
8400 DCI.CombineTo(N, Res: ESLoad);
8401
8402 // Next, combine the load away, we give it a bogus result value but a real
8403 // chain result. The result value is dead because the shuffle is dead.
8404 DCI.CombineTo(N: Load.getNode(), Res0: ESLoad, Res1: ESLoad.getValue(R: 1));
8405
8406 // Return N so it doesn't get rechecked!
8407 return SDValue(N, 0);
8408 }
8409 }
8410
8411 return SDValue();
8412}
8413
8414SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
8415 SDNode *N, DAGCombinerInfo &DCI) const {
8416 SelectionDAG &DAG = DCI.DAG;
8417
8418 if (!Subtarget.hasVector())
8419 return SDValue();
8420
8421 // Look through bitcasts that retain the number of vector elements.
8422 SDValue Op = N->getOperand(Num: 0);
8423 if (Op.getOpcode() == ISD::BITCAST &&
8424 Op.getValueType().isVector() &&
8425 Op.getOperand(i: 0).getValueType().isVector() &&
8426 Op.getValueType().getVectorNumElements() ==
8427 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8428 Op = Op.getOperand(i: 0);
8429
8430 // Pull BSWAP out of a vector extraction.
8431 if (Op.getOpcode() == ISD::BSWAP && Op.hasOneUse()) {
8432 EVT VecVT = Op.getValueType();
8433 EVT EltVT = VecVT.getVectorElementType();
8434 Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), VT: EltVT,
8435 N1: Op.getOperand(i: 0), N2: N->getOperand(Num: 1));
8436 DCI.AddToWorklist(N: Op.getNode());
8437 Op = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Op);
8438 if (EltVT != N->getValueType(ResNo: 0)) {
8439 DCI.AddToWorklist(N: Op.getNode());
8440 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
8441 }
8442 return Op;
8443 }
8444
8445 // Try to simplify a vector extraction.
8446 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
8447 SDValue Op0 = N->getOperand(Num: 0);
8448 EVT VecVT = Op0.getValueType();
8449 if (canTreatAsByteVector(VT: VecVT))
8450 return combineExtract(DL: SDLoc(N), ResVT: N->getValueType(ResNo: 0), VecVT, Op: Op0,
8451 Index: IndexN->getZExtValue(), DCI, Force: false);
8452 }
8453 return SDValue();
8454}
8455
8456SDValue SystemZTargetLowering::combineJOIN_DWORDS(
8457 SDNode *N, DAGCombinerInfo &DCI) const {
8458 SelectionDAG &DAG = DCI.DAG;
8459 // (join_dwords X, X) == (replicate X)
8460 if (N->getOperand(Num: 0) == N->getOperand(Num: 1))
8461 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8462 Operand: N->getOperand(Num: 0));
8463 return SDValue();
8464}
8465
8466static SDValue MergeInputChains(SDNode *N1, SDNode *N2) {
8467 SDValue Chain1 = N1->getOperand(Num: 0);
8468 SDValue Chain2 = N2->getOperand(Num: 0);
8469
8470 // Trivial case: both nodes take the same chain.
8471 if (Chain1 == Chain2)
8472 return Chain1;
8473
8474 // FIXME - we could handle more complex cases via TokenFactor,
8475 // assuming we can verify that this would not create a cycle.
8476 return SDValue();
8477}
8478
8479SDValue SystemZTargetLowering::combineFP_ROUND(
8480 SDNode *N, DAGCombinerInfo &DCI) const {
8481
8482 if (!Subtarget.hasVector())
8483 return SDValue();
8484
8485 // (fpround (extract_vector_elt X 0))
8486 // (fpround (extract_vector_elt X 1)) ->
8487 // (extract_vector_elt (VROUND X) 0)
8488 // (extract_vector_elt (VROUND X) 2)
8489 //
8490 // This is a special case since the target doesn't really support v2f32s.
8491 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8492 SelectionDAG &DAG = DCI.DAG;
8493 SDValue Op0 = N->getOperand(Num: OpNo);
8494 if (N->getValueType(ResNo: 0) == MVT::f32 && Op0.hasOneUse() &&
8495 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8496 Op0.getOperand(i: 0).getValueType() == MVT::v2f64 &&
8497 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8498 Op0.getConstantOperandVal(i: 1) == 0) {
8499 SDValue Vec = Op0.getOperand(i: 0);
8500 for (auto *U : Vec->users()) {
8501 if (U != Op0.getNode() && U->hasOneUse() &&
8502 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8503 U->getOperand(Num: 0) == Vec &&
8504 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8505 U->getConstantOperandVal(Num: 1) == 1) {
8506 SDValue OtherRound = SDValue(*U->user_begin(), 0);
8507 if (OtherRound.getOpcode() == N->getOpcode() &&
8508 OtherRound.getOperand(i: OpNo) == SDValue(U, 0) &&
8509 OtherRound.getValueType() == MVT::f32) {
8510 SDValue VRound, Chain;
8511 if (N->isStrictFPOpcode()) {
8512 Chain = MergeInputChains(N1: N, N2: OtherRound.getNode());
8513 if (!Chain)
8514 continue;
8515 VRound = DAG.getNode(Opcode: SystemZISD::STRICT_VROUND, DL: SDLoc(N),
8516 ResultTys: {MVT::v4f32, MVT::Other}, Ops: {Chain, Vec});
8517 Chain = VRound.getValue(R: 1);
8518 } else
8519 VRound = DAG.getNode(Opcode: SystemZISD::VROUND, DL: SDLoc(N),
8520 VT: MVT::v4f32, Operand: Vec);
8521 DCI.AddToWorklist(N: VRound.getNode());
8522 SDValue Extract1 =
8523 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f32,
8524 N1: VRound, N2: DAG.getConstant(Val: 2, DL: SDLoc(U), VT: MVT::i32));
8525 DCI.AddToWorklist(N: Extract1.getNode());
8526 DAG.ReplaceAllUsesOfValueWith(From: OtherRound, To: Extract1);
8527 if (Chain)
8528 DAG.ReplaceAllUsesOfValueWith(From: OtherRound.getValue(R: 1), To: Chain);
8529 SDValue Extract0 =
8530 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f32,
8531 N1: VRound, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8532 if (Chain)
8533 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8534 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8535 return Extract0;
8536 }
8537 }
8538 }
8539 }
8540 return SDValue();
8541}
8542
8543SDValue SystemZTargetLowering::combineFP_EXTEND(
8544 SDNode *N, DAGCombinerInfo &DCI) const {
8545
8546 if (!Subtarget.hasVector())
8547 return SDValue();
8548
8549 // (fpextend (extract_vector_elt X 0))
8550 // (fpextend (extract_vector_elt X 2)) ->
8551 // (extract_vector_elt (VEXTEND X) 0)
8552 // (extract_vector_elt (VEXTEND X) 1)
8553 //
8554 // This is a special case since the target doesn't really support v2f32s.
8555 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8556 SelectionDAG &DAG = DCI.DAG;
8557 SDValue Op0 = N->getOperand(Num: OpNo);
8558 if (N->getValueType(ResNo: 0) == MVT::f64 && Op0.hasOneUse() &&
8559 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8560 Op0.getOperand(i: 0).getValueType() == MVT::v4f32 &&
8561 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8562 Op0.getConstantOperandVal(i: 1) == 0) {
8563 SDValue Vec = Op0.getOperand(i: 0);
8564 for (auto *U : Vec->users()) {
8565 if (U != Op0.getNode() && U->hasOneUse() &&
8566 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8567 U->getOperand(Num: 0) == Vec &&
8568 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8569 U->getConstantOperandVal(Num: 1) == 2) {
8570 SDValue OtherExtend = SDValue(*U->user_begin(), 0);
8571 if (OtherExtend.getOpcode() == N->getOpcode() &&
8572 OtherExtend.getOperand(i: OpNo) == SDValue(U, 0) &&
8573 OtherExtend.getValueType() == MVT::f64) {
8574 SDValue VExtend, Chain;
8575 if (N->isStrictFPOpcode()) {
8576 Chain = MergeInputChains(N1: N, N2: OtherExtend.getNode());
8577 if (!Chain)
8578 continue;
8579 VExtend = DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL: SDLoc(N),
8580 ResultTys: {MVT::v2f64, MVT::Other}, Ops: {Chain, Vec});
8581 Chain = VExtend.getValue(R: 1);
8582 } else
8583 VExtend = DAG.getNode(Opcode: SystemZISD::VEXTEND, DL: SDLoc(N),
8584 VT: MVT::v2f64, Operand: Vec);
8585 DCI.AddToWorklist(N: VExtend.getNode());
8586 SDValue Extract1 =
8587 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f64,
8588 N1: VExtend, N2: DAG.getConstant(Val: 1, DL: SDLoc(U), VT: MVT::i32));
8589 DCI.AddToWorklist(N: Extract1.getNode());
8590 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend, To: Extract1);
8591 if (Chain)
8592 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend.getValue(R: 1), To: Chain);
8593 SDValue Extract0 =
8594 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f64,
8595 N1: VExtend, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8596 if (Chain)
8597 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8598 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8599 return Extract0;
8600 }
8601 }
8602 }
8603 }
8604 return SDValue();
8605}
8606
8607SDValue SystemZTargetLowering::combineINT_TO_FP(
8608 SDNode *N, DAGCombinerInfo &DCI) const {
8609 if (DCI.Level != BeforeLegalizeTypes)
8610 return SDValue();
8611 SelectionDAG &DAG = DCI.DAG;
8612 LLVMContext &Ctx = *DAG.getContext();
8613 unsigned Opcode = N->getOpcode();
8614 EVT OutVT = N->getValueType(ResNo: 0);
8615 Type *OutLLVMTy = OutVT.getTypeForEVT(Context&: Ctx);
8616 SDValue Op = N->getOperand(Num: 0);
8617 unsigned OutScalarBits = OutLLVMTy->getScalarSizeInBits();
8618 unsigned InScalarBits = Op->getValueType(ResNo: 0).getScalarSizeInBits();
8619
8620 // Insert an extension before type-legalization to avoid scalarization, e.g.:
8621 // v2f64 = uint_to_fp v2i16
8622 // =>
8623 // v2f64 = uint_to_fp (v2i64 zero_extend v2i16)
8624 if (OutLLVMTy->isVectorTy() && OutScalarBits > InScalarBits &&
8625 OutScalarBits <= 64) {
8626 unsigned NumElts = cast<FixedVectorType>(Val: OutLLVMTy)->getNumElements();
8627 EVT ExtVT = EVT::getVectorVT(
8628 Context&: Ctx, VT: EVT::getIntegerVT(Context&: Ctx, BitWidth: OutLLVMTy->getScalarSizeInBits()), NumElements: NumElts);
8629 unsigned ExtOpcode =
8630 (Opcode == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND);
8631 SDValue ExtOp = DAG.getNode(Opcode: ExtOpcode, DL: SDLoc(N), VT: ExtVT, Operand: Op);
8632 return DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: ExtOp);
8633 }
8634 return SDValue();
8635}
8636
8637SDValue SystemZTargetLowering::combineFCOPYSIGN(
8638 SDNode *N, DAGCombinerInfo &DCI) const {
8639 SelectionDAG &DAG = DCI.DAG;
8640 EVT VT = N->getValueType(ResNo: 0);
8641 SDValue ValOp = N->getOperand(Num: 0);
8642 SDValue SignOp = N->getOperand(Num: 1);
8643
8644 // Remove the rounding which is not needed.
8645 if (SignOp.getOpcode() == ISD::FP_ROUND) {
8646 SDValue WideOp = SignOp.getOperand(i: 0);
8647 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N), VT, N1: ValOp, N2: WideOp);
8648 }
8649
8650 return SDValue();
8651}
8652
8653SDValue SystemZTargetLowering::combineBSWAP(
8654 SDNode *N, DAGCombinerInfo &DCI) const {
8655 SelectionDAG &DAG = DCI.DAG;
8656 // Combine BSWAP (LOAD) into LRVH/LRV/LRVG/VLBR
8657 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8658 N->getOperand(Num: 0).hasOneUse() &&
8659 canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0))) {
8660 SDValue Load = N->getOperand(Num: 0);
8661 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8662
8663 // Create the byte-swapping load.
8664 SDValue Ops[] = {
8665 LD->getChain(), // Chain
8666 LD->getBasePtr() // Ptr
8667 };
8668 EVT LoadVT = N->getValueType(ResNo: 0);
8669 if (LoadVT == MVT::i16)
8670 LoadVT = MVT::i32;
8671 SDValue BSLoad =
8672 DAG.getMemIntrinsicNode(Opcode: SystemZISD::LRV, dl: SDLoc(N),
8673 VTList: DAG.getVTList(VT1: LoadVT, VT2: MVT::Other),
8674 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8675
8676 // If this is an i16 load, insert the truncate.
8677 SDValue ResVal = BSLoad;
8678 if (N->getValueType(ResNo: 0) == MVT::i16)
8679 ResVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: MVT::i16, Operand: BSLoad);
8680
8681 // First, combine the bswap away. This makes the value produced by the
8682 // load dead.
8683 DCI.CombineTo(N, Res: ResVal);
8684
8685 // Next, combine the load away, we give it a bogus result value but a real
8686 // chain result. The result value is dead because the bswap is dead.
8687 DCI.CombineTo(N: Load.getNode(), Res0: ResVal, Res1: BSLoad.getValue(R: 1));
8688
8689 // Return N so it doesn't get rechecked!
8690 return SDValue(N, 0);
8691 }
8692
8693 // Look through bitcasts that retain the number of vector elements.
8694 SDValue Op = N->getOperand(Num: 0);
8695 if (Op.getOpcode() == ISD::BITCAST &&
8696 Op.getValueType().isVector() &&
8697 Op.getOperand(i: 0).getValueType().isVector() &&
8698 Op.getValueType().getVectorNumElements() ==
8699 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8700 Op = Op.getOperand(i: 0);
8701
8702 // Push BSWAP into a vector insertion if at least one side then simplifies.
8703 if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT && Op.hasOneUse()) {
8704 SDValue Vec = Op.getOperand(i: 0);
8705 SDValue Elt = Op.getOperand(i: 1);
8706 SDValue Idx = Op.getOperand(i: 2);
8707
8708 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Vec) ||
8709 Vec.getOpcode() == ISD::BSWAP || Vec.isUndef() ||
8710 DAG.isConstantIntBuildVectorOrConstantInt(N: Elt) ||
8711 Elt.getOpcode() == ISD::BSWAP || Elt.isUndef() ||
8712 (canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0)) &&
8713 ISD::isNON_EXTLoad(N: Elt.getNode()) && Elt.hasOneUse())) {
8714 EVT VecVT = N->getValueType(ResNo: 0);
8715 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
8716 if (VecVT != Vec.getValueType()) {
8717 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8718 DCI.AddToWorklist(N: Vec.getNode());
8719 }
8720 if (EltVT != Elt.getValueType()) {
8721 Elt = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8722 DCI.AddToWorklist(N: Elt.getNode());
8723 }
8724 Vec = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8725 DCI.AddToWorklist(N: Vec.getNode());
8726 Elt = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8727 DCI.AddToWorklist(N: Elt.getNode());
8728 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: VecVT,
8729 N1: Vec, N2: Elt, N3: Idx);
8730 }
8731 }
8732
8733 // Push BSWAP into a vector shuffle if at least one side then simplifies.
8734 ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(Val&: Op);
8735 if (SV && Op.hasOneUse()) {
8736 SDValue Op0 = Op.getOperand(i: 0);
8737 SDValue Op1 = Op.getOperand(i: 1);
8738
8739 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Op0) ||
8740 Op0.getOpcode() == ISD::BSWAP || Op0.isUndef() ||
8741 DAG.isConstantIntBuildVectorOrConstantInt(N: Op1) ||
8742 Op1.getOpcode() == ISD::BSWAP || Op1.isUndef()) {
8743 EVT VecVT = N->getValueType(ResNo: 0);
8744 if (VecVT != Op0.getValueType()) {
8745 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8746 DCI.AddToWorklist(N: Op0.getNode());
8747 }
8748 if (VecVT != Op1.getValueType()) {
8749 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8750 DCI.AddToWorklist(N: Op1.getNode());
8751 }
8752 Op0 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8753 DCI.AddToWorklist(N: Op0.getNode());
8754 Op1 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8755 DCI.AddToWorklist(N: Op1.getNode());
8756 return DAG.getVectorShuffle(VT: VecVT, dl: SDLoc(N), N1: Op0, N2: Op1, Mask: SV->getMask());
8757 }
8758 }
8759
8760 return SDValue();
8761}
8762
8763SDValue SystemZTargetLowering::combineSETCC(
8764 SDNode *N, DAGCombinerInfo &DCI) const {
8765 SelectionDAG &DAG = DCI.DAG;
8766 const ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
8767 const SDValue LHS = N->getOperand(Num: 0);
8768 const SDValue RHS = N->getOperand(Num: 1);
8769 bool CmpNull = isNullConstant(V: RHS);
8770 bool CmpAllOnes = isAllOnesConstant(V: RHS);
8771 EVT VT = N->getValueType(ResNo: 0);
8772 SDLoc DL(N);
8773
8774 // Match icmp_eq/ne(bitcast(icmp(X,Y)),0/-1) reduction patterns, and
8775 // change the outer compare to a i128 compare. This will normally
8776 // allow the reduction to be recognized in adjustICmp128, and even if
8777 // not, the i128 compare will still generate better code.
8778 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && (CmpNull || CmpAllOnes)) {
8779 SDValue Src = peekThroughBitcasts(V: LHS);
8780 if (Src.getOpcode() == ISD::SETCC &&
8781 Src.getValueType().isFixedLengthVector() &&
8782 Src.getValueType().getScalarType() == MVT::i1) {
8783 EVT CmpVT = Src.getOperand(i: 0).getValueType();
8784 if (CmpVT.getSizeInBits() == 128) {
8785 EVT IntVT = CmpVT.changeVectorElementTypeToInteger();
8786 SDValue LHS =
8787 DAG.getBitcast(VT: MVT::i128, V: DAG.getSExtOrTrunc(Op: Src, DL, VT: IntVT));
8788 SDValue RHS = CmpNull ? DAG.getConstant(Val: 0, DL, VT: MVT::i128)
8789 : DAG.getAllOnesConstant(DL, VT: MVT::i128);
8790 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: LHS, N2: RHS, N3: N->getOperand(Num: 2),
8791 Flags: N->getFlags());
8792 }
8793 }
8794 }
8795
8796 return SDValue();
8797}
8798
8799static std::pair<SDValue, int> findCCUse(const SDValue &Val,
8800 unsigned Depth = 0) {
8801 // Limit depth of potentially exponential walk.
8802 if (Depth > 5)
8803 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8804
8805 switch (Val.getOpcode()) {
8806 default:
8807 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8808 case SystemZISD::IPM:
8809 if (Val.getOperand(i: 0).getOpcode() == SystemZISD::CLC ||
8810 Val.getOperand(i: 0).getOpcode() == SystemZISD::STRCMP)
8811 return std::make_pair(x: Val.getOperand(i: 0), y: SystemZ::CCMASK_ICMP);
8812 return std::make_pair(x: Val.getOperand(i: 0), y: SystemZ::CCMASK_ANY);
8813 case SystemZISD::SELECT_CCMASK: {
8814 SDValue Op4CCReg = Val.getOperand(i: 4);
8815 if (Op4CCReg.getOpcode() == SystemZISD::ICMP ||
8816 Op4CCReg.getOpcode() == SystemZISD::TM) {
8817 auto [OpCC, OpCCValid] = findCCUse(Val: Op4CCReg.getOperand(i: 0), Depth: Depth + 1);
8818 if (OpCC != SDValue())
8819 return std::make_pair(x&: OpCC, y&: OpCCValid);
8820 }
8821 auto *CCValid = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 2));
8822 if (!CCValid)
8823 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8824 int CCValidVal = CCValid->getZExtValue();
8825 return std::make_pair(x&: Op4CCReg, y&: CCValidVal);
8826 }
8827 case ISD::ADD:
8828 case ISD::AND:
8829 case ISD::OR:
8830 case ISD::XOR:
8831 case ISD::SHL:
8832 case ISD::SRA:
8833 case ISD::SRL:
8834 auto [Op0CC, Op0CCValid] = findCCUse(Val: Val.getOperand(i: 0), Depth: Depth + 1);
8835 if (Op0CC != SDValue())
8836 return std::make_pair(x&: Op0CC, y&: Op0CCValid);
8837 return findCCUse(Val: Val.getOperand(i: 1), Depth: Depth + 1);
8838 }
8839}
8840
8841static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
8842 SelectionDAG &DAG);
8843
8844SmallVector<SDValue, 4> static simplifyAssumingCCVal(SDValue &Val, SDValue &CC,
8845 SelectionDAG &DAG) {
8846 SDLoc DL(Val);
8847 auto Opcode = Val.getOpcode();
8848 switch (Opcode) {
8849 default:
8850 return {};
8851 case ISD::Constant:
8852 return {Val, Val, Val, Val};
8853 case SystemZISD::IPM: {
8854 SDValue IPMOp0 = Val.getOperand(i: 0);
8855 if (IPMOp0 != CC)
8856 return {};
8857 SmallVector<SDValue, 4> ShiftedCCVals;
8858 for (auto CC : {0, 1, 2, 3})
8859 ShiftedCCVals.emplace_back(
8860 Args: DAG.getConstant(Val: (CC << SystemZ::IPM_CC), DL, VT: MVT::i32));
8861 return ShiftedCCVals;
8862 }
8863 case SystemZISD::SELECT_CCMASK: {
8864 SDValue TrueVal = Val.getOperand(i: 0), FalseVal = Val.getOperand(i: 1);
8865 auto *CCValid = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 2));
8866 auto *CCMask = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 3));
8867 if (!CCValid || !CCMask)
8868 return {};
8869
8870 int CCValidVal = CCValid->getZExtValue();
8871 int CCMaskVal = CCMask->getZExtValue();
8872 // Pruning search tree early - Moving CC test and combineCCMask ahead of
8873 // recursive call to simplifyAssumingCCVal.
8874 SDValue Op4CCReg = Val.getOperand(i: 4);
8875 if (Op4CCReg != CC)
8876 combineCCMask(CCReg&: Op4CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG);
8877 if (Op4CCReg != CC)
8878 return {};
8879 const auto &&TrueSDVals = simplifyAssumingCCVal(Val&: TrueVal, CC, DAG);
8880 const auto &&FalseSDVals = simplifyAssumingCCVal(Val&: FalseVal, CC, DAG);
8881 if (TrueSDVals.empty() || FalseSDVals.empty())
8882 return {};
8883 SmallVector<SDValue, 4> MergedSDVals;
8884 for (auto &CCVal : {0, 1, 2, 3})
8885 MergedSDVals.emplace_back(Args: ((CCMaskVal & (1 << (3 - CCVal))) != 0)
8886 ? TrueSDVals[CCVal]
8887 : FalseSDVals[CCVal]);
8888 return MergedSDVals;
8889 }
8890 case ISD::ADD:
8891 case ISD::AND:
8892 case ISD::OR:
8893 case ISD::XOR:
8894 case ISD::SRA:
8895 // Avoid introducing CC spills (because ADD/AND/OR/XOR/SRA
8896 // would clobber CC).
8897 if (!Val.hasOneUse())
8898 return {};
8899 [[fallthrough]];
8900 case ISD::SHL:
8901 case ISD::SRL:
8902 SDValue Op0 = Val.getOperand(i: 0), Op1 = Val.getOperand(i: 1);
8903 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: Op0, CC, DAG);
8904 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: Op1, CC, DAG);
8905 if (Op0SDVals.empty() || Op1SDVals.empty())
8906 return {};
8907 SmallVector<SDValue, 4> BinaryOpSDVals;
8908 for (auto CCVal : {0, 1, 2, 3})
8909 BinaryOpSDVals.emplace_back(Args: DAG.getNode(
8910 Opcode, DL, VT: Val.getValueType(), N1: Op0SDVals[CCVal], N2: Op1SDVals[CCVal]));
8911 return BinaryOpSDVals;
8912 }
8913}
8914
8915static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
8916 SelectionDAG &DAG) {
8917 // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
8918 // set by the CCReg instruction using the CCValid / CCMask masks,
8919 // If the CCReg instruction is itself a ICMP / TM testing the condition
8920 // code set by some other instruction, see whether we can directly
8921 // use that condition code.
8922 auto *CCNode = CCReg.getNode();
8923 if (!CCNode)
8924 return false;
8925
8926 if (CCNode->getOpcode() == SystemZISD::TM) {
8927 if (CCValid != SystemZ::CCMASK_TM)
8928 return false;
8929 auto emulateTMCCMask = [](const SDValue &Op0Val, const SDValue &Op1Val) {
8930 auto *Op0Node = dyn_cast<ConstantSDNode>(Val: Op0Val.getNode());
8931 auto *Op1Node = dyn_cast<ConstantSDNode>(Val: Op1Val.getNode());
8932 if (!Op0Node || !Op1Node)
8933 return -1;
8934 auto Op0APVal = Op0Node->getAPIntValue();
8935 auto Op1APVal = Op1Node->getAPIntValue();
8936 auto Result = Op0APVal & Op1APVal;
8937 bool AllOnes = Result == Op1APVal;
8938 bool AllZeros = Result == 0;
8939 bool IsLeftMostBitSet = Result[Op1APVal.getActiveBits() - 1] != 0;
8940 return AllZeros ? 0 : AllOnes ? 3 : IsLeftMostBitSet ? 2 : 1;
8941 };
8942 SDValue Op0 = CCNode->getOperand(Num: 0);
8943 SDValue Op1 = CCNode->getOperand(Num: 1);
8944 auto [Op0CC, Op0CCValid] = findCCUse(Val: Op0);
8945 if (Op0CC == SDValue())
8946 return false;
8947 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: Op0, CC&: Op0CC, DAG);
8948 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: Op1, CC&: Op0CC, DAG);
8949 if (Op0SDVals.empty() || Op1SDVals.empty())
8950 return false;
8951 int NewCCMask = 0;
8952 for (auto CC : {0, 1, 2, 3}) {
8953 auto CCVal = emulateTMCCMask(Op0SDVals[CC], Op1SDVals[CC]);
8954 if (CCVal < 0)
8955 return false;
8956 NewCCMask <<= 1;
8957 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
8958 }
8959 NewCCMask &= Op0CCValid;
8960 CCReg = Op0CC;
8961 CCMask = NewCCMask;
8962 CCValid = Op0CCValid;
8963 return true;
8964 }
8965 if (CCNode->getOpcode() != SystemZISD::ICMP ||
8966 CCValid != SystemZ::CCMASK_ICMP)
8967 return false;
8968
8969 SDValue CmpOp0 = CCNode->getOperand(Num: 0);
8970 SDValue CmpOp1 = CCNode->getOperand(Num: 1);
8971 SDValue CmpOp2 = CCNode->getOperand(Num: 2);
8972 auto [Op0CC, Op0CCValid] = findCCUse(Val: CmpOp0);
8973 if (Op0CC != SDValue()) {
8974 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: CmpOp0, CC&: Op0CC, DAG);
8975 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: CmpOp1, CC&: Op0CC, DAG);
8976 if (Op0SDVals.empty() || Op1SDVals.empty())
8977 return false;
8978
8979 auto *CmpType = dyn_cast<ConstantSDNode>(Val&: CmpOp2);
8980 auto CmpTypeVal = CmpType->getZExtValue();
8981 const auto compareCCSigned = [&CmpTypeVal](const SDValue &Op0Val,
8982 const SDValue &Op1Val) {
8983 auto *Op0Node = dyn_cast<ConstantSDNode>(Val: Op0Val.getNode());
8984 auto *Op1Node = dyn_cast<ConstantSDNode>(Val: Op1Val.getNode());
8985 if (!Op0Node || !Op1Node)
8986 return -1;
8987 auto Op0APVal = Op0Node->getAPIntValue();
8988 auto Op1APVal = Op1Node->getAPIntValue();
8989 if (CmpTypeVal == SystemZICMP::SignedOnly)
8990 return Op0APVal == Op1APVal ? 0 : Op0APVal.slt(RHS: Op1APVal) ? 1 : 2;
8991 return Op0APVal == Op1APVal ? 0 : Op0APVal.ult(RHS: Op1APVal) ? 1 : 2;
8992 };
8993 int NewCCMask = 0;
8994 for (auto CC : {0, 1, 2, 3}) {
8995 auto CCVal = compareCCSigned(Op0SDVals[CC], Op1SDVals[CC]);
8996 if (CCVal < 0)
8997 return false;
8998 NewCCMask <<= 1;
8999 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
9000 }
9001 NewCCMask &= Op0CCValid;
9002 CCMask = NewCCMask;
9003 CCReg = Op0CC;
9004 CCValid = Op0CCValid;
9005 return true;
9006 }
9007
9008 return false;
9009}
9010
9011// Merging versus split in multiple branches cost.
9012TargetLoweringBase::CondMergingParams
9013SystemZTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
9014 const Value *Lhs,
9015 const Value *Rhs,
9016 const Function *) const {
9017 const auto isFlagOutOpCC = [](const Value *V) {
9018 using namespace llvm::PatternMatch;
9019 const Value *RHSVal;
9020 const APInt *RHSC;
9021 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
9022 // PatternMatch.h provides concise tree-based pattern match of llvm IR.
9023 if (match(V: I->getOperand(i: 0), P: m_And(L: m_Value(V&: RHSVal), R: m_APInt(Res&: RHSC))) ||
9024 match(V: I, P: m_Cmp(L: m_Value(V&: RHSVal), R: m_APInt(Res&: RHSC)))) {
9025 if (const auto *CB = dyn_cast<CallBase>(Val: RHSVal)) {
9026 if (CB->isInlineAsm()) {
9027 const InlineAsm *IA = cast<InlineAsm>(Val: CB->getCalledOperand());
9028 return IA && IA->getConstraintString().contains(Other: "{@cc}");
9029 }
9030 }
9031 }
9032 }
9033 return false;
9034 };
9035 // Pattern (ICmp %asm) or (ICmp (And %asm)).
9036 // Cost of longest dependency chain (ICmp, And) is 2. CostThreshold or
9037 // BaseCost can be set >=2. If cost of instruction <= CostThreshold
9038 // conditionals will be merged or else conditionals will be split.
9039 if (isFlagOutOpCC(Lhs) && isFlagOutOpCC(Rhs))
9040 return {.BaseCost: 3, .LikelyBias: 0, .UnlikelyBias: -1};
9041 // Default.
9042 return {.BaseCost: -1, .LikelyBias: -1, .UnlikelyBias: -1};
9043}
9044
9045SDValue SystemZTargetLowering::combineBR_CCMASK(SDNode *N,
9046 DAGCombinerInfo &DCI) const {
9047 SelectionDAG &DAG = DCI.DAG;
9048
9049 // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK.
9050 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
9051 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9052 if (!CCValid || !CCMask)
9053 return SDValue();
9054
9055 int CCValidVal = CCValid->getZExtValue();
9056 int CCMaskVal = CCMask->getZExtValue();
9057 SDValue Chain = N->getOperand(Num: 0);
9058 SDValue CCReg = N->getOperand(Num: 4);
9059 // If combineCMask was able to merge or simplify ccvalid or ccmask, re-emit
9060 // the modified BR_CCMASK with the new values.
9061 // In order to avoid conditional branches with full or empty cc masks, do not
9062 // do this if ccmask is 0 or equal to ccvalid.
9063 if (combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG) && CCMaskVal != 0 &&
9064 CCMaskVal != CCValidVal)
9065 return DAG.getNode(Opcode: SystemZISD::BR_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
9066 N1: Chain,
9067 N2: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
9068 N3: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32),
9069 N4: N->getOperand(Num: 3), N5: CCReg);
9070 return SDValue();
9071}
9072
9073SDValue SystemZTargetLowering::combineSELECT_CCMASK(
9074 SDNode *N, DAGCombinerInfo &DCI) const {
9075 SelectionDAG &DAG = DCI.DAG;
9076
9077 // Combine SELECT_CCMASK (ICMP (SELECT_CCMASK)) into a single SELECT_CCMASK.
9078 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9079 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3));
9080 if (!CCValid || !CCMask)
9081 return SDValue();
9082
9083 int CCValidVal = CCValid->getZExtValue();
9084 int CCMaskVal = CCMask->getZExtValue();
9085 SDValue CCReg = N->getOperand(Num: 4);
9086
9087 bool IsCombinedCCReg = combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG);
9088
9089 // Populate SDVals vector for each condition code ccval for given Val, which
9090 // can again be another nested select_ccmask with the same CC.
9091 const auto constructCCSDValsFromSELECT = [&CCReg](SDValue &Val) {
9092 if (Val.getOpcode() == SystemZISD::SELECT_CCMASK) {
9093 SmallVector<SDValue, 4> Res;
9094 if (Val.getOperand(i: 4) != CCReg)
9095 return SmallVector<SDValue, 4>{};
9096 SDValue TrueVal = Val.getOperand(i: 0), FalseVal = Val.getOperand(i: 1);
9097 auto *CCMask = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 3));
9098 if (!CCMask)
9099 return SmallVector<SDValue, 4>{};
9100
9101 int CCMaskVal = CCMask->getZExtValue();
9102 for (auto &CC : {0, 1, 2, 3})
9103 Res.emplace_back(Args&: ((CCMaskVal & (1 << (3 - CC))) != 0) ? TrueVal
9104 : FalseVal);
9105 return Res;
9106 }
9107 return SmallVector<SDValue, 4>{Val, Val, Val, Val};
9108 };
9109 // Attempting to optimize TrueVal/FalseVal in outermost select_ccmask either
9110 // with CCReg found by combineCCMask or original CCReg.
9111 SDValue TrueVal = N->getOperand(Num: 0);
9112 SDValue FalseVal = N->getOperand(Num: 1);
9113 auto &&TrueSDVals = simplifyAssumingCCVal(Val&: TrueVal, CC&: CCReg, DAG);
9114 auto &&FalseSDVals = simplifyAssumingCCVal(Val&: FalseVal, CC&: CCReg, DAG);
9115 // TrueSDVals/FalseSDVals might be empty in case of non-constant
9116 // TrueVal/FalseVal for select_ccmask, which can not be optimized further.
9117 if (TrueSDVals.empty())
9118 TrueSDVals = constructCCSDValsFromSELECT(TrueVal);
9119 if (FalseSDVals.empty())
9120 FalseSDVals = constructCCSDValsFromSELECT(FalseVal);
9121 if (!TrueSDVals.empty() && !FalseSDVals.empty()) {
9122 SmallSet<SDValue, 4> MergedSDValsSet;
9123 // Ignoring CC values outside CCValiid.
9124 for (auto CC : {0, 1, 2, 3}) {
9125 if ((CCValidVal & ((1 << (3 - CC)))) != 0)
9126 MergedSDValsSet.insert(V: ((CCMaskVal & (1 << (3 - CC))) != 0)
9127 ? TrueSDVals[CC]
9128 : FalseSDVals[CC]);
9129 }
9130 if (MergedSDValsSet.size() == 1)
9131 return *MergedSDValsSet.begin();
9132 if (MergedSDValsSet.size() == 2) {
9133 auto BeginIt = MergedSDValsSet.begin();
9134 SDValue NewTrueVal = *BeginIt, NewFalseVal = *next(x: BeginIt);
9135 if (NewTrueVal == FalseVal || NewFalseVal == TrueVal)
9136 std::swap(a&: NewTrueVal, b&: NewFalseVal);
9137 int NewCCMask = 0;
9138 for (auto CC : {0, 1, 2, 3}) {
9139 NewCCMask <<= 1;
9140 NewCCMask |= ((CCMaskVal & (1 << (3 - CC))) != 0)
9141 ? (TrueSDVals[CC] == NewTrueVal)
9142 : (FalseSDVals[CC] == NewTrueVal);
9143 }
9144 CCMaskVal = NewCCMask;
9145 CCMaskVal &= CCValidVal;
9146 TrueVal = NewTrueVal;
9147 FalseVal = NewFalseVal;
9148 IsCombinedCCReg = true;
9149 }
9150 }
9151 // If the condition is trivially false or trivially true after
9152 // combineCCMask, just collapse this SELECT_CCMASK to the indicated value
9153 // (possibly modified by constructCCSDValsFromSELECT).
9154 if (CCMaskVal == 0)
9155 return FalseVal;
9156 if (CCMaskVal == CCValidVal)
9157 return TrueVal;
9158
9159 if (IsCombinedCCReg)
9160 return DAG.getNode(
9161 Opcode: SystemZISD::SELECT_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: TrueVal,
9162 N2: FalseVal, N3: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
9163 N4: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32), N5: CCReg);
9164
9165 return SDValue();
9166}
9167
9168SDValue SystemZTargetLowering::combineGET_CCMASK(
9169 SDNode *N, DAGCombinerInfo &DCI) const {
9170
9171 // Optimize away GET_CCMASK (SELECT_CCMASK) if the CC masks are compatible
9172 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
9173 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9174 if (!CCValid || !CCMask)
9175 return SDValue();
9176 int CCValidVal = CCValid->getZExtValue();
9177 int CCMaskVal = CCMask->getZExtValue();
9178
9179 SDValue Select = N->getOperand(Num: 0);
9180 if (Select->getOpcode() == ISD::TRUNCATE)
9181 Select = Select->getOperand(Num: 0);
9182 if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
9183 return SDValue();
9184
9185 auto *SelectCCValid = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 2));
9186 auto *SelectCCMask = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 3));
9187 if (!SelectCCValid || !SelectCCMask)
9188 return SDValue();
9189 int SelectCCValidVal = SelectCCValid->getZExtValue();
9190 int SelectCCMaskVal = SelectCCMask->getZExtValue();
9191
9192 auto *TrueVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 0));
9193 auto *FalseVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 1));
9194 if (!TrueVal || !FalseVal)
9195 return SDValue();
9196 if (TrueVal->getZExtValue() == 1 && FalseVal->getZExtValue() == 0)
9197 ;
9198 else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() == 1)
9199 SelectCCMaskVal ^= SelectCCValidVal;
9200 else
9201 return SDValue();
9202
9203 if (SelectCCValidVal & ~CCValidVal)
9204 return SDValue();
9205 if (SelectCCMaskVal != (CCMaskVal & SelectCCValidVal))
9206 return SDValue();
9207
9208 return Select->getOperand(Num: 4);
9209}
9210
9211SDValue SystemZTargetLowering::combineIntDIVREM(
9212 SDNode *N, DAGCombinerInfo &DCI) const {
9213 SelectionDAG &DAG = DCI.DAG;
9214 EVT VT = N->getValueType(ResNo: 0);
9215 // In the case where the divisor is a vector of constants a cheaper
9216 // sequence of instructions can replace the divide. BuildSDIV is called to
9217 // do this during DAG combining, but it only succeeds when it can build a
9218 // multiplication node. The only option for SystemZ is ISD::SMUL_LOHI, and
9219 // since it is not Legal but Custom it can only happen before
9220 // legalization. Therefore we must scalarize this early before Combine
9221 // 1. For widened vectors, this is already the result of type legalization.
9222 if (DCI.Level == BeforeLegalizeTypes && VT.isVector() && isTypeLegal(VT) &&
9223 DAG.isConstantIntBuildVectorOrConstantInt(N: N->getOperand(Num: 1)))
9224 return DAG.UnrollVectorOp(N);
9225 return SDValue();
9226}
9227
9228
9229// Transform a right shift of a multiply-and-add into a multiply-and-add-high.
9230// This is closely modeled after the common-code combineShiftToMULH.
9231SDValue SystemZTargetLowering::combineShiftToMulAddHigh(
9232 SDNode *N, DAGCombinerInfo &DCI) const {
9233 SelectionDAG &DAG = DCI.DAG;
9234 SDLoc DL(N);
9235
9236 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
9237 "SRL or SRA node is required here!");
9238
9239 if (!Subtarget.hasVector())
9240 return SDValue();
9241
9242 // Check the shift amount. Proceed with the transformation if the shift
9243 // amount is constant.
9244 ConstantSDNode *ShiftAmtSrc = isConstOrConstSplat(N: N->getOperand(Num: 1));
9245 if (!ShiftAmtSrc)
9246 return SDValue();
9247
9248 // The operation feeding into the shift must be an add.
9249 SDValue ShiftOperand = N->getOperand(Num: 0);
9250 if (ShiftOperand.getOpcode() != ISD::ADD)
9251 return SDValue();
9252
9253 // One operand of the add must be a multiply.
9254 SDValue MulOp = ShiftOperand.getOperand(i: 0);
9255 SDValue AddOp = ShiftOperand.getOperand(i: 1);
9256 if (MulOp.getOpcode() != ISD::MUL) {
9257 if (AddOp.getOpcode() != ISD::MUL)
9258 return SDValue();
9259 std::swap(a&: MulOp, b&: AddOp);
9260 }
9261
9262 // All operands must be equivalent extend nodes.
9263 SDValue LeftOp = MulOp.getOperand(i: 0);
9264 SDValue RightOp = MulOp.getOperand(i: 1);
9265
9266 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND;
9267 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND;
9268
9269 if (!IsSignExt && !IsZeroExt)
9270 return SDValue();
9271
9272 EVT NarrowVT = LeftOp.getOperand(i: 0).getValueType();
9273 unsigned NarrowVTSize = NarrowVT.getScalarSizeInBits();
9274
9275 SDValue MulhRightOp;
9276 if (ConstantSDNode *Constant = isConstOrConstSplat(N: RightOp)) {
9277 unsigned ActiveBits = IsSignExt
9278 ? Constant->getAPIntValue().getSignificantBits()
9279 : Constant->getAPIntValue().getActiveBits();
9280 if (ActiveBits > NarrowVTSize)
9281 return SDValue();
9282 MulhRightOp = DAG.getConstant(
9283 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
9284 VT: NarrowVT);
9285 } else {
9286 if (LeftOp.getOpcode() != RightOp.getOpcode())
9287 return SDValue();
9288 // Check that the two extend nodes are the same type.
9289 if (NarrowVT != RightOp.getOperand(i: 0).getValueType())
9290 return SDValue();
9291 MulhRightOp = RightOp.getOperand(i: 0);
9292 }
9293
9294 SDValue MulhAddOp;
9295 if (ConstantSDNode *Constant = isConstOrConstSplat(N: AddOp)) {
9296 unsigned ActiveBits = IsSignExt
9297 ? Constant->getAPIntValue().getSignificantBits()
9298 : Constant->getAPIntValue().getActiveBits();
9299 if (ActiveBits > NarrowVTSize)
9300 return SDValue();
9301 MulhAddOp = DAG.getConstant(
9302 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
9303 VT: NarrowVT);
9304 } else {
9305 if (LeftOp.getOpcode() != AddOp.getOpcode())
9306 return SDValue();
9307 // Check that the two extend nodes are the same type.
9308 if (NarrowVT != AddOp.getOperand(i: 0).getValueType())
9309 return SDValue();
9310 MulhAddOp = AddOp.getOperand(i: 0);
9311 }
9312
9313 EVT WideVT = LeftOp.getValueType();
9314 // Proceed with the transformation if the wide types match.
9315 assert((WideVT == RightOp.getValueType()) &&
9316 "Cannot have a multiply node with two different operand types.");
9317 assert((WideVT == AddOp.getValueType()) &&
9318 "Cannot have an add node with two different operand types.");
9319
9320 // Proceed with the transformation if the wide type is twice as large
9321 // as the narrow type.
9322 if (WideVT.getScalarSizeInBits() != 2 * NarrowVTSize)
9323 return SDValue();
9324
9325 // Check the shift amount with the narrow type size.
9326 // Proceed with the transformation if the shift amount is the width
9327 // of the narrow type.
9328 unsigned ShiftAmt = ShiftAmtSrc->getZExtValue();
9329 if (ShiftAmt != NarrowVTSize)
9330 return SDValue();
9331
9332 // Proceed if we support the multiply-and-add-high operation.
9333 if (!(NarrowVT == MVT::v16i8 || NarrowVT == MVT::v8i16 ||
9334 NarrowVT == MVT::v4i32 ||
9335 (Subtarget.hasVectorEnhancements3() &&
9336 (NarrowVT == MVT::v2i64 || NarrowVT == MVT::i128))))
9337 return SDValue();
9338
9339 // Emit the VMAH (signed) or VMALH (unsigned) operation.
9340 SDValue Result = DAG.getNode(Opcode: IsSignExt ? SystemZISD::VMAH : SystemZISD::VMALH,
9341 DL, VT: NarrowVT, N1: LeftOp.getOperand(i: 0),
9342 N2: MulhRightOp, N3: MulhAddOp);
9343 bool IsSigned = N->getOpcode() == ISD::SRA;
9344 return DAG.getExtOrTrunc(IsSigned, Op: Result, DL, VT: WideVT);
9345}
9346
9347// Op is an operand of a multiplication. Check whether this can be folded
9348// into an even/odd widening operation; if so, return the opcode to be used
9349// and update Op to the appropriate sub-operand. Note that the caller must
9350// verify that *both* operands of the multiplication support the operation.
9351static unsigned detectEvenOddMultiplyOperand(const SelectionDAG &DAG,
9352 const SystemZSubtarget &Subtarget,
9353 SDValue &Op) {
9354 EVT VT = Op.getValueType();
9355
9356 // Check for (sign/zero_extend_vector_inreg (vector_shuffle)) corresponding
9357 // to selecting the even or odd vector elements.
9358 if (VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
9359 (Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
9360 Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG)) {
9361 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
9362 unsigned NumElts = VT.getVectorNumElements();
9363 Op = Op.getOperand(i: 0);
9364 if (Op.getValueType().getVectorNumElements() == 2 * NumElts &&
9365 Op.getOpcode() == ISD::VECTOR_SHUFFLE) {
9366 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
9367 ArrayRef<int> ShuffleMask = SVN->getMask();
9368 bool CanUseEven = true, CanUseOdd = true;
9369 for (unsigned Elt = 0; Elt < NumElts; Elt++) {
9370 if (ShuffleMask[Elt] == -1)
9371 continue;
9372 if (unsigned(ShuffleMask[Elt]) != 2 * Elt)
9373 CanUseEven = false;
9374 if (unsigned(ShuffleMask[Elt]) != 2 * Elt + 1)
9375 CanUseOdd = false;
9376 }
9377 Op = Op.getOperand(i: 0);
9378 if (CanUseEven)
9379 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9380 if (CanUseOdd)
9381 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9382 }
9383 }
9384
9385 // For z17, we can also support the v2i64->i128 case, which looks like
9386 // (sign/zero_extend (extract_vector_elt X 0/1))
9387 if (VT == MVT::i128 && Subtarget.hasVectorEnhancements3() &&
9388 (Op.getOpcode() == ISD::SIGN_EXTEND ||
9389 Op.getOpcode() == ISD::ZERO_EXTEND)) {
9390 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND;
9391 Op = Op.getOperand(i: 0);
9392 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
9393 Op.getOperand(i: 0).getValueType() == MVT::v2i64 &&
9394 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
9395 unsigned Elem = Op.getConstantOperandVal(i: 1);
9396 Op = Op.getOperand(i: 0);
9397 if (Elem == 0)
9398 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9399 if (Elem == 1)
9400 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9401 }
9402 }
9403
9404 return 0;
9405}
9406
9407SDValue SystemZTargetLowering::combineMUL(
9408 SDNode *N, DAGCombinerInfo &DCI) const {
9409 SelectionDAG &DAG = DCI.DAG;
9410
9411 // Detect even/odd widening multiplication.
9412 SDValue Op0 = N->getOperand(Num: 0);
9413 SDValue Op1 = N->getOperand(Num: 1);
9414 unsigned OpcodeCand0 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op0);
9415 unsigned OpcodeCand1 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op1);
9416 if (OpcodeCand0 && OpcodeCand0 == OpcodeCand1)
9417 return DAG.getNode(Opcode: OpcodeCand0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1);
9418
9419 return SDValue();
9420}
9421
9422SDValue SystemZTargetLowering::combineINTRINSIC(
9423 SDNode *N, DAGCombinerInfo &DCI) const {
9424 SelectionDAG &DAG = DCI.DAG;
9425
9426 unsigned Id = N->getConstantOperandVal(Num: 1);
9427 switch (Id) {
9428 // VECTOR LOAD (RIGHTMOST) WITH LENGTH with a length operand of 15
9429 // or larger is simply a vector load.
9430 case Intrinsic::s390_vll:
9431 case Intrinsic::s390_vlrl:
9432 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2)))
9433 if (C->getZExtValue() >= 15)
9434 return DAG.getLoad(VT: N->getValueType(ResNo: 0), dl: SDLoc(N), Chain: N->getOperand(Num: 0),
9435 Ptr: N->getOperand(Num: 3), PtrInfo: MachinePointerInfo());
9436 break;
9437 // Likewise for VECTOR STORE (RIGHTMOST) WITH LENGTH.
9438 case Intrinsic::s390_vstl:
9439 case Intrinsic::s390_vstrl:
9440 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3)))
9441 if (C->getZExtValue() >= 15)
9442 return DAG.getStore(Chain: N->getOperand(Num: 0), dl: SDLoc(N), Val: N->getOperand(Num: 2),
9443 Ptr: N->getOperand(Num: 4), PtrInfo: MachinePointerInfo());
9444 break;
9445 }
9446
9447 return SDValue();
9448}
9449
9450SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
9451 if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
9452 return N->getOperand(Num: 0);
9453 return N;
9454}
9455
9456SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
9457 DAGCombinerInfo &DCI) const {
9458 switch(N->getOpcode()) {
9459 default: break;
9460 case ISD::ZERO_EXTEND: return combineZERO_EXTEND(N, DCI);
9461 case ISD::SIGN_EXTEND: return combineSIGN_EXTEND(N, DCI);
9462 case ISD::SIGN_EXTEND_INREG: return combineSIGN_EXTEND_INREG(N, DCI);
9463 case SystemZISD::MERGE_HIGH:
9464 case SystemZISD::MERGE_LOW: return combineMERGE(N, DCI);
9465 case ISD::LOAD: return combineLOAD(N, DCI);
9466 case ISD::STORE: return combineSTORE(N, DCI);
9467 case ISD::VECTOR_SHUFFLE: return combineVECTOR_SHUFFLE(N, DCI);
9468 case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI);
9469 case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI);
9470 case ISD::STRICT_FP_ROUND:
9471 case ISD::FP_ROUND: return combineFP_ROUND(N, DCI);
9472 case ISD::STRICT_FP_EXTEND:
9473 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DCI);
9474 case ISD::SINT_TO_FP:
9475 case ISD::UINT_TO_FP: return combineINT_TO_FP(N, DCI);
9476 case ISD::FCOPYSIGN: return combineFCOPYSIGN(N, DCI);
9477 case ISD::BSWAP: return combineBSWAP(N, DCI);
9478 case ISD::SETCC: return combineSETCC(N, DCI);
9479 case SystemZISD::BR_CCMASK: return combineBR_CCMASK(N, DCI);
9480 case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI);
9481 case SystemZISD::GET_CCMASK: return combineGET_CCMASK(N, DCI);
9482 case ISD::SRL:
9483 case ISD::SRA: return combineShiftToMulAddHigh(N, DCI);
9484 case ISD::MUL: return combineMUL(N, DCI);
9485 case ISD::SDIV:
9486 case ISD::UDIV:
9487 case ISD::SREM:
9488 case ISD::UREM: return combineIntDIVREM(N, DCI);
9489 case ISD::INTRINSIC_W_CHAIN:
9490 case ISD::INTRINSIC_VOID: return combineINTRINSIC(N, DCI);
9491 }
9492
9493 return SDValue();
9494}
9495
9496// Return the demanded elements for the OpNo source operand of Op. DemandedElts
9497// are for Op.
9498static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts,
9499 unsigned OpNo) {
9500 EVT VT = Op.getValueType();
9501 unsigned NumElts = (VT.isVector() ? VT.getVectorNumElements() : 1);
9502 APInt SrcDemE;
9503 unsigned Opcode = Op.getOpcode();
9504 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9505 unsigned Id = Op.getConstantOperandVal(i: 0);
9506 switch (Id) {
9507 case Intrinsic::s390_vpksh: // PACKS
9508 case Intrinsic::s390_vpksf:
9509 case Intrinsic::s390_vpksg:
9510 case Intrinsic::s390_vpkshs: // PACKS_CC
9511 case Intrinsic::s390_vpksfs:
9512 case Intrinsic::s390_vpksgs:
9513 case Intrinsic::s390_vpklsh: // PACKLS
9514 case Intrinsic::s390_vpklsf:
9515 case Intrinsic::s390_vpklsg:
9516 case Intrinsic::s390_vpklshs: // PACKLS_CC
9517 case Intrinsic::s390_vpklsfs:
9518 case Intrinsic::s390_vpklsgs:
9519 // VECTOR PACK truncates the elements of two source vectors into one.
9520 SrcDemE = DemandedElts;
9521 if (OpNo == 2)
9522 SrcDemE.lshrInPlace(ShiftAmt: NumElts / 2);
9523 SrcDemE = SrcDemE.trunc(width: NumElts / 2);
9524 break;
9525 // VECTOR UNPACK extends half the elements of the source vector.
9526 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9527 case Intrinsic::s390_vuphh:
9528 case Intrinsic::s390_vuphf:
9529 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9530 case Intrinsic::s390_vuplhh:
9531 case Intrinsic::s390_vuplhf:
9532 SrcDemE = APInt(NumElts * 2, 0);
9533 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: 0);
9534 break;
9535 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9536 case Intrinsic::s390_vuplhw:
9537 case Intrinsic::s390_vuplf:
9538 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9539 case Intrinsic::s390_vupllh:
9540 case Intrinsic::s390_vupllf:
9541 SrcDemE = APInt(NumElts * 2, 0);
9542 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: NumElts);
9543 break;
9544 case Intrinsic::s390_vpdi: {
9545 // VECTOR PERMUTE DWORD IMMEDIATE selects one element from each source.
9546 SrcDemE = APInt(NumElts, 0);
9547 if (!DemandedElts[OpNo - 1])
9548 break;
9549 unsigned Mask = Op.getConstantOperandVal(i: 3);
9550 unsigned MaskBit = ((OpNo - 1) ? 1 : 4);
9551 // Demand input element 0 or 1, given by the mask bit value.
9552 SrcDemE.setBit((Mask & MaskBit)? 1 : 0);
9553 break;
9554 }
9555 case Intrinsic::s390_vsldb: {
9556 // VECTOR SHIFT LEFT DOUBLE BY BYTE
9557 assert(VT == MVT::v16i8 && "Unexpected type.");
9558 unsigned FirstIdx = Op.getConstantOperandVal(i: 3);
9559 assert (FirstIdx > 0 && FirstIdx < 16 && "Unused operand.");
9560 unsigned NumSrc0Els = 16 - FirstIdx;
9561 SrcDemE = APInt(NumElts, 0);
9562 if (OpNo == 1) {
9563 APInt DemEls = DemandedElts.trunc(width: NumSrc0Els);
9564 SrcDemE.insertBits(SubBits: DemEls, bitPosition: FirstIdx);
9565 } else {
9566 APInt DemEls = DemandedElts.lshr(shiftAmt: NumSrc0Els);
9567 SrcDemE.insertBits(SubBits: DemEls, bitPosition: 0);
9568 }
9569 break;
9570 }
9571 case Intrinsic::s390_vperm:
9572 SrcDemE = APInt::getAllOnes(numBits: NumElts);
9573 break;
9574 default:
9575 llvm_unreachable("Unhandled intrinsic.");
9576 break;
9577 }
9578 } else {
9579 switch (Opcode) {
9580 case SystemZISD::JOIN_DWORDS:
9581 // Scalar operand.
9582 SrcDemE = APInt(1, 1);
9583 break;
9584 case SystemZISD::SELECT_CCMASK:
9585 SrcDemE = DemandedElts;
9586 break;
9587 default:
9588 llvm_unreachable("Unhandled opcode.");
9589 break;
9590 }
9591 }
9592 return SrcDemE;
9593}
9594
9595static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known,
9596 const APInt &DemandedElts,
9597 const SelectionDAG &DAG, unsigned Depth,
9598 unsigned OpNo) {
9599 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9600 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9601 KnownBits LHSKnown =
9602 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9603 KnownBits RHSKnown =
9604 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9605 Known = LHSKnown.intersectWith(RHS: RHSKnown);
9606}
9607
9608void
9609SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
9610 KnownBits &Known,
9611 const APInt &DemandedElts,
9612 const SelectionDAG &DAG,
9613 unsigned Depth) const {
9614 Known.resetAll();
9615
9616 // Intrinsic CC result is returned in the two low bits.
9617 unsigned Tmp0, Tmp1; // not used
9618 if (Op.getResNo() == 1 && isIntrinsicWithCC(Op, Opcode&: Tmp0, CCValid&: Tmp1)) {
9619 Known.Zero.setBitsFrom(2);
9620 return;
9621 }
9622 EVT VT = Op.getValueType();
9623 if (Op.getResNo() != 0 || VT == MVT::Untyped)
9624 return;
9625 assert (Known.getBitWidth() == VT.getScalarSizeInBits() &&
9626 "KnownBits does not match VT in bitwidth");
9627 assert ((!VT.isVector() ||
9628 (DemandedElts.getBitWidth() == VT.getVectorNumElements())) &&
9629 "DemandedElts does not match VT number of elements");
9630 unsigned BitWidth = Known.getBitWidth();
9631 unsigned Opcode = Op.getOpcode();
9632 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9633 bool IsLogical = false;
9634 unsigned Id = Op.getConstantOperandVal(i: 0);
9635 switch (Id) {
9636 case Intrinsic::s390_vpksh: // PACKS
9637 case Intrinsic::s390_vpksf:
9638 case Intrinsic::s390_vpksg:
9639 case Intrinsic::s390_vpkshs: // PACKS_CC
9640 case Intrinsic::s390_vpksfs:
9641 case Intrinsic::s390_vpksgs:
9642 case Intrinsic::s390_vpklsh: // PACKLS
9643 case Intrinsic::s390_vpklsf:
9644 case Intrinsic::s390_vpklsg:
9645 case Intrinsic::s390_vpklshs: // PACKLS_CC
9646 case Intrinsic::s390_vpklsfs:
9647 case Intrinsic::s390_vpklsgs:
9648 case Intrinsic::s390_vpdi:
9649 case Intrinsic::s390_vsldb:
9650 case Intrinsic::s390_vperm:
9651 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 1);
9652 break;
9653 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9654 case Intrinsic::s390_vuplhh:
9655 case Intrinsic::s390_vuplhf:
9656 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9657 case Intrinsic::s390_vupllh:
9658 case Intrinsic::s390_vupllf:
9659 IsLogical = true;
9660 [[fallthrough]];
9661 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9662 case Intrinsic::s390_vuphh:
9663 case Intrinsic::s390_vuphf:
9664 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9665 case Intrinsic::s390_vuplhw:
9666 case Intrinsic::s390_vuplf: {
9667 SDValue SrcOp = Op.getOperand(i: 1);
9668 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 0);
9669 Known = DAG.computeKnownBits(Op: SrcOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9670 if (IsLogical) {
9671 Known = Known.zext(BitWidth);
9672 } else
9673 Known = Known.sext(BitWidth);
9674 break;
9675 }
9676 default:
9677 break;
9678 }
9679 } else {
9680 switch (Opcode) {
9681 case SystemZISD::JOIN_DWORDS:
9682 case SystemZISD::SELECT_CCMASK:
9683 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 0);
9684 break;
9685 case SystemZISD::REPLICATE: {
9686 SDValue SrcOp = Op.getOperand(i: 0);
9687 Known = DAG.computeKnownBits(Op: SrcOp, Depth: Depth + 1);
9688 if (Known.getBitWidth() < BitWidth && isa<ConstantSDNode>(Val: SrcOp))
9689 Known = Known.sext(BitWidth); // VREPI sign extends the immedate.
9690 break;
9691 }
9692 default:
9693 break;
9694 }
9695 }
9696
9697 // Known has the width of the source operand(s). Adjust if needed to match
9698 // the passed bitwidth.
9699 if (Known.getBitWidth() != BitWidth)
9700 Known = Known.anyextOrTrunc(BitWidth);
9701}
9702
9703static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts,
9704 const SelectionDAG &DAG, unsigned Depth,
9705 unsigned OpNo) {
9706 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9707 unsigned LHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9708 if (LHS == 1) return 1; // Early out.
9709 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9710 unsigned RHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9711 if (RHS == 1) return 1; // Early out.
9712 unsigned Common = std::min(a: LHS, b: RHS);
9713 unsigned SrcBitWidth = Op.getOperand(i: OpNo).getScalarValueSizeInBits();
9714 EVT VT = Op.getValueType();
9715 unsigned VTBits = VT.getScalarSizeInBits();
9716 if (SrcBitWidth > VTBits) { // PACK
9717 unsigned SrcExtraBits = SrcBitWidth - VTBits;
9718 if (Common > SrcExtraBits)
9719 return (Common - SrcExtraBits);
9720 return 1;
9721 }
9722 assert (SrcBitWidth == VTBits && "Expected operands of same bitwidth.");
9723 return Common;
9724}
9725
9726unsigned
9727SystemZTargetLowering::ComputeNumSignBitsForTargetNode(
9728 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9729 unsigned Depth) const {
9730 if (Op.getResNo() != 0)
9731 return 1;
9732 unsigned Opcode = Op.getOpcode();
9733 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9734 unsigned Id = Op.getConstantOperandVal(i: 0);
9735 switch (Id) {
9736 case Intrinsic::s390_vpksh: // PACKS
9737 case Intrinsic::s390_vpksf:
9738 case Intrinsic::s390_vpksg:
9739 case Intrinsic::s390_vpkshs: // PACKS_CC
9740 case Intrinsic::s390_vpksfs:
9741 case Intrinsic::s390_vpksgs:
9742 case Intrinsic::s390_vpklsh: // PACKLS
9743 case Intrinsic::s390_vpklsf:
9744 case Intrinsic::s390_vpklsg:
9745 case Intrinsic::s390_vpklshs: // PACKLS_CC
9746 case Intrinsic::s390_vpklsfs:
9747 case Intrinsic::s390_vpklsgs:
9748 case Intrinsic::s390_vpdi:
9749 case Intrinsic::s390_vsldb:
9750 case Intrinsic::s390_vperm:
9751 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 1);
9752 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9753 case Intrinsic::s390_vuphh:
9754 case Intrinsic::s390_vuphf:
9755 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9756 case Intrinsic::s390_vuplhw:
9757 case Intrinsic::s390_vuplf: {
9758 SDValue PackedOp = Op.getOperand(i: 1);
9759 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 1);
9760 unsigned Tmp = DAG.ComputeNumSignBits(Op: PackedOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9761 EVT VT = Op.getValueType();
9762 unsigned VTBits = VT.getScalarSizeInBits();
9763 Tmp += VTBits - PackedOp.getScalarValueSizeInBits();
9764 return Tmp;
9765 }
9766 default:
9767 break;
9768 }
9769 } else {
9770 switch (Opcode) {
9771 case SystemZISD::SELECT_CCMASK:
9772 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 0);
9773 default:
9774 break;
9775 }
9776 }
9777
9778 return 1;
9779}
9780
9781bool SystemZTargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
9782 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9783 UndefPoisonKind Kind, unsigned Depth) const {
9784 switch (Op->getOpcode()) {
9785 case SystemZISD::PCREL_WRAPPER:
9786 case SystemZISD::PCREL_OFFSET:
9787 return true;
9788 }
9789 return false;
9790}
9791
9792unsigned
9793SystemZTargetLowering::getStackProbeSize(const MachineFunction &MF) const {
9794 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
9795 unsigned StackAlign = TFI->getStackAlignment();
9796 assert(StackAlign >=1 && isPowerOf2_32(StackAlign) &&
9797 "Unexpected stack alignment");
9798 // The default stack probe size is 4096 if the function has no
9799 // stack-probe-size attribute.
9800 unsigned StackProbeSize =
9801 MF.getFunction().getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
9802 // Round down to the stack alignment.
9803 StackProbeSize &= ~(StackAlign - 1);
9804 return StackProbeSize ? StackProbeSize : StackAlign;
9805}
9806
9807//===----------------------------------------------------------------------===//
9808// Custom insertion
9809//===----------------------------------------------------------------------===//
9810
9811// Force base value Base into a register before MI. Return the register.
9812static Register forceReg(MachineInstr &MI, MachineOperand &Base,
9813 const SystemZInstrInfo *TII) {
9814 MachineBasicBlock *MBB = MI.getParent();
9815 MachineFunction &MF = *MBB->getParent();
9816 MachineRegisterInfo &MRI = MF.getRegInfo();
9817
9818 if (Base.isReg()) {
9819 // Copy Base into a new virtual register to help register coalescing in
9820 // cases with multiple uses.
9821 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9822 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::COPY), DestReg: Reg)
9823 .add(MO: Base);
9824 return Reg;
9825 }
9826
9827 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9828 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::LA), DestReg: Reg)
9829 .add(MO: Base)
9830 .addImm(Val: 0)
9831 .addReg(RegNo: 0);
9832 return Reg;
9833}
9834
9835// The CC operand of MI might be missing a kill marker because there
9836// were multiple uses of CC, and ISel didn't know which to mark.
9837// Figure out whether MI should have had a kill marker.
9838static bool checkCCKill(MachineInstr &MI, MachineBasicBlock *MBB) {
9839 // Scan forward through BB for a use/def of CC.
9840 MachineBasicBlock::iterator miI(std::next(x: MachineBasicBlock::iterator(MI)));
9841 for (MachineBasicBlock::iterator miE = MBB->end(); miI != miE; ++miI) {
9842 const MachineInstr &MI = *miI;
9843 if (MI.readsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9844 return false;
9845 if (MI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9846 break; // Should have kill-flag - update below.
9847 }
9848
9849 // If we hit the end of the block, check whether CC is live into a
9850 // successor.
9851 if (miI == MBB->end()) {
9852 for (const MachineBasicBlock *Succ : MBB->successors())
9853 if (Succ->isLiveIn(Reg: SystemZ::CC))
9854 return false;
9855 }
9856
9857 return true;
9858}
9859
9860// Return true if it is OK for this Select pseudo-opcode to be cascaded
9861// together with other Select pseudo-opcodes into a single basic-block with
9862// a conditional jump around it.
9863static bool isSelectPseudo(MachineInstr &MI) {
9864 switch (MI.getOpcode()) {
9865 case SystemZ::Select32:
9866 case SystemZ::Select64:
9867 case SystemZ::Select128:
9868 case SystemZ::SelectF32:
9869 case SystemZ::SelectF64:
9870 case SystemZ::SelectF128:
9871 case SystemZ::SelectVR32:
9872 case SystemZ::SelectVR64:
9873 case SystemZ::SelectVR128:
9874 return true;
9875
9876 default:
9877 return false;
9878 }
9879}
9880
9881// Helper function, which inserts PHI functions into SinkMBB:
9882// %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
9883// where %FalseValue(i) and %TrueValue(i) are taken from Selects.
9884static void createPHIsForSelects(SmallVector<MachineInstr*, 8> &Selects,
9885 MachineBasicBlock *TrueMBB,
9886 MachineBasicBlock *FalseMBB,
9887 MachineBasicBlock *SinkMBB) {
9888 MachineFunction *MF = TrueMBB->getParent();
9889 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
9890
9891 MachineInstr *FirstMI = Selects.front();
9892 unsigned CCValid = FirstMI->getOperand(i: 3).getImm();
9893 unsigned CCMask = FirstMI->getOperand(i: 4).getImm();
9894
9895 MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
9896
9897 // As we are creating the PHIs, we have to be careful if there is more than
9898 // one. Later Selects may reference the results of earlier Selects, but later
9899 // PHIs have to reference the individual true/false inputs from earlier PHIs.
9900 // That also means that PHI construction must work forward from earlier to
9901 // later, and that the code must maintain a mapping from earlier PHI's
9902 // destination registers, and the registers that went into the PHI.
9903 DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
9904
9905 for (auto *MI : Selects) {
9906 Register DestReg = MI->getOperand(i: 0).getReg();
9907 Register TrueReg = MI->getOperand(i: 1).getReg();
9908 Register FalseReg = MI->getOperand(i: 2).getReg();
9909
9910 // If this Select we are generating is the opposite condition from
9911 // the jump we generated, then we have to swap the operands for the
9912 // PHI that is going to be generated.
9913 if (MI->getOperand(i: 4).getImm() == (CCValid ^ CCMask))
9914 std::swap(a&: TrueReg, b&: FalseReg);
9915
9916 if (auto It = RegRewriteTable.find(Val: TrueReg); It != RegRewriteTable.end())
9917 TrueReg = It->second.first;
9918
9919 if (auto It = RegRewriteTable.find(Val: FalseReg); It != RegRewriteTable.end())
9920 FalseReg = It->second.second;
9921
9922 DebugLoc DL = MI->getDebugLoc();
9923 BuildMI(BB&: *SinkMBB, I: SinkInsertionPoint, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg)
9924 .addReg(RegNo: TrueReg).addMBB(MBB: TrueMBB)
9925 .addReg(RegNo: FalseReg).addMBB(MBB: FalseMBB);
9926
9927 // Add this PHI to the rewrite table.
9928 RegRewriteTable[DestReg] = std::make_pair(x&: TrueReg, y&: FalseReg);
9929 }
9930
9931 MF->getProperties().resetNoPHIs();
9932}
9933
9934MachineBasicBlock *
9935SystemZTargetLowering::emitAdjCallStack(MachineInstr &MI,
9936 MachineBasicBlock *BB) const {
9937 MachineFunction &MF = *BB->getParent();
9938 MachineFrameInfo &MFI = MF.getFrameInfo();
9939 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
9940 assert(TFL->hasReservedCallFrame(MF) &&
9941 "ADJSTACKDOWN and ADJSTACKUP should be no-ops");
9942 (void)TFL;
9943 // Get the MaxCallFrameSize value and erase MI since it serves no further
9944 // purpose as the call frame is statically reserved in the prolog. Set
9945 // AdjustsStack as MI is *not* mapped as a frame instruction.
9946 uint32_t NumBytes = MI.getOperand(i: 0).getImm();
9947 if (NumBytes > MFI.getMaxCallFrameSize())
9948 MFI.setMaxCallFrameSize(NumBytes);
9949 MFI.setAdjustsStack(true);
9950
9951 MI.eraseFromParent();
9952 return BB;
9953}
9954
9955// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI.
9956MachineBasicBlock *
9957SystemZTargetLowering::emitSelect(MachineInstr &MI,
9958 MachineBasicBlock *MBB) const {
9959 assert(isSelectPseudo(MI) && "Bad call to emitSelect()");
9960 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9961
9962 unsigned CCValid = MI.getOperand(i: 3).getImm();
9963 unsigned CCMask = MI.getOperand(i: 4).getImm();
9964
9965 // If we have a sequence of Select* pseudo instructions using the
9966 // same condition code value, we want to expand all of them into
9967 // a single pair of basic blocks using the same condition.
9968 SmallVector<MachineInstr*, 8> Selects;
9969 SmallVector<MachineInstr*, 8> DbgValues;
9970 Selects.push_back(Elt: &MI);
9971 unsigned Count = 0;
9972 for (MachineInstr &NextMI : llvm::make_range(
9973 x: std::next(x: MachineBasicBlock::iterator(MI)), y: MBB->end())) {
9974 if (isSelectPseudo(MI&: NextMI)) {
9975 assert(NextMI.getOperand(3).getImm() == CCValid &&
9976 "Bad CCValid operands since CC was not redefined.");
9977 if (NextMI.getOperand(i: 4).getImm() == CCMask ||
9978 NextMI.getOperand(i: 4).getImm() == (CCValid ^ CCMask)) {
9979 Selects.push_back(Elt: &NextMI);
9980 continue;
9981 }
9982 break;
9983 }
9984 if (NextMI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
9985 NextMI.usesCustomInsertionHook())
9986 break;
9987 bool User = false;
9988 for (auto *SelMI : Selects)
9989 if (NextMI.readsVirtualRegister(Reg: SelMI->getOperand(i: 0).getReg())) {
9990 User = true;
9991 break;
9992 }
9993 if (NextMI.isDebugInstr()) {
9994 if (User) {
9995 assert(NextMI.isDebugValue() && "Unhandled debug opcode.");
9996 DbgValues.push_back(Elt: &NextMI);
9997 }
9998 } else if (User || ++Count > 20)
9999 break;
10000 }
10001
10002 MachineInstr *LastMI = Selects.back();
10003 bool CCKilled = (LastMI->killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
10004 checkCCKill(MI&: *LastMI, MBB));
10005 MachineBasicBlock *StartMBB = MBB;
10006 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI: LastMI, MBB);
10007 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10008
10009 // Unless CC was killed in the last Select instruction, mark it as
10010 // live-in to both FalseMBB and JoinMBB.
10011 if (!CCKilled) {
10012 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
10013 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10014 }
10015
10016 // StartMBB:
10017 // BRC CCMask, JoinMBB
10018 // # fallthrough to FalseMBB
10019 MBB = StartMBB;
10020 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
10021 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
10022 MBB->addSuccessor(Succ: JoinMBB);
10023 MBB->addSuccessor(Succ: FalseMBB);
10024
10025 // FalseMBB:
10026 // # fallthrough to JoinMBB
10027 MBB = FalseMBB;
10028 MBB->addSuccessor(Succ: JoinMBB);
10029
10030 // JoinMBB:
10031 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ]
10032 // ...
10033 MBB = JoinMBB;
10034 createPHIsForSelects(Selects, TrueMBB: StartMBB, FalseMBB, SinkMBB: MBB);
10035 for (auto *SelMI : Selects)
10036 SelMI->eraseFromParent();
10037
10038 MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();
10039 for (auto *DbgMI : DbgValues)
10040 MBB->splice(Where: InsertPos, Other: StartMBB, From: DbgMI);
10041
10042 return JoinMBB;
10043}
10044
10045// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI.
10046// StoreOpcode is the store to use and Invert says whether the store should
10047// happen when the condition is false rather than true. If a STORE ON
10048// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0.
10049MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI,
10050 MachineBasicBlock *MBB,
10051 unsigned StoreOpcode,
10052 unsigned STOCOpcode,
10053 bool Invert) const {
10054 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10055
10056 Register SrcReg = MI.getOperand(i: 0).getReg();
10057 MachineOperand Base = MI.getOperand(i: 1);
10058 int64_t Disp = MI.getOperand(i: 2).getImm();
10059 Register IndexReg = MI.getOperand(i: 3).getReg();
10060 unsigned CCValid = MI.getOperand(i: 4).getImm();
10061 unsigned CCMask = MI.getOperand(i: 5).getImm();
10062 DebugLoc DL = MI.getDebugLoc();
10063
10064 StoreOpcode = TII->getOpcodeForOffset(Opcode: StoreOpcode, Offset: Disp);
10065
10066 // ISel pattern matching also adds a load memory operand of the same
10067 // address, so take special care to find the storing memory operand.
10068 MachineMemOperand *MMO = nullptr;
10069 for (auto *I : MI.memoperands())
10070 if (I->isStore()) {
10071 MMO = I;
10072 break;
10073 }
10074
10075 // Use STOCOpcode if possible. We could use different store patterns in
10076 // order to avoid matching the index register, but the performance trade-offs
10077 // might be more complicated in that case.
10078 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
10079 if (Invert)
10080 CCMask ^= CCValid;
10081
10082 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: STOCOpcode))
10083 .addReg(RegNo: SrcReg)
10084 .add(MO: Base)
10085 .addImm(Val: Disp)
10086 .addImm(Val: CCValid)
10087 .addImm(Val: CCMask)
10088 .addMemOperand(MMO);
10089
10090 MI.eraseFromParent();
10091 return MBB;
10092 }
10093
10094 // Get the condition needed to branch around the store.
10095 if (!Invert)
10096 CCMask ^= CCValid;
10097
10098 MachineBasicBlock *StartMBB = MBB;
10099 MachineBasicBlock *JoinMBB = SystemZ::splitBlockBefore(MI, MBB);
10100 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10101
10102 // Unless CC was killed in the CondStore instruction, mark it as
10103 // live-in to both FalseMBB and JoinMBB.
10104 if (!MI.killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) &&
10105 !checkCCKill(MI, MBB: JoinMBB)) {
10106 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
10107 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10108 }
10109
10110 // StartMBB:
10111 // BRC CCMask, JoinMBB
10112 // # fallthrough to FalseMBB
10113 MBB = StartMBB;
10114 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10115 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
10116 MBB->addSuccessor(Succ: JoinMBB);
10117 MBB->addSuccessor(Succ: FalseMBB);
10118
10119 // FalseMBB:
10120 // store %SrcReg, %Disp(%Index,%Base)
10121 // # fallthrough to JoinMBB
10122 MBB = FalseMBB;
10123 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: StoreOpcode))
10124 .addReg(RegNo: SrcReg)
10125 .add(MO: Base)
10126 .addImm(Val: Disp)
10127 .addReg(RegNo: IndexReg)
10128 .addMemOperand(MMO);
10129 MBB->addSuccessor(Succ: JoinMBB);
10130
10131 MI.eraseFromParent();
10132 return JoinMBB;
10133}
10134
10135// Implement EmitInstrWithCustomInserter for pseudo [SU]Cmp128Hi instruction MI.
10136MachineBasicBlock *
10137SystemZTargetLowering::emitICmp128Hi(MachineInstr &MI,
10138 MachineBasicBlock *MBB,
10139 bool Unsigned) const {
10140 MachineFunction &MF = *MBB->getParent();
10141 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10142 MachineRegisterInfo &MRI = MF.getRegInfo();
10143
10144 // Synthetic instruction to compare 128-bit values.
10145 // Sets CC 1 if Op0 > Op1, sets a different CC otherwise.
10146 Register Op0 = MI.getOperand(i: 0).getReg();
10147 Register Op1 = MI.getOperand(i: 1).getReg();
10148
10149 MachineBasicBlock *StartMBB = MBB;
10150 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI, MBB);
10151 MachineBasicBlock *HiEqMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10152
10153 // StartMBB:
10154 //
10155 // Use VECTOR ELEMENT COMPARE [LOGICAL] to compare the high parts.
10156 // Swap the inputs to get:
10157 // CC 1 if high(Op0) > high(Op1)
10158 // CC 2 if high(Op0) < high(Op1)
10159 // CC 0 if high(Op0) == high(Op1)
10160 //
10161 // If CC != 0, we'd done, so jump over the next instruction.
10162 //
10163 // VEC[L]G Op1, Op0
10164 // JNE JoinMBB
10165 // # fallthrough to HiEqMBB
10166 MBB = StartMBB;
10167 int HiOpcode = Unsigned? SystemZ::VECLG : SystemZ::VECG;
10168 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: HiOpcode))
10169 .addReg(RegNo: Op1).addReg(RegNo: Op0);
10170 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
10171 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: JoinMBB);
10172 MBB->addSuccessor(Succ: JoinMBB);
10173 MBB->addSuccessor(Succ: HiEqMBB);
10174
10175 // HiEqMBB:
10176 //
10177 // Otherwise, use VECTOR COMPARE HIGH LOGICAL.
10178 // Since we already know the high parts are equal, the CC
10179 // result will only depend on the low parts:
10180 // CC 1 if low(Op0) > low(Op1)
10181 // CC 3 if low(Op0) <= low(Op1)
10182 //
10183 // VCHLGS Tmp, Op0, Op1
10184 // # fallthrough to JoinMBB
10185 MBB = HiEqMBB;
10186 Register Temp = MRI.createVirtualRegister(RegClass: &SystemZ::VR128BitRegClass);
10187 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::VCHLGS), DestReg: Temp)
10188 .addReg(RegNo: Op0).addReg(RegNo: Op1);
10189 MBB->addSuccessor(Succ: JoinMBB);
10190
10191 // Mark CC as live-in to JoinMBB.
10192 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10193
10194 MI.eraseFromParent();
10195 return JoinMBB;
10196}
10197
10198// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_LOADW_* or
10199// ATOMIC_SWAPW instruction MI. BinOpcode is the instruction that performs
10200// the binary operation elided by "*", or 0 for ATOMIC_SWAPW. Invert says
10201// whether the field should be inverted after performing BinOpcode (e.g. for
10202// NAND).
10203MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
10204 MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode,
10205 bool Invert) const {
10206 MachineFunction &MF = *MBB->getParent();
10207 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10208 MachineRegisterInfo &MRI = MF.getRegInfo();
10209
10210 // Extract the operands. Base can be a register or a frame index.
10211 // Src2 can be a register or immediate.
10212 Register Dest = MI.getOperand(i: 0).getReg();
10213 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10214 int64_t Disp = MI.getOperand(i: 2).getImm();
10215 MachineOperand Src2 = earlyUseOperand(Op: MI.getOperand(i: 3));
10216 Register BitShift = MI.getOperand(i: 4).getReg();
10217 Register NegBitShift = MI.getOperand(i: 5).getReg();
10218 unsigned BitSize = MI.getOperand(i: 6).getImm();
10219 DebugLoc DL = MI.getDebugLoc();
10220
10221 // Get the right opcodes for the displacement.
10222 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10223 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10224 assert(LOpcode && CSOpcode && "Displacement out of range");
10225
10226 // Create virtual registers for temporary results.
10227 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10228 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10229 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10230 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10231 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10232
10233 // Insert a basic block for the main loop.
10234 MachineBasicBlock *StartMBB = MBB;
10235 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10236 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10237
10238 // StartMBB:
10239 // ...
10240 // %OrigVal = L Disp(%Base)
10241 // # fall through to LoopMBB
10242 MBB = StartMBB;
10243 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10244 MBB->addSuccessor(Succ: LoopMBB);
10245
10246 // LoopMBB:
10247 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ]
10248 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10249 // %RotatedNewVal = OP %RotatedOldVal, %Src2
10250 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10251 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10252 // JNE LoopMBB
10253 // # fall through to DoneMBB
10254 MBB = LoopMBB;
10255 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10256 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
10257 .addReg(RegNo: Dest).addMBB(MBB: LoopMBB);
10258 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
10259 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
10260 if (Invert) {
10261 // Perform the operation normally and then invert every bit of the field.
10262 Register Tmp = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10263 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: Tmp)
10264 .addReg(RegNo: RotatedOldVal)
10265 .add(MO: Src2)
10266 .setOperandDead(3);
10267 // XILF with the upper BitSize bits set.
10268 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::XILF), DestReg: RotatedNewVal)
10269 .addReg(RegNo: Tmp)
10270 .addImm(Val: -1U << (32 - BitSize))
10271 .setOperandDead(3);
10272 } else if (BinOpcode)
10273 // A simply binary operation.
10274 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: RotatedNewVal)
10275 .addReg(RegNo: RotatedOldVal)
10276 .add(MO: Src2)
10277 .setOperandDead(3);
10278 else
10279 // Use RISBG to rotate Src2 into position and use it to replace the
10280 // field in RotatedOldVal.
10281 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedNewVal)
10282 .addReg(RegNo: RotatedOldVal)
10283 .addReg(RegNo: Src2.getReg())
10284 .addImm(Val: 32)
10285 .addImm(Val: 31 + BitSize)
10286 .addImm(Val: 32 - BitSize)
10287 .setOperandDead(6);
10288 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
10289 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
10290 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
10291 .addReg(RegNo: OldVal)
10292 .addReg(RegNo: NewVal)
10293 .add(MO: Base)
10294 .addImm(Val: Disp);
10295 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10296 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10297 MBB->addSuccessor(Succ: LoopMBB);
10298 MBB->addSuccessor(Succ: DoneMBB);
10299
10300 MI.eraseFromParent();
10301 return DoneMBB;
10302}
10303
10304// Implement EmitInstrWithCustomInserter for subword pseudo
10305// ATOMIC_LOADW_{,U}{MIN,MAX} instruction MI. CompareOpcode is the
10306// instruction that should be used to compare the current field with the
10307// minimum or maximum value. KeepOldMask is the BRC condition-code mask
10308// for when the current field should be kept.
10309MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
10310 MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode,
10311 unsigned KeepOldMask) const {
10312 MachineFunction &MF = *MBB->getParent();
10313 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10314 MachineRegisterInfo &MRI = MF.getRegInfo();
10315
10316 // Extract the operands. Base can be a register or a frame index.
10317 Register Dest = MI.getOperand(i: 0).getReg();
10318 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10319 int64_t Disp = MI.getOperand(i: 2).getImm();
10320 Register Src2 = MI.getOperand(i: 3).getReg();
10321 Register BitShift = MI.getOperand(i: 4).getReg();
10322 Register NegBitShift = MI.getOperand(i: 5).getReg();
10323 unsigned BitSize = MI.getOperand(i: 6).getImm();
10324 DebugLoc DL = MI.getDebugLoc();
10325
10326 // Get the right opcodes for the displacement.
10327 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10328 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10329 assert(LOpcode && CSOpcode && "Displacement out of range");
10330
10331 // Create virtual registers for temporary results.
10332 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10333 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10334 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10335 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10336 Register RotatedAltVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10337 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10338
10339 // Insert 3 basic blocks for the loop.
10340 MachineBasicBlock *StartMBB = MBB;
10341 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10342 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10343 MachineBasicBlock *UseAltMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10344 MachineBasicBlock *UpdateMBB = SystemZ::emitBlockAfter(MBB: UseAltMBB);
10345
10346 // StartMBB:
10347 // ...
10348 // %OrigVal = L Disp(%Base)
10349 // # fall through to LoopMBB
10350 MBB = StartMBB;
10351 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10352 MBB->addSuccessor(Succ: LoopMBB);
10353
10354 // LoopMBB:
10355 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ]
10356 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10357 // CompareOpcode %RotatedOldVal, %Src2
10358 // BRC KeepOldMask, UpdateMBB
10359 MBB = LoopMBB;
10360 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10361 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
10362 .addReg(RegNo: Dest).addMBB(MBB: UpdateMBB);
10363 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
10364 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
10365 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CompareOpcode))
10366 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2);
10367 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10368 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: KeepOldMask).addMBB(MBB: UpdateMBB);
10369 MBB->addSuccessor(Succ: UpdateMBB);
10370 MBB->addSuccessor(Succ: UseAltMBB);
10371
10372 // UseAltMBB:
10373 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0
10374 // # fall through to UpdateMBB
10375 MBB = UseAltMBB;
10376 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedAltVal)
10377 .addReg(RegNo: RotatedOldVal)
10378 .addReg(RegNo: Src2)
10379 .addImm(Val: 32)
10380 .addImm(Val: 31 + BitSize)
10381 .addImm(Val: 0)
10382 .setOperandDead(6);
10383 MBB->addSuccessor(Succ: UpdateMBB);
10384
10385 // UpdateMBB:
10386 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ],
10387 // [ %RotatedAltVal, UseAltMBB ]
10388 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10389 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10390 // JNE LoopMBB
10391 // # fall through to DoneMBB
10392 MBB = UpdateMBB;
10393 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RotatedNewVal)
10394 .addReg(RegNo: RotatedOldVal).addMBB(MBB: LoopMBB)
10395 .addReg(RegNo: RotatedAltVal).addMBB(MBB: UseAltMBB);
10396 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
10397 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
10398 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
10399 .addReg(RegNo: OldVal)
10400 .addReg(RegNo: NewVal)
10401 .add(MO: Base)
10402 .addImm(Val: Disp);
10403 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10404 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10405 MBB->addSuccessor(Succ: LoopMBB);
10406 MBB->addSuccessor(Succ: DoneMBB);
10407
10408 MI.eraseFromParent();
10409 return DoneMBB;
10410}
10411
10412// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_CMP_SWAPW
10413// instruction MI.
10414MachineBasicBlock *
10415SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
10416 MachineBasicBlock *MBB) const {
10417 MachineFunction &MF = *MBB->getParent();
10418 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10419 MachineRegisterInfo &MRI = MF.getRegInfo();
10420
10421 // Extract the operands. Base can be a register or a frame index.
10422 Register Dest = MI.getOperand(i: 0).getReg();
10423 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10424 int64_t Disp = MI.getOperand(i: 2).getImm();
10425 Register CmpVal = MI.getOperand(i: 3).getReg();
10426 Register OrigSwapVal = MI.getOperand(i: 4).getReg();
10427 Register BitShift = MI.getOperand(i: 5).getReg();
10428 Register NegBitShift = MI.getOperand(i: 6).getReg();
10429 int64_t BitSize = MI.getOperand(i: 7).getImm();
10430 DebugLoc DL = MI.getDebugLoc();
10431
10432 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
10433
10434 // Get the right opcodes for the displacement and zero-extension.
10435 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10436 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10437 unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
10438 assert(LOpcode && CSOpcode && "Displacement out of range");
10439
10440 // Create virtual registers for temporary results.
10441 Register OrigOldVal = MRI.createVirtualRegister(RegClass: RC);
10442 Register OldVal = MRI.createVirtualRegister(RegClass: RC);
10443 Register SwapVal = MRI.createVirtualRegister(RegClass: RC);
10444 Register StoreVal = MRI.createVirtualRegister(RegClass: RC);
10445 Register OldValRot = MRI.createVirtualRegister(RegClass: RC);
10446 Register RetryOldVal = MRI.createVirtualRegister(RegClass: RC);
10447 Register RetrySwapVal = MRI.createVirtualRegister(RegClass: RC);
10448
10449 // Insert 2 basic blocks for the loop.
10450 MachineBasicBlock *StartMBB = MBB;
10451 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10452 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10453 MachineBasicBlock *SetMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10454
10455 // StartMBB:
10456 // ...
10457 // %OrigOldVal = L Disp(%Base)
10458 // # fall through to LoopMBB
10459 MBB = StartMBB;
10460 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigOldVal)
10461 .add(MO: Base)
10462 .addImm(Val: Disp)
10463 .addReg(RegNo: 0);
10464 MBB->addSuccessor(Succ: LoopMBB);
10465
10466 // LoopMBB:
10467 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
10468 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
10469 // %OldValRot = RLL %OldVal, BitSize(%BitShift)
10470 // ^^ The low BitSize bits contain the field
10471 // of interest.
10472 // %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
10473 // ^^ Replace the upper 32-BitSize bits of the
10474 // swap value with those that we loaded and rotated.
10475 // %Dest = LL[CH] %OldValRot
10476 // CR %Dest, %CmpVal
10477 // JNE DoneMBB
10478 // # Fall through to SetMBB
10479 MBB = LoopMBB;
10480 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10481 .addReg(RegNo: OrigOldVal).addMBB(MBB: StartMBB)
10482 .addReg(RegNo: RetryOldVal).addMBB(MBB: SetMBB);
10483 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: SwapVal)
10484 .addReg(RegNo: OrigSwapVal).addMBB(MBB: StartMBB)
10485 .addReg(RegNo: RetrySwapVal).addMBB(MBB: SetMBB);
10486 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: OldValRot)
10487 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: BitSize);
10488 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RetrySwapVal)
10489 .addReg(RegNo: SwapVal)
10490 .addReg(RegNo: OldValRot)
10491 .addImm(Val: 32)
10492 .addImm(Val: 63 - BitSize)
10493 .addImm(Val: 0)
10494 .setOperandDead(6);
10495 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: ZExtOpcode), DestReg: Dest)
10496 .addReg(RegNo: OldValRot);
10497 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CR))
10498 .addReg(RegNo: Dest).addReg(RegNo: CmpVal);
10499 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10500 .addImm(Val: SystemZ::CCMASK_ICMP)
10501 .addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: DoneMBB);
10502 MBB->addSuccessor(Succ: DoneMBB);
10503 MBB->addSuccessor(Succ: SetMBB);
10504
10505 // SetMBB:
10506 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
10507 // ^^ Rotate the new field to its proper position.
10508 // %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
10509 // JNE LoopMBB
10510 // # fall through to ExitMBB
10511 MBB = SetMBB;
10512 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: StoreVal)
10513 .addReg(RegNo: RetrySwapVal).addReg(RegNo: NegBitShift).addImm(Val: -BitSize);
10514 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: RetryOldVal)
10515 .addReg(RegNo: OldVal)
10516 .addReg(RegNo: StoreVal)
10517 .add(MO: Base)
10518 .addImm(Val: Disp);
10519 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10520 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10521 MBB->addSuccessor(Succ: LoopMBB);
10522 MBB->addSuccessor(Succ: DoneMBB);
10523
10524 // If the CC def wasn't dead in the ATOMIC_CMP_SWAPW, mark CC as live-in
10525 // to the block after the loop. At this point, CC may have been defined
10526 // either by the CR in LoopMBB or by the CS in SetMBB.
10527 if (!MI.registerDefIsDead(Reg: SystemZ::CC, /*TRI=*/nullptr))
10528 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10529
10530 MI.eraseFromParent();
10531 return DoneMBB;
10532}
10533
10534// Emit a move from two GR64s to a GR128.
10535MachineBasicBlock *
10536SystemZTargetLowering::emitPair128(MachineInstr &MI,
10537 MachineBasicBlock *MBB) const {
10538 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10539 const DebugLoc &DL = MI.getDebugLoc();
10540
10541 Register Dest = MI.getOperand(i: 0).getReg();
10542 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest)
10543 .add(MO: MI.getOperand(i: 1))
10544 .addImm(Val: SystemZ::subreg_h64)
10545 .add(MO: MI.getOperand(i: 2))
10546 .addImm(Val: SystemZ::subreg_l64);
10547 MI.eraseFromParent();
10548 return MBB;
10549}
10550
10551// Emit an extension from a GR64 to a GR128. ClearEven is true
10552// if the high register of the GR128 value must be cleared or false if
10553// it's "don't care".
10554MachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI,
10555 MachineBasicBlock *MBB,
10556 bool ClearEven) const {
10557 MachineFunction &MF = *MBB->getParent();
10558 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10559 MachineRegisterInfo &MRI = MF.getRegInfo();
10560 DebugLoc DL = MI.getDebugLoc();
10561
10562 Register Dest = MI.getOperand(i: 0).getReg();
10563 Register Src = MI.getOperand(i: 1).getReg();
10564 Register In128 = MRI.createVirtualRegister(RegClass: &SystemZ::GR128BitRegClass);
10565
10566 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: In128);
10567 if (ClearEven) {
10568 Register NewIn128 = MRI.createVirtualRegister(RegClass: &SystemZ::GR128BitRegClass);
10569 Register Zero64 = MRI.createVirtualRegister(RegClass: &SystemZ::GR64BitRegClass);
10570
10571 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LLILL), DestReg: Zero64)
10572 .addImm(Val: 0);
10573 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: NewIn128)
10574 .addReg(RegNo: In128).addReg(RegNo: Zero64).addImm(Val: SystemZ::subreg_h64);
10575 In128 = NewIn128;
10576 }
10577 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Dest)
10578 .addReg(RegNo: In128).addReg(RegNo: Src).addImm(Val: SystemZ::subreg_l64);
10579
10580 MI.eraseFromParent();
10581 return MBB;
10582}
10583
10584MachineBasicBlock *
10585SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
10586 MachineBasicBlock *MBB,
10587 unsigned Opcode, bool IsMemset) const {
10588 MachineFunction &MF = *MBB->getParent();
10589 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10590 MachineRegisterInfo &MRI = MF.getRegInfo();
10591 DebugLoc DL = MI.getDebugLoc();
10592
10593 MachineOperand DestBase = earlyUseOperand(Op: MI.getOperand(i: 0));
10594 uint64_t DestDisp = MI.getOperand(i: 1).getImm();
10595 MachineOperand SrcBase = MachineOperand::CreateReg(Reg: 0U, isDef: false);
10596 uint64_t SrcDisp;
10597
10598 // Fold the displacement Disp if it is out of range.
10599 auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
10600 if (!isUInt<12>(x: Disp)) {
10601 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10602 unsigned Opcode = TII->getOpcodeForOffset(Opcode: SystemZ::LA, Offset: Disp);
10603 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode), DestReg: Reg)
10604 .add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10605 Base = MachineOperand::CreateReg(Reg, isDef: false);
10606 Disp = 0;
10607 }
10608 };
10609
10610 if (!IsMemset) {
10611 SrcBase = earlyUseOperand(Op: MI.getOperand(i: 2));
10612 SrcDisp = MI.getOperand(i: 3).getImm();
10613 } else {
10614 SrcBase = DestBase;
10615 SrcDisp = DestDisp++;
10616 foldDisplIfNeeded(DestBase, DestDisp);
10617 }
10618
10619 MachineOperand &LengthMO = MI.getOperand(i: IsMemset ? 2 : 4);
10620 bool IsImmForm = LengthMO.isImm();
10621 bool IsRegForm = !IsImmForm;
10622
10623 // Build and insert one Opcode of Length, with special treatment for memset.
10624 auto insertMemMemOp = [&](MachineBasicBlock *InsMBB,
10625 MachineBasicBlock::iterator InsPos,
10626 MachineOperand DBase, uint64_t DDisp,
10627 MachineOperand SBase, uint64_t SDisp,
10628 unsigned Length) -> void {
10629 assert(Length > 0 && Length <= 256 && "Building memory op with bad length.");
10630 if (IsMemset) {
10631 MachineOperand ByteMO = earlyUseOperand(Op: MI.getOperand(i: 3));
10632 if (ByteMO.isImm())
10633 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVI))
10634 .add(MO: SBase).addImm(Val: SDisp).add(MO: ByteMO);
10635 else
10636 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STC))
10637 .add(MO: ByteMO).add(MO: SBase).addImm(Val: SDisp).addReg(RegNo: 0);
10638 if (--Length == 0)
10639 return;
10640 }
10641 BuildMI(BB&: *MBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode))
10642 .add(MO: DBase).addImm(Val: DDisp).addImm(Val: Length)
10643 .add(MO: SBase).addImm(Val: SDisp)
10644 .setMemRefs(MI.memoperands());
10645 };
10646
10647 bool NeedsLoop = false;
10648 uint64_t ImmLength = 0;
10649 Register LenAdjReg = SystemZ::NoRegister;
10650 if (IsImmForm) {
10651 ImmLength = LengthMO.getImm();
10652 ImmLength += IsMemset ? 2 : 1; // Add back the subtracted adjustment.
10653 if (ImmLength == 0) {
10654 MI.eraseFromParent();
10655 return MBB;
10656 }
10657 if (Opcode == SystemZ::CLC) {
10658 if (ImmLength > 3 * 256)
10659 // A two-CLC sequence is a clear win over a loop, not least because
10660 // it needs only one branch. A three-CLC sequence needs the same
10661 // number of branches as a loop (i.e. 2), but is shorter. That
10662 // brings us to lengths greater than 768 bytes. It seems relatively
10663 // likely that a difference will be found within the first 768 bytes,
10664 // so we just optimize for the smallest number of branch
10665 // instructions, in order to avoid polluting the prediction buffer
10666 // too much.
10667 NeedsLoop = true;
10668 } else if (ImmLength > 6 * 256)
10669 // The heuristic we use is to prefer loops for anything that would
10670 // require 7 or more MVCs. With these kinds of sizes there isn't much
10671 // to choose between straight-line code and looping code, since the
10672 // time will be dominated by the MVCs themselves.
10673 NeedsLoop = true;
10674 } else {
10675 NeedsLoop = true;
10676 LenAdjReg = LengthMO.getReg();
10677 }
10678
10679 // When generating more than one CLC, all but the last will need to
10680 // branch to the end when a difference is found.
10681 MachineBasicBlock *EndMBB =
10682 (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
10683 ? SystemZ::splitBlockAfter(MI, MBB)
10684 : nullptr);
10685
10686 if (NeedsLoop) {
10687 Register StartCountReg =
10688 MRI.createVirtualRegister(RegClass: &SystemZ::GR64BitRegClass);
10689 if (IsImmForm) {
10690 TII->loadImmediate(MBB&: *MBB, MBBI: MI, Reg: StartCountReg, Value: ImmLength / 256);
10691 ImmLength &= 255;
10692 } else {
10693 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SRLG), DestReg: StartCountReg)
10694 .addReg(RegNo: LenAdjReg)
10695 .addReg(RegNo: 0)
10696 .addImm(Val: 8);
10697 }
10698
10699 bool HaveSingleBase = DestBase.isIdenticalTo(Other: SrcBase);
10700 auto loadZeroAddress = [&]() -> MachineOperand {
10701 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10702 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LGHI), DestReg: Reg).addImm(Val: 0);
10703 return MachineOperand::CreateReg(Reg, isDef: false);
10704 };
10705 if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
10706 DestBase = loadZeroAddress();
10707 if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
10708 SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
10709
10710 MachineBasicBlock *StartMBB = nullptr;
10711 MachineBasicBlock *LoopMBB = nullptr;
10712 MachineBasicBlock *NextMBB = nullptr;
10713 MachineBasicBlock *DoneMBB = nullptr;
10714 MachineBasicBlock *AllDoneMBB = nullptr;
10715
10716 Register StartSrcReg = forceReg(MI, Base&: SrcBase, TII);
10717 Register StartDestReg =
10718 (HaveSingleBase ? StartSrcReg : forceReg(MI, Base&: DestBase, TII));
10719
10720 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
10721 Register ThisSrcReg = MRI.createVirtualRegister(RegClass: RC);
10722 Register ThisDestReg =
10723 (HaveSingleBase ? ThisSrcReg : MRI.createVirtualRegister(RegClass: RC));
10724 Register NextSrcReg = MRI.createVirtualRegister(RegClass: RC);
10725 Register NextDestReg =
10726 (HaveSingleBase ? NextSrcReg : MRI.createVirtualRegister(RegClass: RC));
10727 RC = &SystemZ::GR64BitRegClass;
10728 Register ThisCountReg = MRI.createVirtualRegister(RegClass: RC);
10729 Register NextCountReg = MRI.createVirtualRegister(RegClass: RC);
10730
10731 if (IsRegForm) {
10732 AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10733 StartMBB = SystemZ::emitBlockAfter(MBB);
10734 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10735 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10736 DoneMBB = SystemZ::emitBlockAfter(MBB: NextMBB);
10737
10738 // MBB:
10739 // # Jump to AllDoneMBB if LenAdjReg means 0, or fall thru to StartMBB.
10740 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10741 .addReg(RegNo: LenAdjReg).addImm(Val: IsMemset ? -2 : -1);
10742 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10743 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10744 .addMBB(MBB: AllDoneMBB);
10745 MBB->addSuccessor(Succ: AllDoneMBB);
10746 if (!IsMemset)
10747 MBB->addSuccessor(Succ: StartMBB);
10748 else {
10749 // MemsetOneCheckMBB:
10750 // # Jump to MemsetOneMBB for a memset of length 1, or
10751 // # fall thru to StartMBB.
10752 MachineBasicBlock *MemsetOneCheckMBB = SystemZ::emitBlockAfter(MBB);
10753 MachineBasicBlock *MemsetOneMBB = SystemZ::emitBlockAfter(MBB: &*MF.rbegin());
10754 MBB->addSuccessor(Succ: MemsetOneCheckMBB);
10755 MBB = MemsetOneCheckMBB;
10756 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10757 .addReg(RegNo: LenAdjReg).addImm(Val: -1);
10758 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10759 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10760 .addMBB(MBB: MemsetOneMBB);
10761 MBB->addSuccessor(Succ: MemsetOneMBB, Prob: {10, 100});
10762 MBB->addSuccessor(Succ: StartMBB, Prob: {90, 100});
10763
10764 // MemsetOneMBB:
10765 // # Jump back to AllDoneMBB after a single MVI or STC.
10766 MBB = MemsetOneMBB;
10767 insertMemMemOp(MBB, MBB->end(),
10768 MachineOperand::CreateReg(Reg: StartDestReg, isDef: false), DestDisp,
10769 MachineOperand::CreateReg(Reg: StartSrcReg, isDef: false), SrcDisp,
10770 1);
10771 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: AllDoneMBB);
10772 MBB->addSuccessor(Succ: AllDoneMBB);
10773 }
10774
10775 // StartMBB:
10776 // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB.
10777 MBB = StartMBB;
10778 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10779 .addReg(RegNo: StartCountReg).addImm(Val: 0);
10780 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10781 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10782 .addMBB(MBB: DoneMBB);
10783 MBB->addSuccessor(Succ: DoneMBB);
10784 MBB->addSuccessor(Succ: LoopMBB);
10785 }
10786 else {
10787 StartMBB = MBB;
10788 DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10789 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10790 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10791
10792 // StartMBB:
10793 // # fall through to LoopMBB
10794 MBB->addSuccessor(Succ: LoopMBB);
10795
10796 DestBase = MachineOperand::CreateReg(Reg: NextDestReg, isDef: false);
10797 SrcBase = MachineOperand::CreateReg(Reg: NextSrcReg, isDef: false);
10798 if (EndMBB && !ImmLength)
10799 // If the loop handled the whole CLC range, DoneMBB will be empty with
10800 // CC live-through into EndMBB, so add it as live-in.
10801 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10802 }
10803
10804 // LoopMBB:
10805 // %ThisDestReg = phi [ %StartDestReg, StartMBB ],
10806 // [ %NextDestReg, NextMBB ]
10807 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ],
10808 // [ %NextSrcReg, NextMBB ]
10809 // %ThisCountReg = phi [ %StartCountReg, StartMBB ],
10810 // [ %NextCountReg, NextMBB ]
10811 // ( PFD 2, 768+DestDisp(%ThisDestReg) )
10812 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg)
10813 // ( JLH EndMBB )
10814 //
10815 // The prefetch is used only for MVC. The JLH is used only for CLC.
10816 MBB = LoopMBB;
10817 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisDestReg)
10818 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10819 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10820 if (!HaveSingleBase)
10821 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisSrcReg)
10822 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10823 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10824 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisCountReg)
10825 .addReg(RegNo: StartCountReg).addMBB(MBB: StartMBB)
10826 .addReg(RegNo: NextCountReg).addMBB(MBB: NextMBB);
10827 if (Opcode == SystemZ::MVC)
10828 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PFD))
10829 .addImm(Val: SystemZ::PFD_WRITE)
10830 .addReg(RegNo: ThisDestReg).addImm(Val: DestDisp - IsMemset + 768).addReg(RegNo: 0);
10831 insertMemMemOp(MBB, MBB->end(),
10832 MachineOperand::CreateReg(Reg: ThisDestReg, isDef: false), DestDisp,
10833 MachineOperand::CreateReg(Reg: ThisSrcReg, isDef: false), SrcDisp, 256);
10834 if (EndMBB) {
10835 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10836 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10837 .addMBB(MBB: EndMBB);
10838 MBB->addSuccessor(Succ: EndMBB);
10839 MBB->addSuccessor(Succ: NextMBB);
10840 }
10841
10842 // NextMBB:
10843 // %NextDestReg = LA 256(%ThisDestReg)
10844 // %NextSrcReg = LA 256(%ThisSrcReg)
10845 // %NextCountReg = AGHI %ThisCountReg, -1
10846 // CGHI %NextCountReg, 0
10847 // JLH LoopMBB
10848 // # fall through to DoneMBB
10849 //
10850 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes.
10851 MBB = NextMBB;
10852 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextDestReg)
10853 .addReg(RegNo: ThisDestReg).addImm(Val: 256).addReg(RegNo: 0);
10854 if (!HaveSingleBase)
10855 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextSrcReg)
10856 .addReg(RegNo: ThisSrcReg).addImm(Val: 256).addReg(RegNo: 0);
10857 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::AGHI), DestReg: NextCountReg)
10858 .addReg(RegNo: ThisCountReg)
10859 .addImm(Val: -1)
10860 .setOperandDead(3);
10861 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10862 .addReg(RegNo: NextCountReg).addImm(Val: 0);
10863 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10864 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10865 .addMBB(MBB: LoopMBB);
10866 MBB->addSuccessor(Succ: LoopMBB);
10867 MBB->addSuccessor(Succ: DoneMBB);
10868
10869 MBB = DoneMBB;
10870 if (IsRegForm) {
10871 // DoneMBB:
10872 // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
10873 // # Use EXecute Relative Long for the remainder of the bytes. The target
10874 // instruction of the EXRL will have a length field of 1 since 0 is an
10875 // illegal value. The number of bytes processed becomes (%LenAdjReg &
10876 // 0xff) + 1.
10877 // # Fall through to AllDoneMBB.
10878 Register RemSrcReg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10879 Register RemDestReg = HaveSingleBase ? RemSrcReg
10880 : MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10881 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemDestReg)
10882 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10883 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10884 if (!HaveSingleBase)
10885 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemSrcReg)
10886 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10887 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10888 if (IsMemset)
10889 insertMemMemOp(MBB, MBB->end(),
10890 MachineOperand::CreateReg(Reg: RemDestReg, isDef: false), DestDisp,
10891 MachineOperand::CreateReg(Reg: RemSrcReg, isDef: false), SrcDisp, 1);
10892 MachineInstrBuilder EXRL_MIB =
10893 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EXRL_Pseudo))
10894 .addImm(Val: Opcode)
10895 .addReg(RegNo: LenAdjReg)
10896 .addReg(RegNo: RemDestReg).addImm(Val: DestDisp)
10897 .addReg(RegNo: RemSrcReg).addImm(Val: SrcDisp);
10898 MBB->addSuccessor(Succ: AllDoneMBB);
10899 MBB = AllDoneMBB;
10900 if (Opcode != SystemZ::MVC) {
10901 EXRL_MIB.addReg(RegNo: SystemZ::CC, Flags: RegState::ImplicitDefine);
10902 if (EndMBB)
10903 MBB->addLiveIn(PhysReg: SystemZ::CC);
10904 }
10905 }
10906 MF.getProperties().resetNoPHIs();
10907 }
10908
10909 // Handle any remaining bytes with straight-line code.
10910 while (ImmLength > 0) {
10911 uint64_t ThisLength = std::min(a: ImmLength, b: uint64_t(256));
10912 // The previous iteration might have created out-of-range displacements.
10913 // Apply them using LA/LAY if so.
10914 foldDisplIfNeeded(DestBase, DestDisp);
10915 foldDisplIfNeeded(SrcBase, SrcDisp);
10916 insertMemMemOp(MBB, MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
10917 DestDisp += ThisLength;
10918 SrcDisp += ThisLength;
10919 ImmLength -= ThisLength;
10920 // If there's another CLC to go, branch to the end if a difference
10921 // was found.
10922 if (EndMBB && ImmLength > 0) {
10923 MachineBasicBlock *NextMBB = SystemZ::splitBlockBefore(MI, MBB);
10924 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10925 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10926 .addMBB(MBB: EndMBB);
10927 MBB->addSuccessor(Succ: EndMBB);
10928 MBB->addSuccessor(Succ: NextMBB);
10929 MBB = NextMBB;
10930 }
10931 }
10932 if (EndMBB) {
10933 MBB->addSuccessor(Succ: EndMBB);
10934 MBB = EndMBB;
10935 MBB->addLiveIn(PhysReg: SystemZ::CC);
10936 }
10937
10938 MI.eraseFromParent();
10939 return MBB;
10940}
10941
10942MachineBasicBlock *
10943SystemZTargetLowering::emitMemmoveImm(MachineInstr &MI,
10944 MachineBasicBlock *MBB) const {
10945 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10946
10947 DebugLoc DL = MI.getDebugLoc();
10948 MachineOperand DstAddr = earlyUseOperand(Op: MI.getOperand(i: 0));
10949 MachineOperand SrcAddr = earlyUseOperand(Op: MI.getOperand(i: 1));
10950 uint64_t Len = MI.getOperand(i: 2).getImm();
10951 assert(Len > 0 && Len <= 256 && "Memmove of of unsupported constant length.");
10952
10953 // Use MVC or MVCRL after comparing the addresses.
10954 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
10955 MachineBasicBlock *MvcMBB = SystemZ::emitBlockAfter(MBB);
10956 MachineBasicBlock *MvcrlMBB = SystemZ::emitBlockAfter(MBB: MvcMBB);
10957 MBB->addSuccessor(Succ: MvcMBB);
10958 MBB->addSuccessor(Succ: MvcrlMBB);
10959 MvcMBB->addSuccessor(Succ: DoneMBB);
10960 MvcrlMBB->addSuccessor(Succ: DoneMBB);
10961
10962 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CLGR)).add(MO: SrcAddr).add(MO: DstAddr);
10963 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10964 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_LT)
10965 .addMBB(MBB: MvcrlMBB);
10966
10967 BuildMI(BB: MvcMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVC))
10968 .add(MO: DstAddr).addImm(Val: 0)
10969 .addImm(Val: Len)
10970 .add(MO: SrcAddr).addImm(Val: 0)
10971 .setMemRefs(MI.memoperands());
10972 BuildMI(BB: MvcMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: DoneMBB);
10973
10974 BuildMI(BB: MvcrlMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: SystemZ::R0L).addImm(Val: Len - 1);
10975 BuildMI(BB: MvcrlMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVCRL))
10976 .add(MO: DstAddr).addImm(Val: 0)
10977 .add(MO: SrcAddr).addImm(Val: 0)
10978 .setMemRefs(MI.memoperands());
10979
10980 MI.eraseFromParent();
10981 return DoneMBB;
10982}
10983
10984// Decompose string pseudo-instruction MI into a loop that continually performs
10985// Opcode until CC != 3.
10986MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
10987 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
10988 MachineFunction &MF = *MBB->getParent();
10989 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10990 MachineRegisterInfo &MRI = MF.getRegInfo();
10991 DebugLoc DL = MI.getDebugLoc();
10992
10993 uint64_t End1Reg = MI.getOperand(i: 0).getReg();
10994 uint64_t Start1Reg = MI.getOperand(i: 1).getReg();
10995 uint64_t Start2Reg = MI.getOperand(i: 2).getReg();
10996 uint64_t CharReg = MI.getOperand(i: 3).getReg();
10997
10998 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass;
10999 uint64_t This1Reg = MRI.createVirtualRegister(RegClass: RC);
11000 uint64_t This2Reg = MRI.createVirtualRegister(RegClass: RC);
11001 uint64_t End2Reg = MRI.createVirtualRegister(RegClass: RC);
11002
11003 MachineBasicBlock *StartMBB = MBB;
11004 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
11005 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
11006
11007 // StartMBB:
11008 // # fall through to LoopMBB
11009 MBB->addSuccessor(Succ: LoopMBB);
11010
11011 // LoopMBB:
11012 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ]
11013 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ]
11014 // R0L = %CharReg
11015 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L
11016 // JO LoopMBB
11017 // # fall through to DoneMBB
11018 //
11019 // The load of R0L can be hoisted by post-RA LICM.
11020 MBB = LoopMBB;
11021
11022 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This1Reg)
11023 .addReg(RegNo: Start1Reg).addMBB(MBB: StartMBB)
11024 .addReg(RegNo: End1Reg).addMBB(MBB: LoopMBB);
11025 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This2Reg)
11026 .addReg(RegNo: Start2Reg).addMBB(MBB: StartMBB)
11027 .addReg(RegNo: End2Reg).addMBB(MBB: LoopMBB);
11028 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: SystemZ::R0L).addReg(RegNo: CharReg);
11029 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode))
11030 .addReg(RegNo: End1Reg, Flags: RegState::Define).addReg(RegNo: End2Reg, Flags: RegState::Define)
11031 .addReg(RegNo: This1Reg).addReg(RegNo: This2Reg);
11032 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11033 .addImm(Val: SystemZ::CCMASK_ANY).addImm(Val: SystemZ::CCMASK_3).addMBB(MBB: LoopMBB);
11034 MBB->addSuccessor(Succ: LoopMBB);
11035 MBB->addSuccessor(Succ: DoneMBB);
11036
11037 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
11038
11039 MI.eraseFromParent();
11040 return DoneMBB;
11041}
11042
11043// Update TBEGIN instruction with final opcode and register clobbers.
11044MachineBasicBlock *SystemZTargetLowering::emitTransactionBegin(
11045 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode,
11046 bool NoFloat) const {
11047 MachineFunction &MF = *MBB->getParent();
11048 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
11049 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11050
11051 // Update opcode.
11052 MI.setDesc(TII->get(Opcode));
11053
11054 // We cannot handle a TBEGIN that clobbers the stack or frame pointer.
11055 // Make sure to add the corresponding GRSM bits if they are missing.
11056 uint64_t Control = MI.getOperand(i: 2).getImm();
11057 static const unsigned GPRControlBit[16] = {
11058 0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000,
11059 0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100
11060 };
11061 Control |= GPRControlBit[15];
11062 if (TFI->hasFP(MF))
11063 Control |= GPRControlBit[11];
11064 MI.getOperand(i: 2).setImm(Control);
11065
11066 // Add GPR clobbers.
11067 for (int I = 0; I < 16; I++) {
11068 if ((Control & GPRControlBit[I]) == 0) {
11069 unsigned Reg = SystemZMC::GR64Regs[I];
11070 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
11071 }
11072 }
11073
11074 // Add FPR/VR clobbers.
11075 if (!NoFloat && (Control & 4) != 0) {
11076 if (Subtarget.hasVector()) {
11077 for (unsigned Reg : SystemZMC::VR128Regs) {
11078 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
11079 }
11080 } else {
11081 for (unsigned Reg : SystemZMC::FP64Regs) {
11082 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
11083 }
11084 }
11085 }
11086
11087 return MBB;
11088}
11089
11090MachineBasicBlock *SystemZTargetLowering::emitLoadAndTestCmp0(
11091 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
11092 MachineFunction &MF = *MBB->getParent();
11093 MachineRegisterInfo *MRI = &MF.getRegInfo();
11094 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11095 DebugLoc DL = MI.getDebugLoc();
11096
11097 Register SrcReg = MI.getOperand(i: 0).getReg();
11098
11099 // Create new virtual register of the same class as source.
11100 const TargetRegisterClass *RC = MRI->getRegClass(Reg: SrcReg);
11101 Register DstReg = MRI->createVirtualRegister(RegClass: RC);
11102
11103 // Replace pseudo with a normal load-and-test that models the def as
11104 // well.
11105 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode), DestReg: DstReg)
11106 .addReg(RegNo: SrcReg)
11107 .setMIFlags(MI.getFlags());
11108 MI.eraseFromParent();
11109
11110 return MBB;
11111}
11112
11113MachineBasicBlock *SystemZTargetLowering::emitProbedAlloca(
11114 MachineInstr &MI, MachineBasicBlock *MBB) const {
11115 MachineFunction &MF = *MBB->getParent();
11116 MachineRegisterInfo *MRI = &MF.getRegInfo();
11117 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11118 DebugLoc DL = MI.getDebugLoc();
11119 const unsigned ProbeSize = getStackProbeSize(MF);
11120 Register DstReg = MI.getOperand(i: 0).getReg();
11121 Register SizeReg = MI.getOperand(i: 2).getReg();
11122
11123 MachineBasicBlock *StartMBB = MBB;
11124 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
11125 MachineBasicBlock *LoopTestMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
11126 MachineBasicBlock *LoopBodyMBB = SystemZ::emitBlockAfter(MBB: LoopTestMBB);
11127 MachineBasicBlock *TailTestMBB = SystemZ::emitBlockAfter(MBB: LoopBodyMBB);
11128 MachineBasicBlock *TailMBB = SystemZ::emitBlockAfter(MBB: TailTestMBB);
11129
11130 MachineMemOperand *VolLdMMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
11131 F: MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, Size: 8, BaseAlignment: Align(1));
11132
11133 Register PHIReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11134 Register IncReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11135
11136 // LoopTestMBB
11137 // BRC TailTestMBB
11138 // # fallthrough to LoopBodyMBB
11139 StartMBB->addSuccessor(Succ: LoopTestMBB);
11140 MBB = LoopTestMBB;
11141 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: PHIReg)
11142 .addReg(RegNo: SizeReg)
11143 .addMBB(MBB: StartMBB)
11144 .addReg(RegNo: IncReg)
11145 .addMBB(MBB: LoopBodyMBB);
11146 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CLGFI))
11147 .addReg(RegNo: PHIReg)
11148 .addImm(Val: ProbeSize);
11149 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11150 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_LT)
11151 .addMBB(MBB: TailTestMBB);
11152 MBB->addSuccessor(Succ: LoopBodyMBB);
11153 MBB->addSuccessor(Succ: TailTestMBB);
11154
11155 // LoopBodyMBB: Allocate and probe by means of a volatile compare.
11156 // J LoopTestMBB
11157 MBB = LoopBodyMBB;
11158 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: IncReg)
11159 .addReg(RegNo: PHIReg)
11160 .addImm(Val: ProbeSize)
11161 .setOperandDead(3);
11162 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: SystemZ::R15D)
11163 .addReg(RegNo: SystemZ::R15D)
11164 .addImm(Val: ProbeSize)
11165 .setOperandDead(3);
11166 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG))
11167 .addReg(RegNo: SystemZ::R15D)
11168 .addReg(RegNo: SystemZ::R15D)
11169 .addImm(Val: ProbeSize - 8)
11170 .addReg(RegNo: 0)
11171 .setOperandDead(4)
11172 .setMemRefs(VolLdMMO);
11173 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: LoopTestMBB);
11174 MBB->addSuccessor(Succ: LoopTestMBB);
11175
11176 // TailTestMBB
11177 // BRC DoneMBB
11178 // # fallthrough to TailMBB
11179 MBB = TailTestMBB;
11180 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
11181 .addReg(RegNo: PHIReg)
11182 .addImm(Val: 0);
11183 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11184 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
11185 .addMBB(MBB: DoneMBB);
11186 MBB->addSuccessor(Succ: TailMBB);
11187 MBB->addSuccessor(Succ: DoneMBB);
11188
11189 // TailMBB
11190 // # fallthrough to DoneMBB
11191 MBB = TailMBB;
11192 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGR), DestReg: SystemZ::R15D)
11193 .addReg(RegNo: SystemZ::R15D)
11194 .addReg(RegNo: PHIReg)
11195 .setOperandDead(3);
11196 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG))
11197 .addReg(RegNo: SystemZ::R15D)
11198 .addReg(RegNo: SystemZ::R15D)
11199 .addImm(Val: -8)
11200 .addReg(RegNo: PHIReg)
11201 .setOperandDead(4)
11202 .setMemRefs(VolLdMMO);
11203 MBB->addSuccessor(Succ: DoneMBB);
11204
11205 // DoneMBB
11206 MBB = DoneMBB;
11207 BuildMI(BB&: *MBB, I: MBB->begin(), MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
11208 .addReg(RegNo: SystemZ::R15D);
11209
11210 MI.eraseFromParent();
11211 return DoneMBB;
11212}
11213
11214SDValue SystemZTargetLowering::
11215getBackchainAddress(SDValue SP, SelectionDAG &DAG) const {
11216 MachineFunction &MF = DAG.getMachineFunction();
11217 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
11218 SDLoc DL(SP);
11219 return DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: SP,
11220 N2: DAG.getIntPtrConstant(Val: TFL->getBackchainOffset(MF), DL));
11221}
11222
11223// Replace a _STACKGUARD_DAG pseudo with a _STACKGUARD pseudo, adding
11224// a dead early-clobber def reg that will be used as a scratch register
11225// when the pseudo is expanded.
11226MachineBasicBlock *SystemZTargetLowering::emitStackGuardPseudo(
11227 MachineInstr &MI, MachineBasicBlock *MBB, unsigned PseudoOp) const {
11228 MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
11229 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11230 DebugLoc DL = MI.getDebugLoc();
11231 Register AddrReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11232 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PseudoOp), DestReg: AddrReg)
11233 .addFrameIndex(Idx: MI.getOperand(i: 0).getIndex())
11234 .addImm(Val: MI.getOperand(i: 1).getImm());
11235 MI.eraseFromParent();
11236 return MBB;
11237}
11238
11239MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
11240 MachineInstr &MI, MachineBasicBlock *MBB) const {
11241 switch (MI.getOpcode()) {
11242 case SystemZ::ADJCALLSTACKDOWN:
11243 case SystemZ::ADJCALLSTACKUP:
11244 return emitAdjCallStack(MI, BB: MBB);
11245
11246 case SystemZ::Select32:
11247 case SystemZ::Select64:
11248 case SystemZ::Select128:
11249 case SystemZ::SelectF32:
11250 case SystemZ::SelectF64:
11251 case SystemZ::SelectF128:
11252 case SystemZ::SelectVR32:
11253 case SystemZ::SelectVR64:
11254 case SystemZ::SelectVR128:
11255 return emitSelect(MI, MBB);
11256
11257 case SystemZ::CondStore8Mux:
11258 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: false);
11259 case SystemZ::CondStore8MuxInv:
11260 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: true);
11261 case SystemZ::CondStore16Mux:
11262 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: false);
11263 case SystemZ::CondStore16MuxInv:
11264 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: true);
11265 case SystemZ::CondStore32Mux:
11266 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: false);
11267 case SystemZ::CondStore32MuxInv:
11268 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: true);
11269 case SystemZ::CondStore8:
11270 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: false);
11271 case SystemZ::CondStore8Inv:
11272 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: true);
11273 case SystemZ::CondStore16:
11274 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: false);
11275 case SystemZ::CondStore16Inv:
11276 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: true);
11277 case SystemZ::CondStore32:
11278 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: false);
11279 case SystemZ::CondStore32Inv:
11280 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: true);
11281 case SystemZ::CondStore64:
11282 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: false);
11283 case SystemZ::CondStore64Inv:
11284 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: true);
11285 case SystemZ::CondStoreF32:
11286 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: false);
11287 case SystemZ::CondStoreF32Inv:
11288 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: true);
11289 case SystemZ::CondStoreF64:
11290 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: false);
11291 case SystemZ::CondStoreF64Inv:
11292 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: true);
11293
11294 case SystemZ::SCmp128Hi:
11295 return emitICmp128Hi(MI, MBB, Unsigned: false);
11296 case SystemZ::UCmp128Hi:
11297 return emitICmp128Hi(MI, MBB, Unsigned: true);
11298
11299 case SystemZ::PAIR128:
11300 return emitPair128(MI, MBB);
11301 case SystemZ::AEXT128:
11302 return emitExt128(MI, MBB, ClearEven: false);
11303 case SystemZ::ZEXT128:
11304 return emitExt128(MI, MBB, ClearEven: true);
11305
11306 case SystemZ::ATOMIC_SWAPW:
11307 return emitAtomicLoadBinary(MI, MBB, BinOpcode: 0);
11308
11309 case SystemZ::ATOMIC_LOADW_AR:
11310 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AR);
11311 case SystemZ::ATOMIC_LOADW_AFI:
11312 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AFI);
11313
11314 case SystemZ::ATOMIC_LOADW_SR:
11315 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::SR);
11316
11317 case SystemZ::ATOMIC_LOADW_NR:
11318 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR);
11319 case SystemZ::ATOMIC_LOADW_NILH:
11320 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH);
11321
11322 case SystemZ::ATOMIC_LOADW_OR:
11323 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OR);
11324 case SystemZ::ATOMIC_LOADW_OILH:
11325 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OILH);
11326
11327 case SystemZ::ATOMIC_LOADW_XR:
11328 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XR);
11329 case SystemZ::ATOMIC_LOADW_XILF:
11330 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XILF);
11331
11332 case SystemZ::ATOMIC_LOADW_NRi:
11333 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR, Invert: true);
11334 case SystemZ::ATOMIC_LOADW_NILHi:
11335 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH, Invert: true);
11336
11337 case SystemZ::ATOMIC_LOADW_MIN:
11338 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
11339 case SystemZ::ATOMIC_LOADW_MAX:
11340 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
11341 case SystemZ::ATOMIC_LOADW_UMIN:
11342 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
11343 case SystemZ::ATOMIC_LOADW_UMAX:
11344 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
11345
11346 case SystemZ::ATOMIC_CMP_SWAPW:
11347 return emitAtomicCmpSwapW(MI, MBB);
11348 case SystemZ::MVCImm:
11349 case SystemZ::MVCReg:
11350 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC);
11351 case SystemZ::NCImm:
11352 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::NC);
11353 case SystemZ::OCImm:
11354 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::OC);
11355 case SystemZ::XCImm:
11356 case SystemZ::XCReg:
11357 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::XC);
11358 case SystemZ::CLCImm:
11359 case SystemZ::CLCReg:
11360 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::CLC);
11361 case SystemZ::MemsetImmImm:
11362 case SystemZ::MemsetImmReg:
11363 case SystemZ::MemsetRegImm:
11364 case SystemZ::MemsetRegReg:
11365 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC, IsMemset: true/*IsMemset*/);
11366 case SystemZ::MemmoveImm:
11367 return emitMemmoveImm(MI, MBB);
11368 case SystemZ::CLSTLoop:
11369 return emitStringWrapper(MI, MBB, Opcode: SystemZ::CLST);
11370 case SystemZ::MVSTLoop:
11371 return emitStringWrapper(MI, MBB, Opcode: SystemZ::MVST);
11372 case SystemZ::SRSTLoop:
11373 return emitStringWrapper(MI, MBB, Opcode: SystemZ::SRST);
11374 case SystemZ::TBEGIN:
11375 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: false);
11376 case SystemZ::TBEGIN_nofloat:
11377 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: true);
11378 case SystemZ::TBEGINC:
11379 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGINC, NoFloat: true);
11380 case SystemZ::LTEBRCompare_Pseudo:
11381 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTEBR);
11382 case SystemZ::LTDBRCompare_Pseudo:
11383 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTDBR);
11384 case SystemZ::LTXBRCompare_Pseudo:
11385 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTXBR);
11386
11387 case SystemZ::PROBED_ALLOCA:
11388 return emitProbedAlloca(MI, MBB);
11389 case SystemZ::EH_SjLj_SetJmp:
11390 return emitEHSjLjSetJmp(MI, MBB);
11391 case SystemZ::EH_SjLj_LongJmp:
11392 return emitEHSjLjLongJmp(MI, MBB);
11393
11394 case TargetOpcode::STACKMAP:
11395 case TargetOpcode::PATCHPOINT:
11396 return emitPatchPoint(MI, MBB);
11397
11398 case SystemZ::MOV_STACKGUARD_DAG:
11399 return emitStackGuardPseudo(MI, MBB, PseudoOp: SystemZ::MOV_STACKGUARD);
11400
11401 case SystemZ::CMP_STACKGUARD_DAG:
11402 return emitStackGuardPseudo(MI, MBB, PseudoOp: SystemZ::CMP_STACKGUARD);
11403
11404 default:
11405 llvm_unreachable("Unexpected instr type to insert");
11406 }
11407}
11408
11409// This is only used by the isel schedulers, and is needed only to prevent
11410// compiler from crashing when list-ilp is used.
11411const TargetRegisterClass *
11412SystemZTargetLowering::getRepRegClassFor(MVT VT) const {
11413 if (VT == MVT::Untyped)
11414 return &SystemZ::ADDR128BitRegClass;
11415 return TargetLowering::getRepRegClassFor(VT);
11416}
11417
11418SDValue SystemZTargetLowering::lowerGET_ROUNDING(SDValue Op,
11419 SelectionDAG &DAG) const {
11420 SDLoc dl(Op);
11421 /*
11422 The rounding method is in FPC Byte 3 bits 6-7, and has the following
11423 settings:
11424 00 Round to nearest
11425 01 Round to 0
11426 10 Round to +inf
11427 11 Round to -inf
11428
11429 FLT_ROUNDS, on the other hand, expects the following:
11430 -1 Undefined
11431 0 Round to 0
11432 1 Round to nearest
11433 2 Round to +inf
11434 3 Round to -inf
11435 */
11436
11437 // Save FPC to register.
11438 SDValue Chain = Op.getOperand(i: 0);
11439 SDValue EFPC(
11440 DAG.getMachineNode(Opcode: SystemZ::EFPC, dl, ResultTys: {MVT::i32, MVT::Other}, Ops: Chain), 0);
11441 Chain = EFPC.getValue(R: 1);
11442
11443 // Transform as necessary
11444 SDValue CWD1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: EFPC,
11445 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
11446 // RetVal = (CWD1 ^ (CWD1 >> 1)) ^ 1
11447 SDValue CWD2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD1,
11448 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: CWD1,
11449 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32)));
11450
11451 SDValue RetVal = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD2,
11452 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11453 RetVal = DAG.getZExtOrTrunc(Op: RetVal, DL: dl, VT: Op.getValueType());
11454
11455 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl);
11456}
11457
11458SDValue SystemZTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
11459 SelectionDAG &DAG) const {
11460 EVT VT = Op.getValueType();
11461 Op = Op.getOperand(i: 0);
11462 EVT OpVT = Op.getValueType();
11463
11464 assert(OpVT.isVector() && "Operand type for VECREDUCE_ADD is not a vector.");
11465
11466 SDLoc DL(Op);
11467
11468 // load a 0 vector for the third operand of VSUM.
11469 SDValue Zero = DAG.getSplatBuildVector(VT: OpVT, DL, Op: DAG.getConstant(Val: 0, DL, VT));
11470
11471 // execute VSUM.
11472 switch (OpVT.getScalarSizeInBits()) {
11473 case 8:
11474 case 16:
11475 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Zero);
11476 [[fallthrough]];
11477 case 32:
11478 case 64:
11479 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::i128, N1: Op,
11480 N2: DAG.getBitcast(VT: Op.getValueType(), V: Zero));
11481 break;
11482 case 128:
11483 break; // VSUM over v1i128 should not happen and would be a noop
11484 default:
11485 llvm_unreachable("Unexpected scalar size.");
11486 }
11487 // Cast to original vector type, retrieve last element.
11488 return DAG.getNode(
11489 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: DAG.getBitcast(VT: OpVT, V: Op),
11490 N2: DAG.getConstant(Val: OpVT.getVectorNumElements() - 1, DL, VT: MVT::i32));
11491}
11492
11493static void printFunctionArgExts(const Function *F, raw_fd_ostream &OS) {
11494 FunctionType *FT = F->getFunctionType();
11495 const AttributeList &Attrs = F->getAttributes();
11496 if (Attrs.hasRetAttrs())
11497 OS << Attrs.getAsString(Index: AttributeList::ReturnIndex) << " ";
11498 OS << *F->getReturnType() << " @" << F->getName() << "(";
11499 for (unsigned I = 0, E = FT->getNumParams(); I != E; ++I) {
11500 if (I)
11501 OS << ", ";
11502 OS << *FT->getParamType(i: I);
11503 AttributeSet ArgAttrs = Attrs.getParamAttrs(ArgNo: I);
11504 for (auto A : {Attribute::SExt, Attribute::ZExt, Attribute::NoExt})
11505 if (ArgAttrs.hasAttribute(Kind: A))
11506 OS << " " << Attribute::getNameFromAttrKind(AttrKind: A);
11507 }
11508 OS << ")\n";
11509}
11510
11511bool SystemZTargetLowering::isInternal(const Function *Fn) const {
11512 std::map<const Function *, bool>::iterator Itr = IsInternalCache.find(x: Fn);
11513 if (Itr == IsInternalCache.end())
11514 Itr = IsInternalCache
11515 .insert(x: std::pair<const Function *, bool>(
11516 Fn, (Fn->hasLocalLinkage() && !Fn->hasAddressTaken())))
11517 .first;
11518 return Itr->second;
11519}
11520
11521void SystemZTargetLowering::
11522verifyNarrowIntegerArgs_Call(const SmallVectorImpl<ISD::OutputArg> &Outs,
11523 const Function *F, SDValue Callee) const {
11524 // Temporarily only do the check when explicitly requested, until it can be
11525 // enabled by default.
11526 if (!EnableIntArgExtCheck)
11527 return;
11528
11529 bool IsInternal = false;
11530 const Function *CalleeFn = nullptr;
11531 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
11532 if ((CalleeFn = dyn_cast<Function>(Val: G->getGlobal())))
11533 IsInternal = isInternal(Fn: CalleeFn);
11534 if (!IsInternal && !verifyNarrowIntegerArgs(Outs)) {
11535 errs() << "ERROR: Missing extension attribute of passed "
11536 << "value in call to function:\n" << "Callee: ";
11537 if (CalleeFn != nullptr)
11538 printFunctionArgExts(F: CalleeFn, OS&: errs());
11539 else
11540 errs() << "-\n";
11541 errs() << "Caller: ";
11542 printFunctionArgExts(F, OS&: errs());
11543 llvm_unreachable("");
11544 }
11545}
11546
11547void SystemZTargetLowering::
11548verifyNarrowIntegerArgs_Ret(const SmallVectorImpl<ISD::OutputArg> &Outs,
11549 const Function *F) const {
11550 // Temporarily only do the check when explicitly requested, until it can be
11551 // enabled by default.
11552 if (!EnableIntArgExtCheck)
11553 return;
11554
11555 if (!isInternal(Fn: F) && !verifyNarrowIntegerArgs(Outs)) {
11556 errs() << "ERROR: Missing extension attribute of returned "
11557 << "value from function:\n";
11558 printFunctionArgExts(F, OS&: errs());
11559 llvm_unreachable("");
11560 }
11561}
11562
11563// Verify that narrow integer arguments are extended as required by the ABI.
11564// Return false if an error is found.
11565bool SystemZTargetLowering::verifyNarrowIntegerArgs(
11566 const SmallVectorImpl<ISD::OutputArg> &Outs) const {
11567 if (!Subtarget.isTargetELF())
11568 return true;
11569
11570 if (EnableIntArgExtCheck.getNumOccurrences()) {
11571 if (!EnableIntArgExtCheck)
11572 return true;
11573 } else if (!getTargetMachine().Options.VerifyArgABICompliance)
11574 return true;
11575
11576 for (unsigned i = 0; i < Outs.size(); ++i) {
11577 MVT VT = Outs[i].VT;
11578 ISD::ArgFlagsTy Flags = Outs[i].Flags;
11579 if (VT.isInteger()) {
11580 assert((VT == MVT::i32 || VT.getSizeInBits() >= 64) &&
11581 "Unexpected integer argument VT.");
11582 if (VT == MVT::i32 &&
11583 !Flags.isSExt() && !Flags.isZExt() && !Flags.isNoExt())
11584 return false;
11585 }
11586 }
11587
11588 return true;
11589}
11590
11591void SystemZTargetLowering::insertSSPDeclarations(
11592 Module &M, const LibcallLoweringInfo &Libcalls) const {
11593 StringRef GuardMode = M.getStackProtectorGuard();
11594
11595 // In the TLS case, no symbol needs to be inserted.
11596 if (GuardMode == "tls" || GuardMode.empty())
11597 return;
11598
11599 // Otherwise (in the global case), insert the appropriate global variable.
11600 TargetLowering::insertSSPDeclarations(M, Libcalls);
11601}
11602