1//===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SystemZTargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SystemZISelLowering.h"
14#include "SystemZCallingConv.h"
15#include "SystemZConstantPoolValue.h"
16#include "SystemZMachineFunctionInfo.h"
17#include "llvm/ADT/SmallSet.h"
18#include "llvm/CodeGen/CallingConvLower.h"
19#include "llvm/CodeGen/ISDOpcodes.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
22#include "llvm/IR/GlobalAlias.h"
23#include "llvm/IR/IntrinsicInst.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsS390.h"
26#include "llvm/IR/Module.h"
27#include "llvm/IR/PatternMatch.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Target/TargetMachine.h"
32#include <cctype>
33#include <optional>
34
35using namespace llvm;
36
37#define DEBUG_TYPE "systemz-lower"
38
39// Temporarily let this be disabled by default until all known problems
40// related to argument extensions are fixed.
41static cl::opt<bool> EnableIntArgExtCheck(
42 "argext-abi-check", cl::init(Val: false),
43 cl::desc("Verify that narrow int args are properly extended per the "
44 "SystemZ ABI."));
45
46namespace {
47// Represents information about a comparison.
48struct Comparison {
49 Comparison(SDValue Op0In, SDValue Op1In, SDValue ChainIn)
50 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
51 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
52
53 // The operands to the comparison.
54 SDValue Op0, Op1;
55
56 // Chain if this is a strict floating-point comparison.
57 SDValue Chain;
58
59 // The opcode that should be used to compare Op0 and Op1.
60 unsigned Opcode;
61
62 // A SystemZICMP value. Only used for integer comparisons.
63 unsigned ICmpType;
64
65 // The mask of CC values that Opcode can produce.
66 unsigned CCValid;
67
68 // The mask of CC values for which the original condition is true.
69 unsigned CCMask;
70};
71} // end anonymous namespace
72
73// Classify VT as either 32 or 64 bit.
74static bool is32Bit(EVT VT) {
75 switch (VT.getSimpleVT().SimpleTy) {
76 case MVT::i32:
77 return true;
78 case MVT::i64:
79 return false;
80 default:
81 llvm_unreachable("Unsupported type");
82 }
83}
84
85// Return a version of MachineOperand that can be safely used before the
86// final use.
87static MachineOperand earlyUseOperand(MachineOperand Op) {
88 if (Op.isReg())
89 Op.setIsKill(false);
90 return Op;
91}
92
93SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
94 const SystemZSubtarget &STI)
95 : TargetLowering(TM, STI), Subtarget(STI) {
96 MVT PtrVT = MVT::getIntegerVT(BitWidth: TM.getPointerSizeInBits(AS: 0));
97
98 auto *Regs = STI.getSpecialRegisters();
99
100 // Set up the register classes.
101 if (Subtarget.hasHighWord())
102 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GRX32BitRegClass);
103 else
104 addRegisterClass(VT: MVT::i32, RC: &SystemZ::GR32BitRegClass);
105 addRegisterClass(VT: MVT::i64, RC: &SystemZ::GR64BitRegClass);
106 if (!useSoftFloat()) {
107 if (Subtarget.hasVector()) {
108 addRegisterClass(VT: MVT::f16, RC: &SystemZ::VR16BitRegClass);
109 addRegisterClass(VT: MVT::f32, RC: &SystemZ::VR32BitRegClass);
110 addRegisterClass(VT: MVT::f64, RC: &SystemZ::VR64BitRegClass);
111 } else {
112 addRegisterClass(VT: MVT::f16, RC: &SystemZ::FP16BitRegClass);
113 addRegisterClass(VT: MVT::f32, RC: &SystemZ::FP32BitRegClass);
114 addRegisterClass(VT: MVT::f64, RC: &SystemZ::FP64BitRegClass);
115 }
116 if (Subtarget.hasVectorEnhancements1())
117 addRegisterClass(VT: MVT::f128, RC: &SystemZ::VR128BitRegClass);
118 else
119 addRegisterClass(VT: MVT::f128, RC: &SystemZ::FP128BitRegClass);
120
121 if (Subtarget.hasVector()) {
122 addRegisterClass(VT: MVT::v16i8, RC: &SystemZ::VR128BitRegClass);
123 addRegisterClass(VT: MVT::v8i16, RC: &SystemZ::VR128BitRegClass);
124 addRegisterClass(VT: MVT::v4i32, RC: &SystemZ::VR128BitRegClass);
125 addRegisterClass(VT: MVT::v2i64, RC: &SystemZ::VR128BitRegClass);
126 addRegisterClass(VT: MVT::v8f16, RC: &SystemZ::VR128BitRegClass);
127 addRegisterClass(VT: MVT::v4f32, RC: &SystemZ::VR128BitRegClass);
128 addRegisterClass(VT: MVT::v2f64, RC: &SystemZ::VR128BitRegClass);
129 }
130
131 if (Subtarget.hasVector())
132 addRegisterClass(VT: MVT::i128, RC: &SystemZ::VR128BitRegClass);
133 }
134
135 // Compute derived properties from the register classes
136 computeRegisterProperties(TRI: Subtarget.getRegisterInfo());
137
138 // Set up special registers.
139 setStackPointerRegisterToSaveRestore(Regs->getStackPointerRegister());
140
141 // TODO: It may be better to default to latency-oriented scheduling, however
142 // LLVM's current latency-oriented scheduler can't handle physreg definitions
143 // such as SystemZ has with CC, so set this to the register-pressure
144 // scheduler, because it can.
145 setSchedulingPreference(Sched::RegPressure);
146
147 setBooleanContents(ZeroOrOneBooleanContent);
148 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
149
150 setMaxAtomicSizeInBitsSupported(128);
151
152 // Instructions are strings of 2-byte aligned 2-byte values.
153 setMinFunctionAlignment(Align(2));
154 // For performance reasons we prefer 16-byte alignment.
155 setPrefFunctionAlignment(Align(16));
156
157 // Handle operations that are handled in a similar way for all types.
158 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
159 I <= MVT::LAST_FP_VALUETYPE;
160 ++I) {
161 MVT VT = MVT::SimpleValueType(I);
162 if (isTypeLegal(VT)) {
163 // Lower SET_CC into an IPM-based sequence.
164 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
165 setOperationAction(Op: ISD::STRICT_FSETCC, VT, Action: Custom);
166 setOperationAction(Op: ISD::STRICT_FSETCCS, VT, Action: Custom);
167
168 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
169 setOperationAction(Op: ISD::SELECT, VT, Action: Expand);
170
171 // Lower SELECT_CC and BR_CC into separate comparisons and branches.
172 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Custom);
173 setOperationAction(Op: ISD::BR_CC, VT, Action: Custom);
174 }
175 }
176
177 // Expand jump table branches as address arithmetic followed by an
178 // indirect jump.
179 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
180
181 // Expand BRCOND into a BR_CC (see above).
182 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
183
184 // Handle integer types except i128.
185 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
186 I <= MVT::LAST_INTEGER_VALUETYPE;
187 ++I) {
188 MVT VT = MVT::SimpleValueType(I);
189 if (isTypeLegal(VT) && VT != MVT::i128) {
190 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
191
192 // Expand individual DIV and REMs into DIVREMs.
193 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
194 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
195 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
196 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
197 setOperationAction(Op: ISD::SDIVREM, VT, Action: Custom);
198 setOperationAction(Op: ISD::UDIVREM, VT, Action: Custom);
199
200 // Support addition/subtraction with overflow.
201 setOperationAction(Op: ISD::SADDO, VT, Action: Custom);
202 setOperationAction(Op: ISD::SSUBO, VT, Action: Custom);
203
204 // Support addition/subtraction with carry.
205 setOperationAction(Op: ISD::UADDO, VT, Action: Custom);
206 setOperationAction(Op: ISD::USUBO, VT, Action: Custom);
207
208 // Support carry in as value rather than glue.
209 setOperationAction(Op: ISD::UADDO_CARRY, VT, Action: Custom);
210 setOperationAction(Op: ISD::USUBO_CARRY, VT, Action: Custom);
211
212 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
213 // available, or if the operand is constant.
214 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT, Action: Custom);
215
216 // Use POPCNT on z196 and above.
217 if (Subtarget.hasPopulationCount())
218 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
219 else
220 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
221
222 // No special instructions for these.
223 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
224 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
225
226 // Use *MUL_LOHI where possible instead of MULH*.
227 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
228 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
229 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Custom);
230 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Custom);
231
232 // The fp<=>i32/i64 conversions are all Legal except for f16 and for
233 // unsigned on z10 (only z196 and above have native support for
234 // unsigned conversions).
235 for (auto Op : {ISD::FP_TO_SINT, ISD::STRICT_FP_TO_SINT,
236 ISD::SINT_TO_FP, ISD::STRICT_SINT_TO_FP})
237 setOperationAction(Op, VT, Action: Custom);
238 for (auto Op : {ISD::FP_TO_UINT, ISD::STRICT_FP_TO_UINT})
239 setOperationAction(Op, VT, Action: Custom);
240 for (auto Op : {ISD::UINT_TO_FP, ISD::STRICT_UINT_TO_FP}) {
241 // Handle unsigned 32-bit input types as signed 64-bit types on z10.
242 auto OpAction =
243 (!Subtarget.hasFPExtension() && VT == MVT::i32) ? Promote : Custom;
244 setOperationAction(Op, VT, Action: OpAction);
245 }
246 }
247 }
248
249 // Handle i128 if legal.
250 if (isTypeLegal(VT: MVT::i128)) {
251 // No special instructions for these.
252 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i128, Action: Expand);
253 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i128, Action: Expand);
254 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i128, Action: Expand);
255 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i128, Action: Expand);
256 setOperationAction(Op: ISD::ROTR, VT: MVT::i128, Action: Expand);
257 setOperationAction(Op: ISD::ROTL, VT: MVT::i128, Action: Expand);
258
259 // We may be able to use VSLDB/VSLD/VSRD for these.
260 setOperationAction(Op: ISD::FSHL, VT: MVT::i128, Action: Custom);
261 setOperationAction(Op: ISD::FSHR, VT: MVT::i128, Action: Custom);
262
263 // No special instructions for these before z17.
264 if (!Subtarget.hasVectorEnhancements3()) {
265 setOperationAction(Op: ISD::MUL, VT: MVT::i128, Action: Expand);
266 setOperationAction(Op: ISD::MULHS, VT: MVT::i128, Action: Expand);
267 setOperationAction(Op: ISD::MULHU, VT: MVT::i128, Action: Expand);
268 setOperationAction(Op: ISD::SDIV, VT: MVT::i128, Action: Expand);
269 setOperationAction(Op: ISD::UDIV, VT: MVT::i128, Action: Expand);
270 setOperationAction(Op: ISD::SREM, VT: MVT::i128, Action: Expand);
271 setOperationAction(Op: ISD::UREM, VT: MVT::i128, Action: Expand);
272 setOperationAction(Op: ISD::CTLZ, VT: MVT::i128, Action: Expand);
273 setOperationAction(Op: ISD::CTTZ, VT: MVT::i128, Action: Expand);
274 } else {
275 // Even if we do have a legal 128-bit multiply, we do not
276 // want 64-bit multiply-high operations to use it.
277 setOperationAction(Op: ISD::MULHS, VT: MVT::i64, Action: Custom);
278 setOperationAction(Op: ISD::MULHU, VT: MVT::i64, Action: Custom);
279 }
280
281 // Support addition/subtraction with carry.
282 setOperationAction(Op: ISD::UADDO, VT: MVT::i128, Action: Custom);
283 setOperationAction(Op: ISD::USUBO, VT: MVT::i128, Action: Custom);
284 setOperationAction(Op: ISD::UADDO_CARRY, VT: MVT::i128, Action: Custom);
285 setOperationAction(Op: ISD::USUBO_CARRY, VT: MVT::i128, Action: Custom);
286
287 // Use VPOPCT and add up partial results.
288 setOperationAction(Op: ISD::CTPOP, VT: MVT::i128, Action: Custom);
289
290 // Additional instructions available with z17.
291 if (Subtarget.hasVectorEnhancements3()) {
292 setOperationAction(Op: ISD::ABS, VT: MVT::i128, Action: Legal);
293
294 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
295 VT: MVT::i128, Action: Legal);
296 }
297 }
298
299 // These need custom handling in order to handle the f16 conversions.
300 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i128, Action: Custom);
301 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i128, Action: Custom);
302 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i128, Action: Custom);
303 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i128, Action: Custom);
304 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i128, Action: Custom);
305 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i128, Action: Custom);
306 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i128, Action: Custom);
307 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i128, Action: Custom);
308
309 // Type legalization will convert 8- and 16-bit atomic operations into
310 // forms that operate on i32s (but still keeping the original memory VT).
311 // Lower them into full i32 operations.
312 setOperationAction(Op: ISD::ATOMIC_SWAP, VT: MVT::i32, Action: Custom);
313 setOperationAction(Op: ISD::ATOMIC_LOAD_ADD, VT: MVT::i32, Action: Custom);
314 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT: MVT::i32, Action: Custom);
315 setOperationAction(Op: ISD::ATOMIC_LOAD_AND, VT: MVT::i32, Action: Custom);
316 setOperationAction(Op: ISD::ATOMIC_LOAD_OR, VT: MVT::i32, Action: Custom);
317 setOperationAction(Op: ISD::ATOMIC_LOAD_XOR, VT: MVT::i32, Action: Custom);
318 setOperationAction(Op: ISD::ATOMIC_LOAD_NAND, VT: MVT::i32, Action: Custom);
319 setOperationAction(Op: ISD::ATOMIC_LOAD_MIN, VT: MVT::i32, Action: Custom);
320 setOperationAction(Op: ISD::ATOMIC_LOAD_MAX, VT: MVT::i32, Action: Custom);
321 setOperationAction(Op: ISD::ATOMIC_LOAD_UMIN, VT: MVT::i32, Action: Custom);
322 setOperationAction(Op: ISD::ATOMIC_LOAD_UMAX, VT: MVT::i32, Action: Custom);
323
324 // Whether or not i128 is not a legal type, we need to custom lower
325 // the atomic operations in order to exploit SystemZ instructions.
326 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i128, Action: Custom);
327 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i128, Action: Custom);
328 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f128, Action: Custom);
329 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f128, Action: Custom);
330
331 // Mark sign/zero extending atomic loads as legal, which will make
332 // DAGCombiner fold extensions into atomic loads if possible.
333 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64,
334 MemVTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Legal);
335 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i32,
336 MemVTs: {MVT::i8, MVT::i16}, Action: Legal);
337 setAtomicLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i16,
338 MemVT: MVT::i8, Action: Legal);
339
340 // We can use the CC result of compare-and-swap to implement
341 // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
342 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i32, Action: Custom);
343 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i64, Action: Custom);
344 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT: MVT::i128, Action: Custom);
345
346 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
347
348 // Traps are legal, as we will convert them to "j .+2".
349 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
350
351 // We have native support for a 64-bit CTLZ, via FLOGR.
352 setOperationAction(Op: ISD::CTLZ, VT: MVT::i32, Action: Promote);
353 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: MVT::i32, Action: Promote);
354 setOperationAction(Op: ISD::CTLZ, VT: MVT::i64, Action: Legal);
355
356 // On z17 we have native support for a 64-bit CTTZ.
357 if (Subtarget.hasMiscellaneousExtensions4()) {
358 setOperationAction(Op: ISD::CTTZ, VT: MVT::i32, Action: Promote);
359 setOperationAction(Op: ISD::CTTZ_ZERO_POISON, VT: MVT::i32, Action: Promote);
360 setOperationAction(Op: ISD::CTTZ, VT: MVT::i64, Action: Legal);
361 }
362
363 // On z15 we have native support for a 64-bit CTPOP.
364 if (Subtarget.hasMiscellaneousExtensions3()) {
365 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Promote);
366 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Legal);
367 }
368
369 // Give LowerOperation the chance to replace 64-bit ORs with subregs.
370 setOperationAction(Op: ISD::OR, VT: MVT::i64, Action: Custom);
371
372 // Expand 128 bit shifts without using a libcall.
373 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64, Action: Expand);
374 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64, Action: Expand);
375 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64, Action: Expand);
376
377 // Also expand 256 bit shifts if i128 is a legal type.
378 if (isTypeLegal(VT: MVT::i128)) {
379 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i128, Action: Expand);
380 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i128, Action: Expand);
381 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i128, Action: Expand);
382 }
383
384 // Handle bitcast from fp128 to i128.
385 if (!isTypeLegal(VT: MVT::i128))
386 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
387
388 // We have native instructions for i8, i16 and i32 extensions, but not i1.
389 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
390 for (MVT VT : MVT::integer_valuetypes()) {
391 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
392 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
393 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
394 }
395
396 // Handle the various types of symbolic address.
397 setOperationAction(Op: ISD::ConstantPool, VT: PtrVT, Action: Custom);
398 setOperationAction(Op: ISD::GlobalAddress, VT: PtrVT, Action: Custom);
399 setOperationAction(Op: ISD::GlobalTLSAddress, VT: PtrVT, Action: Custom);
400 setOperationAction(Op: ISD::BlockAddress, VT: PtrVT, Action: Custom);
401 setOperationAction(Op: ISD::JumpTable, VT: PtrVT, Action: Custom);
402
403 // We need to handle dynamic allocations specially because of the
404 // 160-byte area at the bottom of the stack.
405 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: PtrVT, Action: Custom);
406 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: PtrVT, Action: Custom);
407
408 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
409 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Custom);
410
411 // Handle prefetches with PFD or PFDRL.
412 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
413
414 // Handle readcyclecounter with STCKF.
415 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Custom);
416
417 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
418 // Assume by default that all vector operations need to be expanded.
419 for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
420 if (getOperationAction(Op: Opcode, VT) == Legal)
421 setOperationAction(Op: Opcode, VT, Action: Expand);
422
423 // Likewise all truncating stores and extending loads.
424 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
425 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
426 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
427 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
428 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
429 }
430
431 if (isTypeLegal(VT)) {
432 // These operations are legal for anything that can be stored in a
433 // vector register, even if there is no native support for the format
434 // as such. In particular, we can do these for v4f32 even though there
435 // are no specific instructions for that format.
436 setOperationAction(Op: ISD::LOAD, VT, Action: Legal);
437 setOperationAction(Op: ISD::STORE, VT, Action: Legal);
438 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
439 setOperationAction(Op: ISD::BITCAST, VT, Action: Legal);
440 setOperationAction(Op: ISD::UNDEF, VT, Action: Legal);
441
442 // Likewise, except that we need to replace the nodes with something
443 // more specific.
444 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
445 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
446 }
447 }
448
449 // Handle integer vector types.
450 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
451 if (isTypeLegal(VT)) {
452 // These operations have direct equivalents.
453 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Legal);
454 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Legal);
455 setOperationAction(Op: ISD::ADD, VT, Action: Legal);
456 setOperationAction(Op: ISD::SUB, VT, Action: Legal);
457 if (VT != MVT::v2i64 || Subtarget.hasVectorEnhancements3()) {
458 setOperationAction(Op: ISD::MUL, VT, Action: Legal);
459 setOperationAction(Op: ISD::MULHS, VT, Action: Legal);
460 setOperationAction(Op: ISD::MULHU, VT, Action: Legal);
461 }
462 if (Subtarget.hasVectorEnhancements3() &&
463 VT != MVT::v16i8 && VT != MVT::v8i16) {
464 setOperationAction(Op: ISD::SDIV, VT, Action: Legal);
465 setOperationAction(Op: ISD::UDIV, VT, Action: Legal);
466 setOperationAction(Op: ISD::SREM, VT, Action: Legal);
467 setOperationAction(Op: ISD::UREM, VT, Action: Legal);
468 }
469 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
470 setOperationAction(Op: ISD::AND, VT, Action: Legal);
471 setOperationAction(Op: ISD::OR, VT, Action: Legal);
472 setOperationAction(Op: ISD::XOR, VT, Action: Legal);
473 if (Subtarget.hasVectorEnhancements1())
474 setOperationAction(Op: ISD::CTPOP, VT, Action: Legal);
475 else
476 setOperationAction(Op: ISD::CTPOP, VT, Action: Custom);
477 setOperationAction(Op: ISD::CTTZ, VT, Action: Legal);
478 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
479
480 // Convert a GPR scalar to a vector by inserting it into element 0.
481 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
482
483 // Use a series of unpacks for extensions.
484 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT, Action: Custom);
485 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT, Action: Custom);
486
487 // Detect shifts/rotates by a scalar amount and convert them into
488 // V*_BY_SCALAR.
489 setOperationAction(Op: ISD::SHL, VT, Action: Custom);
490 setOperationAction(Op: ISD::SRA, VT, Action: Custom);
491 setOperationAction(Op: ISD::SRL, VT, Action: Custom);
492 setOperationAction(Op: ISD::ROTL, VT, Action: Custom);
493
494 // Add ISD::VECREDUCE_ADD as custom in order to implement
495 // it with VZERO+VSUM
496 setOperationAction(Op: ISD::VECREDUCE_ADD, VT, Action: Custom);
497
498 // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
499 // and inverting the result as necessary.
500 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
501
502 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT,
503 Action: Legal);
504 }
505 }
506
507 if (Subtarget.hasVector()) {
508 // There should be no need to check for float types other than v2f64
509 // since <2 x f32> isn't a legal type.
510 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
511 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
512 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
513 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
514 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
515 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
516 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
517 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
518
519 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
520 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2f64, Action: Legal);
521 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
522 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2f64, Action: Legal);
523 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
524 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2f64, Action: Legal);
525 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
526 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2f64, Action: Legal);
527 }
528
529 if (Subtarget.hasVectorEnhancements2()) {
530 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
531 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
532 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
533 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
534 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
535 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
536 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
537 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
538
539 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
540 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4f32, Action: Legal);
541 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
542 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4f32, Action: Legal);
543 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
544 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4f32, Action: Legal);
545 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
546 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4f32, Action: Legal);
547 }
548
549 // Handle floating-point types.
550 if (!useSoftFloat()) {
551 // Promote all f16 operations to float, with some exceptions below.
552 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
553 setOperationAction(Op: Opc, VT: MVT::f16, Action: Promote);
554 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Expand);
555 for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) {
556 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f16, Action: Expand);
557 setTruncStoreAction(ValVT: VT, MemVT: MVT::f16, Action: Expand);
558 }
559 for (auto Op : {ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE})
560 setOperationAction(Op, VT: MVT::f16, Action: Subtarget.hasVector() ? Legal : Custom);
561 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f16, Action: LibCall);
562 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: LibCall);
563 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
564
565 for (auto Op : {ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN})
566 setOperationAction(Op, VT: MVT::f16, Action: Legal);
567 }
568
569 for (unsigned I = MVT::FIRST_FP_VALUETYPE;
570 I <= MVT::LAST_FP_VALUETYPE;
571 ++I) {
572 MVT VT = MVT::SimpleValueType(I);
573 if (isTypeLegal(VT) && VT != MVT::f16) {
574 // We can use FI for FRINT.
575 setOperationAction(Op: ISD::FRINT, VT, Action: Legal);
576
577 // We can use the extended form of FI for other rounding operations.
578 if (Subtarget.hasFPExtension()) {
579 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Legal);
580 setOperationAction(Op: ISD::FFLOOR, VT, Action: Legal);
581 setOperationAction(Op: ISD::FCEIL, VT, Action: Legal);
582 setOperationAction(Op: ISD::FTRUNC, VT, Action: Legal);
583 setOperationAction(Op: ISD::FROUND, VT, Action: Legal);
584 setOperationAction(Op: ISD::FROUNDEVEN, VT, Action: Legal);
585 }
586
587 // No special instructions for these.
588 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
589 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
590 setOperationAction(Op: ISD::FSINCOS, VT, Action: Expand);
591 setOperationAction(Op: ISD::FREM, VT, Action: LibCall);
592 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
593
594 // Special treatment.
595 setOperationAction(Op: ISD::IS_FPCLASS, VT, Action: Custom);
596
597 // Handle constrained floating-point operations.
598 setOperationAction(Op: ISD::STRICT_FADD, VT, Action: Legal);
599 setOperationAction(Op: ISD::STRICT_FSUB, VT, Action: Legal);
600 setOperationAction(Op: ISD::STRICT_FMUL, VT, Action: Legal);
601 setOperationAction(Op: ISD::STRICT_FDIV, VT, Action: Legal);
602 setOperationAction(Op: ISD::STRICT_FMA, VT, Action: Legal);
603 setOperationAction(Op: ISD::STRICT_FSQRT, VT, Action: Legal);
604 setOperationAction(Op: ISD::STRICT_FRINT, VT, Action: Legal);
605 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT, Action: Legal);
606 if (Subtarget.hasFPExtension()) {
607 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT, Action: Legal);
608 setOperationAction(Op: ISD::STRICT_FFLOOR, VT, Action: Legal);
609 setOperationAction(Op: ISD::STRICT_FCEIL, VT, Action: Legal);
610 setOperationAction(Op: ISD::STRICT_FTRUNC, VT, Action: Legal);
611 setOperationAction(Op: ISD::STRICT_FROUND, VT, Action: Legal);
612 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT, Action: Legal);
613 }
614
615 // Extension from f16 needs libcall.
616 setOperationAction(Op: ISD::FP_EXTEND, VT, Action: Custom);
617 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT, Action: Custom);
618 }
619 }
620
621 // Handle floating-point vector types.
622 if (Subtarget.hasVector()) {
623 // Scalar-to-vector conversion is just a subreg.
624 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8f16, Action: Legal);
625 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Legal);
626 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
627
628 // Some insertions and extractions can be done directly but others
629 // need to go via integers.
630 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8f16, Action: Custom);
631 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
632 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
633 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v8f16, Action: Custom);
634 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
635 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
636
637 // These operations have direct equivalents.
638 setOperationAction(Op: ISD::FADD, VT: MVT::v2f64, Action: Legal);
639 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Legal);
640 setOperationAction(Op: ISD::FSUB, VT: MVT::v2f64, Action: Legal);
641 setOperationAction(Op: ISD::FMUL, VT: MVT::v2f64, Action: Legal);
642 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Legal);
643 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Legal);
644 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Legal);
645 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Legal);
646 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Legal);
647 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v2f64, Action: Legal);
648 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Legal);
649 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Legal);
650 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Legal);
651 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f64, Action: Legal);
652 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
653
654 // Handle constrained floating-point operations.
655 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v2f64, Action: Legal);
656 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v2f64, Action: Legal);
657 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v2f64, Action: Legal);
658 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v2f64, Action: Legal);
659 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v2f64, Action: Legal);
660 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v2f64, Action: Legal);
661 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v2f64, Action: Legal);
662 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v2f64, Action: Legal);
663 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v2f64, Action: Legal);
664 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v2f64, Action: Legal);
665 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v2f64, Action: Legal);
666 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v2f64, Action: Legal);
667 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v2f64, Action: Legal);
668
669 setOperationAction(Op: ISD::SETCC, VT: MVT::v2f64, Action: Custom);
670 setOperationAction(Op: ISD::SETCC, VT: MVT::v4f32, Action: Custom);
671 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v2f64, Action: Custom);
672 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::v4f32, Action: Custom);
673 if (Subtarget.hasVectorEnhancements1()) {
674 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v2f64, Action: Custom);
675 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::v4f32, Action: Custom);
676 }
677 }
678
679 // The vector enhancements facility 1 has instructions for these.
680 if (Subtarget.hasVectorEnhancements1()) {
681 setOperationAction(Op: ISD::FADD, VT: MVT::v4f32, Action: Legal);
682 setOperationAction(Op: ISD::FNEG, VT: MVT::v4f32, Action: Legal);
683 setOperationAction(Op: ISD::FSUB, VT: MVT::v4f32, Action: Legal);
684 setOperationAction(Op: ISD::FMUL, VT: MVT::v4f32, Action: Legal);
685 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Legal);
686 setOperationAction(Op: ISD::FDIV, VT: MVT::v4f32, Action: Legal);
687 setOperationAction(Op: ISD::FABS, VT: MVT::v4f32, Action: Legal);
688 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Legal);
689 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Legal);
690 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Legal);
691 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Legal);
692 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Legal);
693 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Legal);
694 setOperationAction(Op: ISD::FROUND, VT: MVT::v4f32, Action: Legal);
695 setOperationAction(Op: ISD::FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
696
697 for (MVT Type : {MVT::f64, MVT::v2f64, MVT::f32, MVT::v4f32, MVT::f128}) {
698 setOperationAction(Op: ISD::FMAXNUM, VT: Type, Action: Legal);
699 setOperationAction(Op: ISD::FMAXIMUM, VT: Type, Action: Legal);
700 setOperationAction(Op: ISD::FMAXIMUMNUM, VT: Type, Action: Legal);
701 setOperationAction(Op: ISD::PSEUDO_FMAX, VT: Type, Action: Legal);
702 setOperationAction(Op: ISD::FMINNUM, VT: Type, Action: Legal);
703 setOperationAction(Op: ISD::FMINIMUM, VT: Type, Action: Legal);
704 setOperationAction(Op: ISD::FMINIMUMNUM, VT: Type, Action: Legal);
705 setOperationAction(Op: ISD::PSEUDO_FMIN, VT: Type, Action: Legal);
706 }
707
708 // Handle constrained floating-point operations.
709 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v4f32, Action: Legal);
710 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v4f32, Action: Legal);
711 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v4f32, Action: Legal);
712 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v4f32, Action: Legal);
713 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v4f32, Action: Legal);
714 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v4f32, Action: Legal);
715 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v4f32, Action: Legal);
716 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v4f32, Action: Legal);
717 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v4f32, Action: Legal);
718 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v4f32, Action: Legal);
719 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v4f32, Action: Legal);
720 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v4f32, Action: Legal);
721 setOperationAction(Op: ISD::STRICT_FROUNDEVEN, VT: MVT::v4f32, Action: Legal);
722 for (auto VT : { MVT::f32, MVT::f64, MVT::f128,
723 MVT::v4f32, MVT::v2f64 }) {
724 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT, Action: Legal);
725 setOperationAction(Op: ISD::STRICT_FMINNUM, VT, Action: Legal);
726 setOperationAction(Op: ISD::STRICT_FMAXIMUM, VT, Action: Legal);
727 setOperationAction(Op: ISD::STRICT_FMINIMUM, VT, Action: Legal);
728 setOperationAction(Op: ISD::STRICT_PSEUDO_FMAX, VT, Action: Legal);
729 setOperationAction(Op: ISD::STRICT_PSEUDO_FMIN, VT, Action: Legal);
730 }
731 }
732
733 // We only have fused f128 multiply-addition on vector registers.
734 if (!Subtarget.hasVectorEnhancements1()) {
735 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Expand);
736 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f128, Action: Expand);
737 }
738
739 // We don't have a copysign instruction on vector registers.
740 if (Subtarget.hasVectorEnhancements1())
741 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f128, Action: Expand);
742
743 // Needed so that we don't try to implement f128 constant loads using
744 // a load-and-extend of a f80 constant (in cases where the constant
745 // would fit in an f80).
746 for (MVT VT : MVT::fp_valuetypes())
747 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: MVT::f80, Action: Expand);
748
749 // We don't have extending load instruction on vector registers.
750 if (Subtarget.hasVectorEnhancements1()) {
751 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
752 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
753 }
754
755 // Floating-point truncation and stores need to be done separately.
756 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
757 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
758 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
759
760 // We have 64-bit FPR<->GPR moves, but need special handling for
761 // 32-bit forms.
762 if (!Subtarget.hasVector()) {
763 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
764 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Custom);
765 }
766
767 // VASTART and VACOPY need to deal with the SystemZ-specific varargs
768 // structure, but VAEND is a no-op.
769 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
770 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Custom);
771 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
772
773 if (Subtarget.isTargetzOS()) {
774 // Handle address space casts between mixed sized pointers.
775 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i32, Action: Custom);
776 setOperationAction(Op: ISD::ADDRSPACECAST, VT: MVT::i64, Action: Custom);
777 }
778
779 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
780
781 // Codes for which we want to perform some z-specific combinations.
782 setTargetDAGCombine({ISD::ZERO_EXTEND,
783 ISD::SIGN_EXTEND,
784 ISD::SIGN_EXTEND_INREG,
785 ISD::LOAD,
786 ISD::STORE,
787 ISD::VECTOR_SHUFFLE,
788 ISD::EXTRACT_VECTOR_ELT,
789 ISD::FP_ROUND,
790 ISD::STRICT_FP_ROUND,
791 ISD::FP_EXTEND,
792 ISD::SINT_TO_FP,
793 ISD::UINT_TO_FP,
794 ISD::STRICT_FP_EXTEND,
795 ISD::FCOPYSIGN,
796 ISD::BSWAP,
797 ISD::SETCC,
798 ISD::SRL,
799 ISD::SRA,
800 ISD::MUL,
801 ISD::SDIV,
802 ISD::UDIV,
803 ISD::SREM,
804 ISD::UREM,
805 ISD::INTRINSIC_VOID,
806 ISD::INTRINSIC_W_CHAIN});
807
808 // Handle intrinsics.
809 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
810 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
811
812 // We're not using SJLJ for exception handling, but they're implemented
813 // solely to support use of __builtin_setjmp / __builtin_longjmp.
814 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
815 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
816
817 // We want to use MVC in preference to even a single load/store pair.
818 MaxStoresPerMemcpy = Subtarget.hasVector() ? 2 : 0;
819 MaxStoresPerMemcpyOptSize = 0;
820
821 // Same with memmove.
822 MaxStoresPerMemmove = Subtarget.hasVector() ? 2 : 0;
823 MaxStoresPerMemmoveOptSize = 0;
824
825 // The main memset sequence is a byte store followed by an MVC.
826 // Two STC or MV..I stores win over that, but the kind of fused stores
827 // generated by target-independent code don't when the byte value is
828 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better
829 // than "STC;MVC". Handle the choice in target-specific code instead.
830 MaxStoresPerMemset = Subtarget.hasVector() ? 2 : 0;
831 MaxStoresPerMemsetOptSize = 0;
832
833 // Default to having -disable-strictnode-mutation on
834 IsStrictFPEnabled = true;
835}
836
837bool SystemZTargetLowering::useSoftFloat() const {
838 return Subtarget.hasSoftFloat();
839}
840
841unsigned SystemZTargetLowering::getVectorTypeBreakdownForCallingConv(
842 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
843 unsigned &NumIntermediates, MVT &RegisterVT) const {
844 // Pass fp16 vectors in VR(s).
845 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16)) {
846 IntermediateVT = RegisterVT = MVT::v8f16;
847 return NumIntermediates =
848 divideCeil(Numerator: VT.getVectorNumElements(), Denominator: SystemZ::VectorBytes / 2);
849 }
850 return TargetLowering::getVectorTypeBreakdownForCallingConv(
851 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
852}
853
854MVT SystemZTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
855 CallingConv::ID CC,
856 EVT VT) const {
857 // 128-bit single-element vector types are passed like other vectors,
858 // not like their element type.
859 if (Subtarget.hasVector() && VT.isVector() && VT.getSizeInBits() == 128 &&
860 VT.getVectorNumElements() == 1)
861 return MVT::v16i8;
862 // Pass fp16 vectors in VR(s).
863 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16))
864 return MVT::v8f16;
865 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
866}
867
868unsigned SystemZTargetLowering::getNumRegistersForCallingConv(
869 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
870 // Pass fp16 vectors in VR(s).
871 if (Subtarget.hasVector() && VT.isVectorOf(EltVT: MVT::f16))
872 return divideCeil(Numerator: VT.getVectorNumElements(), Denominator: SystemZ::VectorBytes / 2);
873 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
874}
875
876EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,
877 LLVMContext &, EVT VT) const {
878 if (!VT.isVector())
879 return MVT::i32;
880 return VT.changeVectorElementTypeToInteger();
881}
882
883bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(
884 const MachineFunction &MF, EVT VT) const {
885 if (useSoftFloat())
886 return false;
887
888 VT = VT.getScalarType();
889
890 if (!VT.isSimple())
891 return false;
892
893 switch (VT.getSimpleVT().SimpleTy) {
894 case MVT::f32:
895 case MVT::f64:
896 return true;
897 case MVT::f128:
898 return Subtarget.hasVectorEnhancements1();
899 default:
900 break;
901 }
902
903 return false;
904}
905
906// Return true if the constant can be generated with a vector instruction,
907// such as VGM, VGMB or VREPI.
908bool SystemZVectorConstantInfo::isVectorConstantLegal(
909 const SystemZSubtarget &Subtarget) {
910 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
911 if (!Subtarget.hasVector() ||
912 (isFP128 && !Subtarget.hasVectorEnhancements1()))
913 return false;
914
915 // Try using VECTOR GENERATE BYTE MASK. This is the architecturally-
916 // preferred way of creating all-zero and all-one vectors so give it
917 // priority over other methods below.
918 unsigned Mask = 0;
919 unsigned I = 0;
920 for (; I < SystemZ::VectorBytes; ++I) {
921 uint64_t Byte = IntBits.lshr(shiftAmt: I * 8).trunc(width: 8).getZExtValue();
922 if (Byte == 0xff)
923 Mask |= 1ULL << I;
924 else if (Byte != 0)
925 break;
926 }
927 if (I == SystemZ::VectorBytes) {
928 Opcode = SystemZISD::BYTE_MASK;
929 OpVals.push_back(Elt: Mask);
930 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: 8), NumElements: 16);
931 return true;
932 }
933
934 if (SplatBitSize > 64)
935 return false;
936
937 auto TryValue = [&](uint64_t Value) -> bool {
938 // Try VECTOR REPLICATE IMMEDIATE
939 int64_t SignedValue = SignExtend64(X: Value, B: SplatBitSize);
940 if (isInt<16>(x: SignedValue)) {
941 OpVals.push_back(Elt: ((unsigned) SignedValue));
942 Opcode = SystemZISD::REPLICATE;
943 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
944 NumElements: SystemZ::VectorBits / SplatBitSize);
945 return true;
946 }
947 // Try VECTOR GENERATE MASK
948 unsigned Start, End;
949 if (TII->isRxSBGMask(Mask: Value, BitSize: SplatBitSize, Start, End)) {
950 // isRxSBGMask returns the bit numbers for a full 64-bit value, with 0
951 // denoting 1 << 63 and 63 denoting 1. Convert them to bit numbers for
952 // an SplatBitSize value, so that 0 denotes 1 << (SplatBitSize-1).
953 OpVals.push_back(Elt: Start - (64 - SplatBitSize));
954 OpVals.push_back(Elt: End - (64 - SplatBitSize));
955 Opcode = SystemZISD::ROTATE_MASK;
956 VecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplatBitSize),
957 NumElements: SystemZ::VectorBits / SplatBitSize);
958 return true;
959 }
960 return false;
961 };
962
963 // First try assuming that any undefined bits above the highest set bit
964 // and below the lowest set bit are 1s. This increases the likelihood of
965 // being able to use a sign-extended element value in VECTOR REPLICATE
966 // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
967 uint64_t SplatBitsZ = SplatBits.getZExtValue();
968 uint64_t SplatUndefZ = SplatUndef.getZExtValue();
969 unsigned LowerBits = llvm::countr_zero(Val: SplatBitsZ);
970 unsigned UpperBits = llvm::countl_zero(Val: SplatBitsZ);
971 uint64_t Lower = SplatUndefZ & maskTrailingOnes<uint64_t>(N: LowerBits);
972 uint64_t Upper = SplatUndefZ & maskLeadingOnes<uint64_t>(N: UpperBits);
973 if (TryValue(SplatBitsZ | Upper | Lower))
974 return true;
975
976 // Now try assuming that any undefined bits between the first and
977 // last defined set bits are set. This increases the chances of
978 // using a non-wraparound mask.
979 uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
980 return TryValue(SplatBitsZ | Middle);
981}
982
983SystemZVectorConstantInfo::SystemZVectorConstantInfo(APInt IntImm) {
984 if (IntImm.isSingleWord()) {
985 IntBits = APInt(128, IntImm.getZExtValue());
986 IntBits <<= (SystemZ::VectorBits - IntImm.getBitWidth());
987 } else
988 IntBits = IntImm;
989 assert(IntBits.getBitWidth() == 128 && "Unsupported APInt.");
990
991 // Find the smallest splat.
992 SplatBits = IntImm;
993 unsigned Width = SplatBits.getBitWidth();
994 while (Width > 8) {
995 unsigned HalfSize = Width / 2;
996 APInt HighValue = SplatBits.lshr(shiftAmt: HalfSize).trunc(width: HalfSize);
997 APInt LowValue = SplatBits.trunc(width: HalfSize);
998
999 // If the two halves do not match, stop here.
1000 if (HighValue != LowValue || 8 > HalfSize)
1001 break;
1002
1003 SplatBits = HighValue;
1004 Width = HalfSize;
1005 }
1006 SplatUndef = 0;
1007 SplatBitSize = Width;
1008}
1009
1010SystemZVectorConstantInfo::SystemZVectorConstantInfo(BuildVectorSDNode *BVN) {
1011 assert(BVN->isConstant() && "Expected a constant BUILD_VECTOR");
1012 bool HasAnyUndefs;
1013
1014 // Get IntBits by finding the 128 bit splat.
1015 BVN->isConstantSplat(SplatValue&: IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 128,
1016 isBigEndian: true);
1017
1018 // Get SplatBits by finding the 8 bit or greater splat.
1019 BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 8,
1020 isBigEndian: true);
1021}
1022
1023bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1024 bool ForCodeSize) const {
1025 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
1026 if (Imm.isZero() || Imm.isNegZero())
1027 return true;
1028
1029 return SystemZVectorConstantInfo(Imm).isVectorConstantLegal(Subtarget);
1030}
1031
1032MachineBasicBlock *
1033SystemZTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
1034 MachineBasicBlock *MBB) const {
1035 DebugLoc DL = MI.getDebugLoc();
1036 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1037 const SystemZRegisterInfo *TRI = Subtarget.getRegisterInfo();
1038
1039 MachineFunction *MF = MBB->getParent();
1040 MachineRegisterInfo &MRI = MF->getRegInfo();
1041
1042 const BasicBlock *BB = MBB->getBasicBlock();
1043 MachineFunction::iterator I = ++MBB->getIterator();
1044
1045 Register DstReg = MI.getOperand(i: 0).getReg();
1046 const TargetRegisterClass *RC = MRI.getRegClass(Reg: DstReg);
1047 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
1048 (void)TRI;
1049 Register MainDstReg = MRI.createVirtualRegister(RegClass: RC);
1050 Register RestoreDstReg = MRI.createVirtualRegister(RegClass: RC);
1051
1052 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1053 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1054 // For v = setjmp(buf), we generate.
1055 // Algorithm:
1056 //
1057 // ---------
1058 // | thisMBB |
1059 // ---------
1060 // |
1061 // ------------------------
1062 // | |
1063 // ---------- ---------------
1064 // | mainMBB | | restoreMBB |
1065 // | v = 0 | | v = 1 |
1066 // ---------- ---------------
1067 // | |
1068 // -------------------------
1069 // |
1070 // -----------------------------
1071 // | sinkMBB |
1072 // | phi(v_mainMBB,v_restoreMBB) |
1073 // -----------------------------
1074 // thisMBB:
1075 // buf[FPOffset] = Frame Pointer if hasFP.
1076 // buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB.
1077 // buf[BCOffset] = Backchain value if building with -mbackchain.
1078 // buf[SPOffset] = Stack Pointer.
1079 // buf[LPOffset] = We never write this slot with R13, gcc stores R13 always.
1080 // SjLjSetup restoreMBB
1081 // mainMBB:
1082 // v_main = 0
1083 // sinkMBB:
1084 // v = phi(v_main, v_restore)
1085 // restoreMBB:
1086 // v_restore = 1
1087
1088 MachineBasicBlock *ThisMBB = MBB;
1089 MachineBasicBlock *MainMBB = MF->CreateMachineBasicBlock(BB);
1090 MachineBasicBlock *SinkMBB = MF->CreateMachineBasicBlock(BB);
1091 MachineBasicBlock *RestoreMBB = MF->CreateMachineBasicBlock(BB);
1092
1093 MF->insert(MBBI: I, MBB: MainMBB);
1094 MF->insert(MBBI: I, MBB: SinkMBB);
1095 MF->push_back(MBB: RestoreMBB);
1096 RestoreMBB->setMachineBlockAddressTaken();
1097
1098 MachineInstrBuilder MIB;
1099
1100 // Transfer the remainder of BB and its successor edges to sinkMBB.
1101 SinkMBB->splice(Where: SinkMBB->begin(), Other: MBB,
1102 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
1103 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
1104
1105 // thisMBB:
1106 const int64_t FPOffset = 0; // Slot 1.
1107 const int64_t LabelOffset = 1 * PVT.getStoreSize(); // Slot 2.
1108 const int64_t BCOffset = 2 * PVT.getStoreSize(); // Slot 3.
1109 const int64_t SPOffset = 3 * PVT.getStoreSize(); // Slot 4.
1110
1111 // Buf address.
1112 Register BufReg = MI.getOperand(i: 1).getReg();
1113
1114 const TargetRegisterClass *PtrRC = getRegClassFor(VT: PVT);
1115 Register LabelReg = MRI.createVirtualRegister(RegClass: PtrRC);
1116
1117 // Prepare IP for longjmp.
1118 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LARL), DestReg: LabelReg)
1119 .addMBB(MBB: RestoreMBB);
1120 // Store IP for return from jmp, slot 2, offset = 1.
1121 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1122 .addReg(RegNo: LabelReg)
1123 .addReg(RegNo: BufReg)
1124 .addImm(Val: LabelOffset)
1125 .addReg(RegNo: 0);
1126
1127 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1128 bool HasFP = Subtarget.getFrameLowering()->hasFP(MF: *MF);
1129 if (HasFP) {
1130 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1131 .addReg(RegNo: SpecialRegs->getFramePointerRegister())
1132 .addReg(RegNo: BufReg)
1133 .addImm(Val: FPOffset)
1134 .addReg(RegNo: 0);
1135 }
1136
1137 // Store SP.
1138 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1139 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1140 .addReg(RegNo: BufReg)
1141 .addImm(Val: SPOffset)
1142 .addReg(RegNo: 0);
1143
1144 // Slot 3(Offset = 2) Backchain value (if building with -mbackchain).
1145 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1146 if (BackChain) {
1147 Register BCReg = MRI.createVirtualRegister(RegClass: PtrRC);
1148 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1149 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1150 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1151 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1152 .addReg(RegNo: 0);
1153
1154 BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1155 .addReg(RegNo: BCReg)
1156 .addReg(RegNo: BufReg)
1157 .addImm(Val: BCOffset)
1158 .addReg(RegNo: 0);
1159 }
1160
1161 // Setup.
1162 MIB = BuildMI(BB&: *ThisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EH_SjLj_Setup))
1163 .addMBB(MBB: RestoreMBB);
1164
1165 const SystemZRegisterInfo *RegInfo = Subtarget.getRegisterInfo();
1166 MIB.addRegMask(Mask: RegInfo->getNoPreservedMask());
1167
1168 ThisMBB->addSuccessor(Succ: MainMBB);
1169 ThisMBB->addSuccessor(Succ: RestoreMBB);
1170
1171 // mainMBB:
1172 BuildMI(BB: MainMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: MainDstReg).addImm(Val: 0);
1173 MainMBB->addSuccessor(Succ: SinkMBB);
1174
1175 // sinkMBB:
1176 BuildMI(BB&: *SinkMBB, I: SinkMBB->begin(), MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: DstReg)
1177 .addReg(RegNo: MainDstReg)
1178 .addMBB(MBB: MainMBB)
1179 .addReg(RegNo: RestoreDstReg)
1180 .addMBB(MBB: RestoreMBB);
1181
1182 // restoreMBB.
1183 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LHI), DestReg: RestoreDstReg).addImm(Val: 1);
1184 BuildMI(BB: RestoreMBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: SinkMBB);
1185 RestoreMBB->addSuccessor(Succ: SinkMBB);
1186
1187 MI.eraseFromParent();
1188
1189 return SinkMBB;
1190}
1191
1192MachineBasicBlock *
1193SystemZTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
1194 MachineBasicBlock *MBB) const {
1195
1196 DebugLoc DL = MI.getDebugLoc();
1197 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1198
1199 MachineFunction *MF = MBB->getParent();
1200 MachineRegisterInfo &MRI = MF->getRegInfo();
1201
1202 MVT PVT = getPointerTy(DL: MF->getDataLayout());
1203 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");
1204 Register BufReg = MI.getOperand(i: 0).getReg();
1205 const TargetRegisterClass *RC = MRI.getRegClass(Reg: BufReg);
1206 auto *SpecialRegs = Subtarget.getSpecialRegisters();
1207
1208 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
1209 Register BCReg = MRI.createVirtualRegister(RegClass: RC);
1210
1211 MachineInstrBuilder MIB;
1212
1213 const int64_t FPOffset = 0;
1214 const int64_t LabelOffset = 1 * PVT.getStoreSize();
1215 const int64_t BCOffset = 2 * PVT.getStoreSize();
1216 const int64_t SPOffset = 3 * PVT.getStoreSize();
1217 const int64_t LPOffset = 4 * PVT.getStoreSize();
1218
1219 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: Tmp)
1220 .addReg(RegNo: BufReg)
1221 .addImm(Val: LabelOffset)
1222 .addReg(RegNo: 0);
1223
1224 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1225 DestReg: SpecialRegs->getFramePointerRegister())
1226 .addReg(RegNo: BufReg)
1227 .addImm(Val: FPOffset)
1228 .addReg(RegNo: 0);
1229
1230 // We are restoring R13 even though we never stored in setjmp from llvm,
1231 // as gcc always stores R13 in builtin_setjmp. We could have mixed code
1232 // gcc setjmp and llvm longjmp.
1233 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: SystemZ::R13D)
1234 .addReg(RegNo: BufReg)
1235 .addImm(Val: LPOffset)
1236 .addReg(RegNo: 0);
1237
1238 bool BackChain = MF->getSubtarget<SystemZSubtarget>().hasBackChain();
1239 if (BackChain) {
1240 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG), DestReg: BCReg)
1241 .addReg(RegNo: BufReg)
1242 .addImm(Val: BCOffset)
1243 .addReg(RegNo: 0);
1244 }
1245
1246 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LG),
1247 DestReg: SpecialRegs->getStackPointerRegister())
1248 .addReg(RegNo: BufReg)
1249 .addImm(Val: SPOffset)
1250 .addReg(RegNo: 0);
1251
1252 if (BackChain) {
1253 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
1254 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STG))
1255 .addReg(RegNo: BCReg)
1256 .addReg(RegNo: SpecialRegs->getStackPointerRegister())
1257 .addImm(Val: TFL->getBackchainOffset(MF&: *MF))
1258 .addReg(RegNo: 0);
1259 }
1260
1261 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BR)).addReg(RegNo: Tmp);
1262
1263 MI.eraseFromParent();
1264 return MBB;
1265}
1266
1267/// Returns true if stack probing through inline assembly is requested.
1268bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
1269 // If the function specifically requests inline stack probes, emit them.
1270 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
1271 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
1272 "inline-asm";
1273 return false;
1274}
1275
1276TargetLowering::AtomicExpansionKind
1277SystemZTargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
1278 return AtomicExpansionKind::None;
1279}
1280
1281TargetLowering::AtomicExpansionKind
1282SystemZTargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
1283 return AtomicExpansionKind::None;
1284}
1285
1286TargetLowering::AtomicExpansionKind
1287SystemZTargetLowering::shouldExpandAtomicRMWInIR(
1288 const AtomicRMWInst *RMW) const {
1289 // Don't expand subword operations as they require special treatment.
1290 if (RMW->getType()->isIntegerTy(BitWidth: 8) || RMW->getType()->isIntegerTy(BitWidth: 16))
1291 return AtomicExpansionKind::None;
1292
1293 // Don't expand if there is a target instruction available.
1294 if (Subtarget.hasInterlockedAccess1() &&
1295 (RMW->getType()->isIntegerTy(BitWidth: 32) || RMW->getType()->isIntegerTy(BitWidth: 64)) &&
1296 (RMW->getOperation() == AtomicRMWInst::BinOp::Add ||
1297 RMW->getOperation() == AtomicRMWInst::BinOp::Sub ||
1298 RMW->getOperation() == AtomicRMWInst::BinOp::And ||
1299 RMW->getOperation() == AtomicRMWInst::BinOp::Or ||
1300 RMW->getOperation() == AtomicRMWInst::BinOp::Xor))
1301 return AtomicExpansionKind::None;
1302
1303 return AtomicExpansionKind::CmpXChg;
1304}
1305
1306bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1307 // We can use CGFI or CLGFI.
1308 return isInt<32>(x: Imm) || isUInt<32>(x: Imm);
1309}
1310
1311bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1312 // We can use ALGFI or SLGFI.
1313 return isUInt<32>(x: Imm) || isUInt<32>(x: -Imm);
1314}
1315
1316bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(
1317 EVT VT, unsigned, Align, MachineMemOperand::Flags, unsigned *Fast) const {
1318 // Unaligned accesses should never be slower than the expanded version.
1319 // We check specifically for aligned accesses in the few cases where
1320 // they are required.
1321 if (Fast)
1322 *Fast = 1;
1323 return true;
1324}
1325
1326bool SystemZTargetLowering::hasAndNot(SDValue Y) const {
1327 EVT VT = Y.getValueType();
1328
1329 // We can use NC(G)RK for types in GPRs ...
1330 if (VT == MVT::i32 || VT == MVT::i64)
1331 return Subtarget.hasMiscellaneousExtensions3();
1332
1333 // ... or VNC for types in VRs.
1334 if (VT.isVector() || VT == MVT::i128)
1335 return Subtarget.hasVector();
1336
1337 return false;
1338}
1339
1340// Information about the addressing mode for a memory access.
1341struct AddressingMode {
1342 // True if a long displacement is supported.
1343 bool LongDisplacement;
1344
1345 // True if use of index register is supported.
1346 bool IndexReg;
1347
1348 AddressingMode(bool LongDispl, bool IdxReg) :
1349 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
1350};
1351
1352// Return the desired addressing mode for a Load which has only one use (in
1353// the same block) which is a Store.
1354static AddressingMode getLoadStoreAddrMode(bool HasVector,
1355 Type *Ty) {
1356 // With vector support a Load->Store combination may be combined to either
1357 // an MVC or vector operations and it seems to work best to allow the
1358 // vector addressing mode.
1359 if (HasVector)
1360 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1361
1362 // Otherwise only the MVC case is special.
1363 bool MVC = Ty->isIntegerTy(BitWidth: 8);
1364 return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/);
1365}
1366
1367// Return the addressing mode which seems most desirable given an LLVM
1368// Instruction pointer.
1369static AddressingMode
1370supportedAddressingMode(Instruction *I, bool HasVector) {
1371 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1372 switch (II->getIntrinsicID()) {
1373 default: break;
1374 case Intrinsic::memset:
1375 case Intrinsic::memmove:
1376 case Intrinsic::memcpy:
1377 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1378 }
1379 }
1380
1381 if (isa<LoadInst>(Val: I) && I->hasOneUse()) {
1382 auto *SingleUser = cast<Instruction>(Val: *I->user_begin());
1383 if (SingleUser->getParent() == I->getParent()) {
1384 if (isa<ICmpInst>(Val: SingleUser)) {
1385 if (auto *C = dyn_cast<ConstantInt>(Val: SingleUser->getOperand(i: 1)))
1386 if (C->getBitWidth() <= 64 &&
1387 (isInt<16>(x: C->getSExtValue()) || isUInt<16>(x: C->getZExtValue())))
1388 // Comparison of memory with 16 bit signed / unsigned immediate
1389 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
1390 } else if (isa<StoreInst>(Val: SingleUser))
1391 // Load->Store
1392 return getLoadStoreAddrMode(HasVector, Ty: I->getType());
1393 }
1394 } else if (auto *StoreI = dyn_cast<StoreInst>(Val: I)) {
1395 if (auto *LoadI = dyn_cast<LoadInst>(Val: StoreI->getValueOperand()))
1396 if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())
1397 // Load->Store
1398 return getLoadStoreAddrMode(HasVector, Ty: LoadI->getType());
1399 }
1400
1401 if (HasVector && (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I))) {
1402
1403 // * Use LDE instead of LE/LEY for z13 to avoid partial register
1404 // dependencies (LDE only supports small offsets).
1405 // * Utilize the vector registers to hold floating point
1406 // values (vector load / store instructions only support small
1407 // offsets).
1408
1409 Type *MemAccessTy = (isa<LoadInst>(Val: I) ? I->getType() :
1410 I->getOperand(i: 0)->getType());
1411 bool IsFPAccess = MemAccessTy->isFloatingPointTy();
1412 bool IsVectorAccess = MemAccessTy->isVectorTy();
1413
1414 // A store of an extracted vector element will be combined into a VSTE type
1415 // instruction.
1416 if (!IsVectorAccess && isa<StoreInst>(Val: I)) {
1417 Value *DataOp = I->getOperand(i: 0);
1418 if (isa<ExtractElementInst>(Val: DataOp))
1419 IsVectorAccess = true;
1420 }
1421
1422 // A load which gets inserted into a vector element will be combined into a
1423 // VLE type instruction.
1424 if (!IsVectorAccess && isa<LoadInst>(Val: I) && I->hasOneUse()) {
1425 User *LoadUser = *I->user_begin();
1426 if (isa<InsertElementInst>(Val: LoadUser))
1427 IsVectorAccess = true;
1428 }
1429
1430 if (IsFPAccess || IsVectorAccess)
1431 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
1432 }
1433
1434 return AddressingMode(true/*LongDispl*/, true/*IdxReg*/);
1435}
1436
1437bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1438 const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const {
1439 // Punt on globals for now, although they can be used in limited
1440 // RELATIVE LONG cases.
1441 if (AM.BaseGV)
1442 return false;
1443
1444 // Require a 20-bit signed offset.
1445 if (!isInt<20>(x: AM.BaseOffs))
1446 return false;
1447
1448 bool RequireD12 =
1449 Subtarget.hasVector() && (Ty->isVectorTy() || Ty->isIntegerTy(BitWidth: 128));
1450 AddressingMode SupportedAM(!RequireD12, true);
1451 if (I != nullptr)
1452 SupportedAM = supportedAddressingMode(I, HasVector: Subtarget.hasVector());
1453
1454 if (!SupportedAM.LongDisplacement && !isUInt<12>(x: AM.BaseOffs))
1455 return false;
1456
1457 if (!SupportedAM.IndexReg)
1458 // No indexing allowed.
1459 return AM.Scale == 0;
1460 else
1461 // Indexing is OK but no scale factor can be applied.
1462 return AM.Scale == 0 || AM.Scale == 1;
1463}
1464
1465bool SystemZTargetLowering::findOptimalMemOpLowering(
1466 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
1467 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
1468 const AttributeList &FuncAttributes, EVT *LargestVT) const {
1469
1470 assert(Limit != ~0U &&
1471 "Expected EmitTargetCodeForMemXXX() to handle AlwaysInline cases.");
1472
1473 if (Op.isZeroMemset())
1474 return false; // Memset zero: Use XC.
1475
1476 const int MVCFastLen = 16;
1477 // Use MVC up to 16 bytes. Small memset uses STC/MVI for first byte.
1478 if ((Op.isMemset() ? Op.size() - 1 : Op.size()) <= MVCFastLen)
1479 return false;
1480
1481 // Avoid unaligned VL/VST:s.
1482 if (!Op.isAligned(AlignCheck: Align(8)) || (Op.size() >= 25 && Op.size() <= 31))
1483 return false;
1484
1485 return TargetLowering::findOptimalMemOpLowering(
1486 Context, MemOps, Limit, Op, DstAS, SrcAS, FuncAttributes, LargestVT);
1487}
1488
1489EVT SystemZTargetLowering::getOptimalMemOpType(
1490 LLVMContext &Context, const MemOp &Op,
1491 const AttributeList &FuncAttributes) const {
1492 return Subtarget.hasVector() ? MVT::v2i64 : MVT::Other;
1493}
1494
1495bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const {
1496 if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
1497 return false;
1498 unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedValue();
1499 unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedValue();
1500 return FromBits > ToBits;
1501}
1502
1503bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const {
1504 if (!FromVT.isInteger() || !ToVT.isInteger())
1505 return false;
1506 unsigned FromBits = FromVT.getFixedSizeInBits();
1507 unsigned ToBits = ToVT.getFixedSizeInBits();
1508 return FromBits > ToBits;
1509}
1510
1511//===----------------------------------------------------------------------===//
1512// Inline asm support
1513//===----------------------------------------------------------------------===//
1514
1515TargetLowering::ConstraintType
1516SystemZTargetLowering::getConstraintType(StringRef Constraint) const {
1517 if (Constraint.size() == 1) {
1518 switch (Constraint[0]) {
1519 case 'a': // Address register
1520 case 'd': // Data register (equivalent to 'r')
1521 case 'f': // Floating-point register
1522 case 'h': // High-part register
1523 case 'r': // General-purpose register
1524 case 'v': // Vector register
1525 return C_RegisterClass;
1526
1527 case 'Q': // Memory with base and unsigned 12-bit displacement
1528 case 'R': // Likewise, plus an index
1529 case 'S': // Memory with base and signed 20-bit displacement
1530 case 'T': // Likewise, plus an index
1531 case 'm': // Equivalent to 'T'.
1532 return C_Memory;
1533
1534 case 'I': // Unsigned 8-bit constant
1535 case 'J': // Unsigned 12-bit constant
1536 case 'K': // Signed 16-bit constant
1537 case 'L': // Signed 20-bit displacement (on all targets we support)
1538 case 'M': // 0x7fffffff
1539 return C_Immediate;
1540
1541 default:
1542 break;
1543 }
1544 } else if (Constraint.size() == 2 && Constraint[0] == 'Z') {
1545 switch (Constraint[1]) {
1546 case 'Q': // Address with base and unsigned 12-bit displacement
1547 case 'R': // Likewise, plus an index
1548 case 'S': // Address with base and signed 20-bit displacement
1549 case 'T': // Likewise, plus an index
1550 return C_Address;
1551
1552 default:
1553 break;
1554 }
1555 } else if (Constraint.size() == 5 && Constraint.starts_with(Prefix: "{")) {
1556 if (StringRef("{@cc}").compare(RHS: Constraint) == 0)
1557 return C_Other;
1558 }
1559 return TargetLowering::getConstraintType(Constraint);
1560}
1561
1562TargetLowering::ConstraintWeight
1563SystemZTargetLowering::getSingleConstraintMatchWeight(
1564 AsmOperandInfo &Info, const char *Constraint) const {
1565 ConstraintWeight Weight = CW_Invalid;
1566 Value *CallOperandVal = Info.CallOperandVal;
1567 // If we don't have a value, we can't do a match,
1568 // but allow it at the lowest weight.
1569 if (!CallOperandVal)
1570 return CW_Default;
1571 Type *type = CallOperandVal->getType();
1572 // Look at the constraint type.
1573 switch (*Constraint) {
1574 default:
1575 Weight = TargetLowering::getSingleConstraintMatchWeight(info&: Info, constraint: Constraint);
1576 break;
1577
1578 case 'a': // Address register
1579 case 'd': // Data register (equivalent to 'r')
1580 case 'h': // High-part register
1581 case 'r': // General-purpose register
1582 Weight =
1583 CallOperandVal->getType()->isIntegerTy() ? CW_Register : CW_Default;
1584 break;
1585
1586 case 'f': // Floating-point register
1587 if (!useSoftFloat())
1588 Weight = type->isFloatingPointTy() ? CW_Register : CW_Default;
1589 break;
1590
1591 case 'v': // Vector register
1592 if (Subtarget.hasVector())
1593 Weight = (type->isVectorTy() || type->isFloatingPointTy()) ? CW_Register
1594 : CW_Default;
1595 break;
1596
1597 case 'I': // Unsigned 8-bit constant
1598 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1599 if (isUInt<8>(x: C->getZExtValue()))
1600 Weight = CW_Constant;
1601 break;
1602
1603 case 'J': // Unsigned 12-bit constant
1604 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1605 if (isUInt<12>(x: C->getZExtValue()))
1606 Weight = CW_Constant;
1607 break;
1608
1609 case 'K': // Signed 16-bit constant
1610 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1611 if (isInt<16>(x: C->getSExtValue()))
1612 Weight = CW_Constant;
1613 break;
1614
1615 case 'L': // Signed 20-bit displacement (on all targets we support)
1616 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1617 if (isInt<20>(x: C->getSExtValue()))
1618 Weight = CW_Constant;
1619 break;
1620
1621 case 'M': // 0x7fffffff
1622 if (auto *C = dyn_cast<ConstantInt>(Val: CallOperandVal))
1623 if (C->getZExtValue() == 0x7fffffff)
1624 Weight = CW_Constant;
1625 break;
1626 }
1627 return Weight;
1628}
1629
1630// Parse a "{tNNN}" register constraint for which the register type "t"
1631// has already been verified. MC is the class associated with "t" and
1632// Map maps 0-based register numbers to LLVM register numbers.
1633static std::pair<unsigned, const TargetRegisterClass *>
1634parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC,
1635 const unsigned *Map, unsigned Size) {
1636 assert(*(Constraint.end()-1) == '}' && "Missing '}'");
1637 if (isdigit(Constraint[2])) {
1638 unsigned Index;
1639 bool Failed =
1640 Constraint.slice(Start: 2, End: Constraint.size() - 1).getAsInteger(Radix: 10, Result&: Index);
1641 if (!Failed && Index < Size && Map[Index])
1642 return std::make_pair(x: Map[Index], y&: RC);
1643 }
1644 return std::make_pair(x: 0U, y: nullptr);
1645}
1646
1647std::pair<unsigned, const TargetRegisterClass *>
1648SystemZTargetLowering::getRegForInlineAsmConstraint(
1649 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
1650 if (Constraint.size() == 1) {
1651 // GCC Constraint Letters
1652 switch (Constraint[0]) {
1653 default: break;
1654 case 'd': // Data register (equivalent to 'r')
1655 case 'r': // General-purpose register
1656 if (VT.getSizeInBits() == 64)
1657 return std::make_pair(x: 0U, y: &SystemZ::GR64BitRegClass);
1658 else if (VT.getSizeInBits() == 128)
1659 return std::make_pair(x: 0U, y: &SystemZ::GR128BitRegClass);
1660 return std::make_pair(x: 0U, y: &SystemZ::GR32BitRegClass);
1661
1662 case 'a': // Address register
1663 if (VT == MVT::i64)
1664 return std::make_pair(x: 0U, y: &SystemZ::ADDR64BitRegClass);
1665 else if (VT == MVT::i128)
1666 return std::make_pair(x: 0U, y: &SystemZ::ADDR128BitRegClass);
1667 return std::make_pair(x: 0U, y: &SystemZ::ADDR32BitRegClass);
1668
1669 case 'h': // High-part register (an LLVM extension)
1670 return std::make_pair(x: 0U, y: &SystemZ::GRH32BitRegClass);
1671
1672 case 'f': // Floating-point register
1673 if (!useSoftFloat()) {
1674 if (VT.getSizeInBits() == 16)
1675 return std::make_pair(x: 0U, y: &SystemZ::FP16BitRegClass);
1676 else if (VT.getSizeInBits() == 64)
1677 return std::make_pair(x: 0U, y: &SystemZ::FP64BitRegClass);
1678 else if (VT.getSizeInBits() == 128)
1679 return std::make_pair(x: 0U, y: &SystemZ::FP128BitRegClass);
1680 return std::make_pair(x: 0U, y: &SystemZ::FP32BitRegClass);
1681 }
1682 break;
1683
1684 case 'v': // Vector register
1685 if (Subtarget.hasVector()) {
1686 if (VT.getSizeInBits() == 16)
1687 return std::make_pair(x: 0U, y: &SystemZ::VR16BitRegClass);
1688 if (VT.getSizeInBits() == 32)
1689 return std::make_pair(x: 0U, y: &SystemZ::VR32BitRegClass);
1690 if (VT.getSizeInBits() == 64)
1691 return std::make_pair(x: 0U, y: &SystemZ::VR64BitRegClass);
1692 return std::make_pair(x: 0U, y: &SystemZ::VR128BitRegClass);
1693 }
1694 break;
1695 }
1696 }
1697 if (Constraint.starts_with(Prefix: "{")) {
1698
1699 // A clobber constraint (e.g. ~{f0}) will have MVT::Other which is illegal
1700 // to check the size on.
1701 auto getVTSizeInBits = [&VT]() {
1702 return VT == MVT::Other ? 0 : VT.getSizeInBits();
1703 };
1704
1705 // We need to override the default register parsing for GPRs and FPRs
1706 // because the interpretation depends on VT. The internal names of
1707 // the registers are also different from the external names
1708 // (F0D and F0S instead of F0, etc.).
1709 if (Constraint[1] == 'r') {
1710 if (getVTSizeInBits() == 32)
1711 return parseRegisterNumber(Constraint, RC: &SystemZ::GR32BitRegClass,
1712 Map: SystemZMC::GR32Regs, Size: 16);
1713 if (getVTSizeInBits() == 128)
1714 return parseRegisterNumber(Constraint, RC: &SystemZ::GR128BitRegClass,
1715 Map: SystemZMC::GR128Regs, Size: 16);
1716 return parseRegisterNumber(Constraint, RC: &SystemZ::GR64BitRegClass,
1717 Map: SystemZMC::GR64Regs, Size: 16);
1718 }
1719 if (Constraint[1] == 'f') {
1720 if (useSoftFloat())
1721 return std::make_pair(
1722 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1723 if (getVTSizeInBits() == 16)
1724 return parseRegisterNumber(Constraint, RC: &SystemZ::FP16BitRegClass,
1725 Map: SystemZMC::FP16Regs, Size: 16);
1726 if (getVTSizeInBits() == 32)
1727 return parseRegisterNumber(Constraint, RC: &SystemZ::FP32BitRegClass,
1728 Map: SystemZMC::FP32Regs, Size: 16);
1729 if (getVTSizeInBits() == 128)
1730 return parseRegisterNumber(Constraint, RC: &SystemZ::FP128BitRegClass,
1731 Map: SystemZMC::FP128Regs, Size: 16);
1732 return parseRegisterNumber(Constraint, RC: &SystemZ::FP64BitRegClass,
1733 Map: SystemZMC::FP64Regs, Size: 16);
1734 }
1735 if (Constraint[1] == 'v') {
1736 if (!Subtarget.hasVector())
1737 return std::make_pair(
1738 x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
1739 if (getVTSizeInBits() == 16)
1740 return parseRegisterNumber(Constraint, RC: &SystemZ::VR16BitRegClass,
1741 Map: SystemZMC::VR16Regs, Size: 32);
1742 if (getVTSizeInBits() == 32)
1743 return parseRegisterNumber(Constraint, RC: &SystemZ::VR32BitRegClass,
1744 Map: SystemZMC::VR32Regs, Size: 32);
1745 if (getVTSizeInBits() == 64)
1746 return parseRegisterNumber(Constraint, RC: &SystemZ::VR64BitRegClass,
1747 Map: SystemZMC::VR64Regs, Size: 32);
1748 return parseRegisterNumber(Constraint, RC: &SystemZ::VR128BitRegClass,
1749 Map: SystemZMC::VR128Regs, Size: 32);
1750 }
1751 if (Constraint[1] == '@') {
1752 if (StringRef("{@cc}").compare(RHS: Constraint) == 0)
1753 return std::make_pair(x: SystemZ::CC, y: &SystemZ::CCRRegClass);
1754 }
1755 }
1756 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
1757}
1758
1759// FIXME? Maybe this could be a TableGen attribute on some registers and
1760// this table could be generated automatically from RegInfo.
1761Register
1762SystemZTargetLowering::getRegisterByName(const char *RegName, LLT VT,
1763 const MachineFunction &MF) const {
1764 Register Reg =
1765 StringSwitch<Register>(RegName)
1766 .Case(S: "r4", Value: Subtarget.isTargetXPLINK64() ? SystemZ::R4D
1767 : SystemZ::NoRegister)
1768 .Case(S: "r15",
1769 Value: Subtarget.isTargetELF() ? SystemZ::R15D : SystemZ::NoRegister)
1770 .Default(Value: Register());
1771
1772 return Reg;
1773}
1774
1775Register SystemZTargetLowering::getExceptionPointerRegister(
1776 const Constant *PersonalityFn) const {
1777 return Subtarget.isTargetXPLINK64() ? SystemZ::R1D : SystemZ::R6D;
1778}
1779
1780Register SystemZTargetLowering::getExceptionSelectorRegister(
1781 const Constant *PersonalityFn) const {
1782 return Subtarget.isTargetXPLINK64() ? SystemZ::R2D : SystemZ::R7D;
1783}
1784
1785// Convert condition code in CCReg to an i32 value.
1786static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg) {
1787 SDLoc DL(CCReg);
1788 SDValue IPM = DAG.getNode(Opcode: SystemZISD::IPM, DL, VT: MVT::i32, Operand: CCReg);
1789 return DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: IPM,
1790 N2: DAG.getConstant(Val: SystemZ::IPM_CC, DL, VT: MVT::i32));
1791}
1792
1793// Lower @cc targets via setcc.
1794SDValue SystemZTargetLowering::LowerAsmOutputForConstraint(
1795 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
1796 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
1797 if (StringRef("{@cc}").compare(RHS: OpInfo.ConstraintCode) != 0)
1798 return SDValue();
1799
1800 // Check that return type is valid.
1801 if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
1802 OpInfo.ConstraintVT.getSizeInBits() < 8)
1803 report_fatal_error(reason: "Glue output operand is of invalid type");
1804
1805 if (Glue.getNode()) {
1806 Glue = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::CC, VT: MVT::i32, Glue);
1807 Chain = Glue.getValue(R: 1);
1808 } else
1809 Glue = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::CC, VT: MVT::i32);
1810 return getCCResult(DAG, CCReg: Glue);
1811}
1812
1813void SystemZTargetLowering::LowerAsmOperandForConstraint(
1814 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
1815 SelectionDAG &DAG) const {
1816 // Only support length 1 constraints for now.
1817 if (Constraint.size() == 1) {
1818 switch (Constraint[0]) {
1819 case 'I': // Unsigned 8-bit constant
1820 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1821 if (isUInt<8>(x: C->getZExtValue()))
1822 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1823 VT: Op.getValueType()));
1824 return;
1825
1826 case 'J': // Unsigned 12-bit constant
1827 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1828 if (isUInt<12>(x: C->getZExtValue()))
1829 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1830 VT: Op.getValueType()));
1831 return;
1832
1833 case 'K': // Signed 16-bit constant
1834 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1835 if (isInt<16>(x: C->getSExtValue()))
1836 Ops.push_back(x: DAG.getSignedTargetConstant(
1837 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1838 return;
1839
1840 case 'L': // Signed 20-bit displacement (on all targets we support)
1841 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1842 if (isInt<20>(x: C->getSExtValue()))
1843 Ops.push_back(x: DAG.getSignedTargetConstant(
1844 Val: C->getSExtValue(), DL: SDLoc(Op), VT: Op.getValueType()));
1845 return;
1846
1847 case 'M': // 0x7fffffff
1848 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op))
1849 if (C->getZExtValue() == 0x7fffffff)
1850 Ops.push_back(x: DAG.getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(Op),
1851 VT: Op.getValueType()));
1852 return;
1853 }
1854 }
1855 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
1856}
1857
1858//===----------------------------------------------------------------------===//
1859// Calling conventions
1860//===----------------------------------------------------------------------===//
1861
1862#define GET_CALLING_CONV_IMPL
1863#include "SystemZGenCallingConv.inc"
1864
1865const MCPhysReg *SystemZTargetLowering::getScratchRegisters(
1866 CallingConv::ID) const {
1867 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1868 SystemZ::R14D, 0 };
1869 return ScratchRegs;
1870}
1871
1872bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType,
1873 Type *ToType) const {
1874 return isTruncateFree(FromType, ToType);
1875}
1876
1877bool SystemZTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
1878 return CI->isTailCall();
1879}
1880
1881// Value is a value that has been passed to us in the location described by VA
1882// (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining
1883// any loads onto Chain.
1884static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
1885 CCValAssign &VA, SDValue Chain,
1886 SDValue Value) {
1887 // If the argument has been promoted from a smaller type, insert an
1888 // assertion to capture this.
1889 if (VA.getLocInfo() == CCValAssign::SExt)
1890 Value = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Value,
1891 N2: DAG.getValueType(VA.getValVT()));
1892 else if (VA.getLocInfo() == CCValAssign::ZExt)
1893 Value = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Value,
1894 N2: DAG.getValueType(VA.getValVT()));
1895
1896 if (VA.isExtInLoc())
1897 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Value);
1898 else if (VA.getLocInfo() == CCValAssign::BCvt) {
1899 // If this is a short vector argument loaded from the stack,
1900 // extend from i64 to full vector size and then bitcast.
1901 assert(VA.getLocVT() == MVT::i64);
1902 assert(VA.getValVT().isVector());
1903 Value = DAG.getBuildVector(VT: MVT::v2i64, DL, Ops: {Value, DAG.getUNDEF(VT: MVT::i64)});
1904 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Value);
1905 } else
1906 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo");
1907 return Value;
1908}
1909
1910// Value is a value of type VA.getValVT() that we need to copy into
1911// the location described by VA. Return a copy of Value converted to
1912// VA.getValVT(). The caller is responsible for handling indirect values.
1913static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
1914 CCValAssign &VA, SDValue Value) {
1915 switch (VA.getLocInfo()) {
1916 case CCValAssign::SExt:
1917 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1918 case CCValAssign::ZExt:
1919 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1920 case CCValAssign::AExt:
1921 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Value);
1922 case CCValAssign::BCvt: {
1923 assert(VA.getLocVT() == MVT::i64 || VA.getLocVT() == MVT::i128);
1924 assert(VA.getValVT().isVector() || VA.getValVT() == MVT::f32 ||
1925 VA.getValVT() == MVT::f64 || VA.getValVT() == MVT::f128);
1926 // For an f32 vararg we need to first promote it to an f64 and then
1927 // bitcast it to an i64.
1928 if (VA.getValVT() == MVT::f32 && VA.getLocVT() == MVT::i64)
1929 Value = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f64, Operand: Value);
1930 MVT BitCastToType = VA.getValVT().isVector() && VA.getLocVT() == MVT::i64
1931 ? MVT::v2i64
1932 : VA.getLocVT();
1933 Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: BitCastToType, Operand: Value);
1934 // For ELF, this is a short vector argument to be stored to the stack,
1935 // bitcast to v2i64 and then extract first element.
1936 if (BitCastToType == MVT::v2i64)
1937 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: VA.getLocVT(), N1: Value,
1938 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
1939 return Value;
1940 }
1941 case CCValAssign::Full:
1942 return Value;
1943 default:
1944 llvm_unreachable("Unhandled getLocInfo()");
1945 }
1946}
1947
1948static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
1949 SDLoc DL(In);
1950 SDValue Lo, Hi;
1951 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
1952 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64, Operand: In);
1953 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64,
1954 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i128, N1: In,
1955 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32)));
1956 } else {
1957 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: In, DL, LoVT: MVT::i64, HiVT: MVT::i64);
1958 }
1959
1960 // FIXME: If v2i64 were a legal type, we could use it instead of
1961 // Untyped here. This might enable improved folding.
1962 SDNode *Pair = DAG.getMachineNode(Opcode: SystemZ::PAIR128, dl: DL,
1963 VT: MVT::Untyped, Op1: Hi, Op2: Lo);
1964 return SDValue(Pair, 0);
1965}
1966
1967static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
1968 SDLoc DL(In);
1969 SDValue Hi = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64,
1970 DL, VT: MVT::i64, Operand: In);
1971 SDValue Lo = DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64,
1972 DL, VT: MVT::i64, Operand: In);
1973
1974 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128)) {
1975 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Lo);
1976 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i128, Operand: Hi);
1977 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i128, N1: Hi,
1978 N2: DAG.getConstant(Val: 64, DL, VT: MVT::i32));
1979 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
1980 } else {
1981 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i128, N1: Lo, N2: Hi);
1982 }
1983}
1984
1985bool SystemZTargetLowering::splitValueIntoRegisterParts(
1986 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
1987 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
1988 EVT ValueVT = Val.getValueType();
1989 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
1990 // Inline assembly operand.
1991 Parts[0] = lowerI128ToGR128(DAG, In: DAG.getBitcast(VT: MVT::i128, V: Val));
1992 return true;
1993 }
1994
1995 return false;
1996}
1997
1998SDValue SystemZTargetLowering::joinRegisterPartsIntoValue(
1999 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
2000 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
2001 if (ValueVT.getSizeInBits() == 128 && NumParts == 1 && PartVT == MVT::Untyped) {
2002 // Inline assembly operand.
2003 SDValue Res = lowerGR128ToI128(DAG, In: Parts[0]);
2004 return DAG.getBitcast(VT: ValueVT, V: Res);
2005 }
2006
2007 return SDValue();
2008}
2009
2010// The first part of a split stack argument is at index I in Args (and
2011// ArgLocs). Return the type of a part and the number of them by reference.
2012template <class ArgTy>
2013static bool analyzeArgSplit(const SmallVectorImpl<ArgTy> &Args,
2014 SmallVector<CCValAssign, 16> &ArgLocs, unsigned I,
2015 MVT &PartVT, unsigned &NumParts) {
2016 if (!Args[I].Flags.isSplit())
2017 return false;
2018 assert(I < ArgLocs.size() && ArgLocs.size() == Args.size() &&
2019 "ArgLocs havoc.");
2020 PartVT = ArgLocs[I].getValVT();
2021 NumParts = 1;
2022 for (unsigned PartIdx = I + 1;; ++PartIdx) {
2023 assert(PartIdx != ArgLocs.size() && "SplitEnd not found.");
2024 assert(ArgLocs[PartIdx].getValVT() == PartVT && "Unsupported split.");
2025 ++NumParts;
2026 if (Args[PartIdx].Flags.isSplitEnd())
2027 break;
2028 }
2029 return true;
2030}
2031
2032SDValue SystemZTargetLowering::LowerFormalArguments(
2033 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
2034 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2035 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2036 MachineFunction &MF = DAG.getMachineFunction();
2037 MachineFrameInfo &MFI = MF.getFrameInfo();
2038 MachineRegisterInfo &MRI = MF.getRegInfo();
2039 SystemZMachineFunctionInfo *FuncInfo =
2040 MF.getInfo<SystemZMachineFunctionInfo>();
2041 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
2042 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
2043
2044 // Assign locations to all of the incoming arguments.
2045 SmallVector<CCValAssign, 16> ArgLocs;
2046 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2047 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_SystemZ);
2048 FuncInfo->setSizeOfFnParams(CCInfo.getStackSize());
2049
2050 unsigned NumFixedGPRs = 0;
2051 unsigned NumFixedFPRs = 0;
2052 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2053 SDValue ArgValue;
2054 CCValAssign &VA = ArgLocs[I];
2055 EVT LocVT = VA.getLocVT();
2056 if (VA.isRegLoc()) {
2057 // Arguments passed in registers
2058 const TargetRegisterClass *RC;
2059 switch (LocVT.getSimpleVT().SimpleTy) {
2060 default:
2061 // Integers smaller than i64 should be promoted to i64.
2062 llvm_unreachable("Unexpected argument type");
2063 case MVT::i32:
2064 NumFixedGPRs += 1;
2065 RC = &SystemZ::GR32BitRegClass;
2066 break;
2067 case MVT::i64:
2068 NumFixedGPRs += 1;
2069 RC = &SystemZ::GR64BitRegClass;
2070 break;
2071 case MVT::f16:
2072 NumFixedFPRs += 1;
2073 RC = &SystemZ::FP16BitRegClass;
2074 break;
2075 case MVT::f32:
2076 NumFixedFPRs += 1;
2077 RC = &SystemZ::FP32BitRegClass;
2078 break;
2079 case MVT::f64:
2080 NumFixedFPRs += 1;
2081 RC = &SystemZ::FP64BitRegClass;
2082 break;
2083 case MVT::f128:
2084 NumFixedFPRs += 2;
2085 RC = &SystemZ::FP128BitRegClass;
2086 break;
2087 case MVT::v16i8:
2088 case MVT::v8i16:
2089 case MVT::v4i32:
2090 case MVT::v2i64:
2091 case MVT::v8f16:
2092 case MVT::v4f32:
2093 case MVT::v2f64:
2094 RC = &SystemZ::VR128BitRegClass;
2095 break;
2096 }
2097
2098 Register VReg = MRI.createVirtualRegister(RegClass: RC);
2099 MRI.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
2100 ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
2101 } else {
2102 assert(VA.isMemLoc() && "Argument not register or memory");
2103
2104 // Create the frame index object for this incoming parameter.
2105 // FIXME: Pre-include call frame size in the offset, should not
2106 // need to manually add it here.
2107 int64_t ArgSPOffset = VA.getLocMemOffset();
2108 if (Subtarget.isTargetXPLINK64()) {
2109 auto &XPRegs =
2110 Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
2111 ArgSPOffset += XPRegs.getCallFrameSize();
2112 }
2113 int FI =
2114 MFI.CreateFixedObject(Size: LocVT.getSizeInBits() / 8, SPOffset: ArgSPOffset, IsImmutable: true);
2115
2116 // Create the SelectionDAG nodes corresponding to a load
2117 // from this parameter. Unpromoted ints and floats are
2118 // passed as right-justified 8-byte values.
2119 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
2120 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32 ||
2121 VA.getLocVT() == MVT::f16) {
2122 unsigned SlotOffs = VA.getLocVT() == MVT::f16 ? 6 : 4;
2123 FIN = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FIN,
2124 N2: DAG.getIntPtrConstant(Val: SlotOffs, DL));
2125 }
2126 ArgValue = DAG.getLoad(VT: LocVT, dl: DL, Chain, Ptr: FIN,
2127 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2128 }
2129
2130 // Convert the value of the argument register into the value that's
2131 // being passed.
2132 if (VA.getLocInfo() == CCValAssign::Indirect) {
2133 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
2134 PtrInfo: MachinePointerInfo()));
2135 // If the original argument was split (e.g. i128), we need
2136 // to load all parts of it here (using the same address).
2137 MVT PartVT;
2138 unsigned NumParts;
2139 if (analyzeArgSplit(Args: Ins, ArgLocs, I, PartVT, NumParts)) {
2140 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2141 ++I;
2142 CCValAssign &PartVA = ArgLocs[I];
2143 unsigned PartOffset = Ins[I].PartOffset;
2144 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue,
2145 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2146 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
2147 PtrInfo: MachinePointerInfo()));
2148 assert(PartOffset && "Offset should be non-zero.");
2149 }
2150 }
2151 } else
2152 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: ArgValue));
2153 }
2154
2155 if (IsVarArg && Subtarget.isTargetXPLINK64()) {
2156 // Save the number of non-varargs registers for later use by va_start, etc.
2157 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2158 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2159
2160 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2161 Subtarget.getSpecialRegisters());
2162
2163 // Likewise the address (in the form of a frame index) of where the
2164 // first stack vararg would be. The 1-byte size here is arbitrary.
2165 // FIXME: Pre-include call frame size in the offset, should not
2166 // need to manually add it here.
2167 int64_t VarArgOffset = CCInfo.getStackSize() + Regs->getCallFrameSize();
2168 int FI = MFI.CreateFixedObject(Size: 1, SPOffset: VarArgOffset, IsImmutable: true);
2169 FuncInfo->setVarArgsFrameIndex(FI);
2170 }
2171
2172 if (IsVarArg && Subtarget.isTargetELF()) {
2173 // Save the number of non-varargs registers for later use by va_start, etc.
2174 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
2175 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
2176
2177 // Likewise the address (in the form of a frame index) of where the
2178 // first stack vararg would be. The 1-byte size here is arbitrary.
2179 int64_t VarArgsOffset = CCInfo.getStackSize();
2180 FuncInfo->setVarArgsFrameIndex(
2181 MFI.CreateFixedObject(Size: 1, SPOffset: VarArgsOffset, IsImmutable: true));
2182
2183 // ...and a similar frame index for the caller-allocated save area
2184 // that will be used to store the incoming registers.
2185 int64_t RegSaveOffset =
2186 -SystemZMC::ELFCallFrameSize + TFL->getRegSpillOffset(MF, Reg: SystemZ::R2D) - 16;
2187 unsigned RegSaveIndex = MFI.CreateFixedObject(Size: 1, SPOffset: RegSaveOffset, IsImmutable: true);
2188 FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
2189
2190 // Store the FPR varargs in the reserved frame slots. (We store the
2191 // GPRs as part of the prologue.)
2192 if (NumFixedFPRs < SystemZ::ELFNumArgFPRs && !useSoftFloat()) {
2193 SDValue MemOps[SystemZ::ELFNumArgFPRs];
2194 for (unsigned I = NumFixedFPRs; I < SystemZ::ELFNumArgFPRs; ++I) {
2195 unsigned Offset = TFL->getRegSpillOffset(MF, Reg: SystemZ::ELFArgFPRs[I]);
2196 int FI =
2197 MFI.CreateFixedObject(Size: 8, SPOffset: -SystemZMC::ELFCallFrameSize + Offset, IsImmutable: true);
2198 SDValue FIN = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
2199 Register VReg = MF.addLiveIn(PReg: SystemZ::ELFArgFPRs[I],
2200 RC: &SystemZ::FP64BitRegClass);
2201 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::f64);
2202 MemOps[I] = DAG.getStore(Chain: ArgValue.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: FIN,
2203 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
2204 }
2205 // Join the stores, which are independent of one another.
2206 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
2207 Ops: ArrayRef(&MemOps[NumFixedFPRs],
2208 SystemZ::ELFNumArgFPRs - NumFixedFPRs));
2209 }
2210 }
2211
2212 if (Subtarget.isTargetXPLINK64()) {
2213 // Create virual register for handling incoming "ADA" special register (R5)
2214 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
2215 Register ADAvReg = MRI.createVirtualRegister(RegClass: RC);
2216 auto *Regs = static_cast<SystemZXPLINK64Registers *>(
2217 Subtarget.getSpecialRegisters());
2218 MRI.addLiveIn(Reg: Regs->getADARegister(), vreg: ADAvReg);
2219 FuncInfo->setADAVirtualRegister(ADAvReg);
2220 }
2221 return Chain;
2222}
2223
2224static bool canUseSiblingCall(const CCState &ArgCCInfo,
2225 SmallVectorImpl<CCValAssign> &ArgLocs,
2226 SmallVectorImpl<ISD::OutputArg> &Outs) {
2227 // Punt if there are any indirect or stack arguments, or if the call
2228 // needs the callee-saved argument register R6, or if the call uses
2229 // the callee-saved register arguments SwiftSelf and SwiftError.
2230 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2231 CCValAssign &VA = ArgLocs[I];
2232 if (VA.getLocInfo() == CCValAssign::Indirect)
2233 return false;
2234 if (!VA.isRegLoc())
2235 return false;
2236 Register Reg = VA.getLocReg();
2237 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
2238 return false;
2239 if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError())
2240 return false;
2241 }
2242 return true;
2243}
2244
2245static SDValue getADAEntry(SelectionDAG &DAG, SDValue Val, SDLoc DL,
2246 unsigned Offset, bool LoadAdr = false) {
2247 MachineFunction &MF = DAG.getMachineFunction();
2248 SystemZMachineFunctionInfo *MFI = MF.getInfo<SystemZMachineFunctionInfo>();
2249 Register ADAvReg = MFI->getADAVirtualRegister();
2250 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2251
2252 SDValue Reg = DAG.getRegister(Reg: ADAvReg, VT: PtrVT);
2253 SDValue Ofs = DAG.getTargetConstant(Val: Offset, DL, VT: PtrVT);
2254
2255 SDValue Result = DAG.getNode(Opcode: SystemZISD::ADA_ENTRY, DL, VT: PtrVT, N1: Val, N2: Reg, N3: Ofs);
2256 if (!LoadAdr)
2257 Result = DAG.getLoad(
2258 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result, PtrInfo: MachinePointerInfo(), Alignment: Align(8),
2259 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2260
2261 return Result;
2262}
2263
2264// ADA access using Global value
2265// Note: for functions, address of descriptor is returned
2266static SDValue getADAEntry(SelectionDAG &DAG, const GlobalValue *GV, SDLoc DL,
2267 EVT PtrVT) {
2268 unsigned ADAtype;
2269 bool LoadAddr = false;
2270 const GlobalAlias *GA = dyn_cast<GlobalAlias>(Val: GV);
2271 bool IsFunction =
2272 (isa<Function>(Val: GV)) || (GA && isa<Function>(Val: GA->getAliaseeObject()));
2273 bool IsInternal = (GV->hasInternalLinkage() || GV->hasPrivateLinkage());
2274
2275 if (IsFunction) {
2276 if (IsInternal) {
2277 ADAtype = SystemZII::MO_ADA_DIRECT_FUNC_DESC;
2278 LoadAddr = true;
2279 } else
2280 ADAtype = SystemZII::MO_ADA_INDIRECT_FUNC_DESC;
2281 } else {
2282 ADAtype = SystemZII::MO_ADA_DATA_SYMBOL_ADDR;
2283 }
2284 SDValue Val = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: ADAtype);
2285
2286 return getADAEntry(DAG, Val, DL, Offset: 0, LoadAdr: LoadAddr);
2287}
2288
2289static bool getzOSCalleeAndADA(SelectionDAG &DAG, SDValue &Callee, SDValue &ADA,
2290 SDLoc &DL, SDValue &Chain) {
2291 unsigned ADADelta = 0; // ADA offset in desc.
2292 unsigned EPADelta = 8; // EPA offset in desc.
2293 MachineFunction &MF = DAG.getMachineFunction();
2294 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
2295
2296 // XPLink calling convention.
2297 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2298 bool IsInternal = (G->getGlobal()->hasInternalLinkage() ||
2299 G->getGlobal()->hasPrivateLinkage());
2300 if (IsInternal) {
2301 SystemZMachineFunctionInfo *MFI =
2302 MF.getInfo<SystemZMachineFunctionInfo>();
2303 Register ADAvReg = MFI->getADAVirtualRegister();
2304 ADA = DAG.getCopyFromReg(Chain, dl: DL, Reg: ADAvReg, VT: PtrVT);
2305 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2306 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2307 return true;
2308 } else {
2309 SDValue GA = DAG.getTargetGlobalAddress(
2310 GV: G->getGlobal(), DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2311 ADA = getADAEntry(DAG, Val: GA, DL, Offset: ADADelta);
2312 Callee = getADAEntry(DAG, Val: GA, DL, Offset: EPADelta);
2313 }
2314 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2315 SDValue ES = DAG.getTargetExternalSymbol(
2316 Sym: E->getSymbol(), VT: PtrVT, TargetFlags: SystemZII::MO_ADA_DIRECT_FUNC_DESC);
2317 ADA = getADAEntry(DAG, Val: ES, DL, Offset: ADADelta);
2318 Callee = getADAEntry(DAG, Val: ES, DL, Offset: EPADelta);
2319 } else {
2320 // Function pointer case
2321 ADA = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2322 N2: DAG.getConstant(Val: ADADelta, DL, VT: PtrVT));
2323 ADA = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: ADA,
2324 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2325 Callee = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Callee,
2326 N2: DAG.getConstant(Val: EPADelta, DL, VT: PtrVT));
2327 Callee = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Callee,
2328 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
2329 }
2330 return false;
2331}
2332
2333SDValue
2334SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
2335 SmallVectorImpl<SDValue> &InVals) const {
2336 SelectionDAG &DAG = CLI.DAG;
2337 SDLoc &DL = CLI.DL;
2338 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2339 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2340 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2341 SDValue Chain = CLI.Chain;
2342 SDValue Callee = CLI.Callee;
2343 bool &IsTailCall = CLI.IsTailCall;
2344 CallingConv::ID CallConv = CLI.CallConv;
2345 bool IsVarArg = CLI.IsVarArg;
2346 MachineFunction &MF = DAG.getMachineFunction();
2347 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
2348 LLVMContext &Ctx = *DAG.getContext();
2349 SystemZCallingConventionRegisters *Regs = Subtarget.getSpecialRegisters();
2350
2351 // FIXME: z/OS support to be added in later.
2352 if (Subtarget.isTargetXPLINK64())
2353 IsTailCall = false;
2354
2355 // Integer args <=32 bits should have an extension attribute.
2356 verifyNarrowIntegerArgs_Call(Outs, F: &MF.getFunction(), Callee);
2357
2358 // Analyze the operands of the call, assigning locations to each operand.
2359 SmallVector<CCValAssign, 16> ArgLocs;
2360 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
2361 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_SystemZ);
2362
2363 // We don't support GuaranteedTailCallOpt, only automatically-detected
2364 // sibling calls.
2365 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs))
2366 IsTailCall = false;
2367
2368 // Get a count of how many bytes are to be pushed on the stack.
2369 unsigned NumBytes = ArgCCInfo.getStackSize();
2370
2371 // Mark the start of the call.
2372 if (!IsTailCall)
2373 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL);
2374
2375 // Copy argument values to their designated locations.
2376 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass;
2377 SmallVector<SDValue, 8> MemOpChains;
2378 SDValue StackPtr;
2379 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
2380 CCValAssign &VA = ArgLocs[I];
2381 SDValue ArgValue = OutVals[I];
2382
2383 if (VA.getLocInfo() == CCValAssign::Indirect) {
2384 // Store the argument in a stack slot and pass its address.
2385 EVT SlotVT;
2386 MVT PartVT;
2387 unsigned NumParts = 1;
2388 if (analyzeArgSplit(Args: Outs, ArgLocs, I, PartVT, NumParts))
2389 SlotVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: PartVT.getSizeInBits() * NumParts);
2390 else
2391 SlotVT = Outs[I].VT;
2392 SDValue SpillSlot = DAG.CreateStackTemporary(VT: SlotVT);
2393 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
2394
2395 MachinePointerInfo StackPtrInfo =
2396 MachinePointerInfo::getFixedStack(MF, FI);
2397 MemOpChains.push_back(
2398 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot, PtrInfo: StackPtrInfo));
2399 // If the original argument was split (e.g. i128), we need
2400 // to store all parts of it here (and pass just one address).
2401 assert(Outs[I].PartOffset == 0);
2402 for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) {
2403 ++I;
2404 SDValue PartValue = OutVals[I];
2405 unsigned PartOffset = Outs[I].PartOffset;
2406 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot,
2407 N2: DAG.getIntPtrConstant(Val: PartOffset, DL));
2408 MemOpChains.push_back(
2409 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
2410 PtrInfo: StackPtrInfo.getWithOffset(O: PartOffset)));
2411 assert(PartOffset && "Offset should be non-zero.");
2412 assert((PartOffset + PartValue.getValueType().getStoreSize() <=
2413 SlotVT.getStoreSize()) && "Not enough space for argument part!");
2414 }
2415 ArgValue = SpillSlot;
2416 } else
2417 ArgValue = convertValVTToLocVT(DAG, DL, VA, Value: ArgValue);
2418
2419 if (VA.isRegLoc()) {
2420 // In XPLINK64, for the 128-bit vararg case, ArgValue is bitcasted to a
2421 // MVT::i128 type. We decompose the 128-bit type to a pair of its high
2422 // and low values.
2423 if (VA.getLocVT() == MVT::i128)
2424 ArgValue = lowerI128ToGR128(DAG, In: ArgValue);
2425 // Queue up the argument copies and emit them at the end.
2426 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
2427 } else {
2428 assert(VA.isMemLoc() && "Argument not register or memory");
2429
2430 // Work out the address of the stack slot. Unpromoted ints and
2431 // floats are passed as right-justified 8-byte values.
2432 if (!StackPtr.getNode())
2433 StackPtr = DAG.getCopyFromReg(Chain, dl: DL,
2434 Reg: Regs->getStackPointerRegister(), VT: PtrVT);
2435 unsigned Offset = Regs->getStackPointerBias() + Regs->getCallFrameSize() +
2436 VA.getLocMemOffset();
2437 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
2438 Offset += 4;
2439 else if (VA.getLocVT() == MVT::f16)
2440 Offset += 6;
2441 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
2442 N2: DAG.getIntPtrConstant(Val: Offset, DL));
2443
2444 // Emit the store.
2445 MemOpChains.push_back(
2446 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address, PtrInfo: MachinePointerInfo()));
2447
2448 // Although long doubles or vectors are passed through the stack when
2449 // they are vararg (non-fixed arguments), if a long double or vector
2450 // occupies the third and fourth slot of the argument list GPR3 should
2451 // still shadow the third slot of the argument list.
2452 if (Subtarget.isTargetXPLINK64() && VA.needsCustom()) {
2453 SDValue ShadowArgValue =
2454 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: ArgValue,
2455 N2: DAG.getIntPtrConstant(Val: 1, DL));
2456 RegsToPass.push_back(Elt: std::make_pair(x: SystemZ::R3D, y&: ShadowArgValue));
2457 }
2458 }
2459 }
2460
2461 // Join the stores, which are independent of one another.
2462 if (!MemOpChains.empty())
2463 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
2464
2465 // Accept direct calls by converting symbolic call addresses to the
2466 // associated Target* opcodes. Force %r1 to be used for indirect
2467 // tail calls.
2468 SDValue Glue;
2469
2470 if (Subtarget.isTargetXPLINK64()) {
2471 SDValue ADA;
2472 bool IsBRASL = getzOSCalleeAndADA(DAG, Callee, ADA, DL, Chain);
2473 if (!IsBRASL) {
2474 unsigned CalleeReg = static_cast<SystemZXPLINK64Registers *>(Regs)
2475 ->getAddressOfCalleeRegister();
2476 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CalleeReg, N: Callee, Glue);
2477 Glue = Chain.getValue(R: 1);
2478 Callee = DAG.getRegister(Reg: CalleeReg, VT: Callee.getValueType());
2479 }
2480 RegsToPass.push_back(Elt: std::make_pair(
2481 x: static_cast<SystemZXPLINK64Registers *>(Regs)->getADARegister(), y&: ADA));
2482 } else {
2483 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
2484 Callee = DAG.getTargetGlobalAddress(GV: G->getGlobal(), DL, VT: PtrVT);
2485 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2486 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
2487 Callee = DAG.getTargetExternalSymbol(Sym: E->getSymbol(), VT: PtrVT);
2488 Callee = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Callee);
2489 } else if (IsTailCall) {
2490 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R1D, N: Callee, Glue);
2491 Glue = Chain.getValue(R: 1);
2492 Callee = DAG.getRegister(Reg: SystemZ::R1D, VT: Callee.getValueType());
2493 }
2494 }
2495
2496 // Build a sequence of copy-to-reg nodes, chained and glued together.
2497 for (const auto &[Reg, N] : RegsToPass) {
2498 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N, Glue);
2499 Glue = Chain.getValue(R: 1);
2500 }
2501
2502 // The first call operand is the chain and the second is the target address.
2503 SmallVector<SDValue, 8> Ops;
2504 Ops.push_back(Elt: Chain);
2505 Ops.push_back(Elt: Callee);
2506
2507 // Add argument registers to the end of the list so that they are
2508 // known live into the call.
2509 for (const auto &[Reg, N] : RegsToPass)
2510 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
2511
2512 // Add a register mask operand representing the call-preserved registers.
2513 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2514 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2515 assert(Mask && "Missing call preserved mask for calling convention");
2516 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
2517
2518 // Glue the call to the argument copies, if any.
2519 if (Glue.getNode())
2520 Ops.push_back(Elt: Glue);
2521
2522 // Emit the call.
2523 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
2524 if (IsTailCall) {
2525 SDValue Ret = DAG.getNode(Opcode: SystemZISD::SIBCALL, DL, VTList: NodeTys, Ops);
2526 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
2527 return Ret;
2528 }
2529 Chain = DAG.getNode(Opcode: SystemZISD::CALL, DL, VTList: NodeTys, Ops);
2530 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
2531 Glue = Chain.getValue(R: 1);
2532
2533 // Mark the end of the call, which is glued to the call itself.
2534 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
2535 Glue = Chain.getValue(R: 1);
2536
2537 // Assign locations to each value returned by this call.
2538 SmallVector<CCValAssign, 16> RetLocs;
2539 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
2540 RetCCInfo.AnalyzeCallResult(Ins, Fn: RetCC_SystemZ);
2541
2542 // Copy all of the result registers out of their specified physreg.
2543 for (CCValAssign &VA : RetLocs) {
2544 // Copy the value out, gluing the copy to the end of the call sequence.
2545 SDValue RetValue = DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(),
2546 VT: VA.getLocVT(), Glue);
2547 Chain = RetValue.getValue(R: 1);
2548 Glue = RetValue.getValue(R: 2);
2549
2550 // Convert the value of the return register into the value that's
2551 // being returned.
2552 InVals.push_back(Elt: convertLocVTToValVT(DAG, DL, VA, Chain, Value: RetValue));
2553 }
2554
2555 return Chain;
2556}
2557
2558// Generate a call taking the given operands as arguments and returning a
2559// result of type RetVT.
2560std::pair<SDValue, SDValue> SystemZTargetLowering::makeExternalCall(
2561 SDValue Chain, SelectionDAG &DAG, const char *CalleeName, EVT RetVT,
2562 ArrayRef<SDValue> Ops, CallingConv::ID CallConv, bool IsSigned, SDLoc DL,
2563 bool DoesNotReturn, bool IsReturnValueUsed) const {
2564 TargetLowering::ArgListTy Args;
2565 Args.reserve(n: Ops.size());
2566
2567 for (SDValue Op : Ops) {
2568 TargetLowering::ArgListEntry Entry(
2569 Op, Op.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
2570 Entry.IsSExt = shouldSignExtendTypeInLibCall(Ty: Entry.Ty, IsSigned);
2571 Entry.IsZExt = !Entry.IsSExt;
2572 Args.push_back(x: Entry);
2573 }
2574
2575 SDValue Callee =
2576 DAG.getExternalSymbol(Sym: CalleeName, VT: getPointerTy(DL: DAG.getDataLayout()));
2577
2578 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2579 TargetLowering::CallLoweringInfo CLI(DAG);
2580 bool SignExtend = shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2581 CLI.setDebugLoc(DL)
2582 .setChain(Chain)
2583 .setCallee(CC: CallConv, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2584 .setNoReturn(DoesNotReturn)
2585 .setDiscardResult(!IsReturnValueUsed)
2586 .setSExtResult(SignExtend)
2587 .setZExtResult(!SignExtend);
2588 return LowerCallTo(CLI);
2589}
2590
2591bool SystemZTargetLowering::CanLowerReturn(
2592 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2593 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
2594 const Type *RetTy) const {
2595 // Special case that we cannot easily detect in RetCC_SystemZ since
2596 // i128 may not be a legal type.
2597 for (auto &Out : Outs)
2598 if (Out.ArgVT.isScalarInteger() && Out.ArgVT.getSizeInBits() > 64)
2599 return false;
2600
2601 SmallVector<CCValAssign, 16> RetLocs;
2602 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Context);
2603 return RetCCInfo.CheckReturn(Outs, Fn: RetCC_SystemZ);
2604}
2605
2606SDValue
2607SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2608 bool IsVarArg,
2609 const SmallVectorImpl<ISD::OutputArg> &Outs,
2610 const SmallVectorImpl<SDValue> &OutVals,
2611 const SDLoc &DL, SelectionDAG &DAG) const {
2612 MachineFunction &MF = DAG.getMachineFunction();
2613
2614 // Integer args <=32 bits should have an extension attribute.
2615 verifyNarrowIntegerArgs_Ret(Outs, F: &MF.getFunction());
2616
2617 // Assign locations to each returned value.
2618 SmallVector<CCValAssign, 16> RetLocs;
2619 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
2620 RetCCInfo.AnalyzeReturn(Outs, Fn: RetCC_SystemZ);
2621
2622 // Quick exit for void returns
2623 if (RetLocs.empty())
2624 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Operand: Chain);
2625
2626 if (CallConv == CallingConv::GHC)
2627 report_fatal_error(reason: "GHC functions return void only");
2628
2629 // Copy the result values into the output registers.
2630 SDValue Glue;
2631 SmallVector<SDValue, 4> RetOps;
2632 RetOps.push_back(Elt: Chain);
2633 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
2634 CCValAssign &VA = RetLocs[I];
2635 SDValue RetValue = OutVals[I];
2636
2637 // Make the return register live on exit.
2638 assert(VA.isRegLoc() && "Can only return in registers!");
2639
2640 // Promote the value as required.
2641 RetValue = convertValVTToLocVT(DAG, DL, VA, Value: RetValue);
2642
2643 // Chain and glue the copies together.
2644 Register Reg = VA.getLocReg();
2645 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: RetValue, Glue);
2646 Glue = Chain.getValue(R: 1);
2647 RetOps.push_back(Elt: DAG.getRegister(Reg, VT: VA.getLocVT()));
2648 }
2649
2650 // Update chain and glue.
2651 RetOps[0] = Chain;
2652 if (Glue.getNode())
2653 RetOps.push_back(Elt: Glue);
2654
2655 return DAG.getNode(Opcode: SystemZISD::RET_GLUE, DL, VT: MVT::Other, Ops: RetOps);
2656}
2657
2658// Return true if Op is an intrinsic node with chain that returns the CC value
2659// as its only (other) argument. Provide the associated SystemZISD opcode and
2660// the mask of valid CC values if so.
2661static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
2662 unsigned &CCValid) {
2663 unsigned Id = Op.getConstantOperandVal(i: 1);
2664 switch (Id) {
2665 case Intrinsic::s390_tbegin:
2666 Opcode = SystemZISD::TBEGIN;
2667 CCValid = SystemZ::CCMASK_TBEGIN;
2668 return true;
2669
2670 case Intrinsic::s390_tbegin_nofloat:
2671 Opcode = SystemZISD::TBEGIN_NOFLOAT;
2672 CCValid = SystemZ::CCMASK_TBEGIN;
2673 return true;
2674
2675 case Intrinsic::s390_tend:
2676 Opcode = SystemZISD::TEND;
2677 CCValid = SystemZ::CCMASK_TEND;
2678 return true;
2679
2680 default:
2681 return false;
2682 }
2683}
2684
2685// Return true if Op is an intrinsic node without chain that returns the
2686// CC value as its final argument. Provide the associated SystemZISD
2687// opcode and the mask of valid CC values if so.
2688static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
2689 unsigned Id = Op.getConstantOperandVal(i: 0);
2690 switch (Id) {
2691 case Intrinsic::s390_vpkshs:
2692 case Intrinsic::s390_vpksfs:
2693 case Intrinsic::s390_vpksgs:
2694 Opcode = SystemZISD::PACKS_CC;
2695 CCValid = SystemZ::CCMASK_VCMP;
2696 return true;
2697
2698 case Intrinsic::s390_vpklshs:
2699 case Intrinsic::s390_vpklsfs:
2700 case Intrinsic::s390_vpklsgs:
2701 Opcode = SystemZISD::PACKLS_CC;
2702 CCValid = SystemZ::CCMASK_VCMP;
2703 return true;
2704
2705 case Intrinsic::s390_vceqbs:
2706 case Intrinsic::s390_vceqhs:
2707 case Intrinsic::s390_vceqfs:
2708 case Intrinsic::s390_vceqgs:
2709 case Intrinsic::s390_vceqqs:
2710 Opcode = SystemZISD::VICMPES;
2711 CCValid = SystemZ::CCMASK_VCMP;
2712 return true;
2713
2714 case Intrinsic::s390_vchbs:
2715 case Intrinsic::s390_vchhs:
2716 case Intrinsic::s390_vchfs:
2717 case Intrinsic::s390_vchgs:
2718 case Intrinsic::s390_vchqs:
2719 Opcode = SystemZISD::VICMPHS;
2720 CCValid = SystemZ::CCMASK_VCMP;
2721 return true;
2722
2723 case Intrinsic::s390_vchlbs:
2724 case Intrinsic::s390_vchlhs:
2725 case Intrinsic::s390_vchlfs:
2726 case Intrinsic::s390_vchlgs:
2727 case Intrinsic::s390_vchlqs:
2728 Opcode = SystemZISD::VICMPHLS;
2729 CCValid = SystemZ::CCMASK_VCMP;
2730 return true;
2731
2732 case Intrinsic::s390_vtm:
2733 Opcode = SystemZISD::VTM;
2734 CCValid = SystemZ::CCMASK_VCMP;
2735 return true;
2736
2737 case Intrinsic::s390_vfaebs:
2738 case Intrinsic::s390_vfaehs:
2739 case Intrinsic::s390_vfaefs:
2740 Opcode = SystemZISD::VFAE_CC;
2741 CCValid = SystemZ::CCMASK_ANY;
2742 return true;
2743
2744 case Intrinsic::s390_vfaezbs:
2745 case Intrinsic::s390_vfaezhs:
2746 case Intrinsic::s390_vfaezfs:
2747 Opcode = SystemZISD::VFAEZ_CC;
2748 CCValid = SystemZ::CCMASK_ANY;
2749 return true;
2750
2751 case Intrinsic::s390_vfeebs:
2752 case Intrinsic::s390_vfeehs:
2753 case Intrinsic::s390_vfeefs:
2754 Opcode = SystemZISD::VFEE_CC;
2755 CCValid = SystemZ::CCMASK_ANY;
2756 return true;
2757
2758 case Intrinsic::s390_vfeezbs:
2759 case Intrinsic::s390_vfeezhs:
2760 case Intrinsic::s390_vfeezfs:
2761 Opcode = SystemZISD::VFEEZ_CC;
2762 CCValid = SystemZ::CCMASK_ANY;
2763 return true;
2764
2765 case Intrinsic::s390_vfenebs:
2766 case Intrinsic::s390_vfenehs:
2767 case Intrinsic::s390_vfenefs:
2768 Opcode = SystemZISD::VFENE_CC;
2769 CCValid = SystemZ::CCMASK_ANY;
2770 return true;
2771
2772 case Intrinsic::s390_vfenezbs:
2773 case Intrinsic::s390_vfenezhs:
2774 case Intrinsic::s390_vfenezfs:
2775 Opcode = SystemZISD::VFENEZ_CC;
2776 CCValid = SystemZ::CCMASK_ANY;
2777 return true;
2778
2779 case Intrinsic::s390_vistrbs:
2780 case Intrinsic::s390_vistrhs:
2781 case Intrinsic::s390_vistrfs:
2782 Opcode = SystemZISD::VISTR_CC;
2783 CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3;
2784 return true;
2785
2786 case Intrinsic::s390_vstrcbs:
2787 case Intrinsic::s390_vstrchs:
2788 case Intrinsic::s390_vstrcfs:
2789 Opcode = SystemZISD::VSTRC_CC;
2790 CCValid = SystemZ::CCMASK_ANY;
2791 return true;
2792
2793 case Intrinsic::s390_vstrczbs:
2794 case Intrinsic::s390_vstrczhs:
2795 case Intrinsic::s390_vstrczfs:
2796 Opcode = SystemZISD::VSTRCZ_CC;
2797 CCValid = SystemZ::CCMASK_ANY;
2798 return true;
2799
2800 case Intrinsic::s390_vstrsb:
2801 case Intrinsic::s390_vstrsh:
2802 case Intrinsic::s390_vstrsf:
2803 Opcode = SystemZISD::VSTRS_CC;
2804 CCValid = SystemZ::CCMASK_ANY;
2805 return true;
2806
2807 case Intrinsic::s390_vstrszb:
2808 case Intrinsic::s390_vstrszh:
2809 case Intrinsic::s390_vstrszf:
2810 Opcode = SystemZISD::VSTRSZ_CC;
2811 CCValid = SystemZ::CCMASK_ANY;
2812 return true;
2813
2814 case Intrinsic::s390_vfcedbs:
2815 case Intrinsic::s390_vfcesbs:
2816 Opcode = SystemZISD::VFCMPES;
2817 CCValid = SystemZ::CCMASK_VCMP;
2818 return true;
2819
2820 case Intrinsic::s390_vfchdbs:
2821 case Intrinsic::s390_vfchsbs:
2822 Opcode = SystemZISD::VFCMPHS;
2823 CCValid = SystemZ::CCMASK_VCMP;
2824 return true;
2825
2826 case Intrinsic::s390_vfchedbs:
2827 case Intrinsic::s390_vfchesbs:
2828 Opcode = SystemZISD::VFCMPHES;
2829 CCValid = SystemZ::CCMASK_VCMP;
2830 return true;
2831
2832 case Intrinsic::s390_vftcidb:
2833 case Intrinsic::s390_vftcisb:
2834 Opcode = SystemZISD::VFTCI;
2835 CCValid = SystemZ::CCMASK_VCMP;
2836 return true;
2837
2838 case Intrinsic::s390_tdc:
2839 Opcode = SystemZISD::TDC;
2840 CCValid = SystemZ::CCMASK_TDC;
2841 return true;
2842
2843 default:
2844 return false;
2845 }
2846}
2847
2848// Emit an intrinsic with chain and an explicit CC register result.
2849static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op,
2850 unsigned Opcode) {
2851 // Copy all operands except the intrinsic ID.
2852 unsigned NumOps = Op.getNumOperands();
2853 SmallVector<SDValue, 6> Ops;
2854 Ops.reserve(N: NumOps - 1);
2855 Ops.push_back(Elt: Op.getOperand(i: 0));
2856 for (unsigned I = 2; I < NumOps; ++I)
2857 Ops.push_back(Elt: Op.getOperand(i: I));
2858
2859 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
2860 SDVTList RawVTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
2861 SDValue Intr = DAG.getNode(Opcode, DL: SDLoc(Op), VTList: RawVTs, Ops);
2862 SDValue OldChain = SDValue(Op.getNode(), 1);
2863 SDValue NewChain = SDValue(Intr.getNode(), 1);
2864 DAG.ReplaceAllUsesOfValueWith(From: OldChain, To: NewChain);
2865 return Intr.getNode();
2866}
2867
2868// Emit an intrinsic with an explicit CC register result.
2869static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op,
2870 unsigned Opcode) {
2871 // Copy all operands except the intrinsic ID.
2872 SDLoc DL(Op);
2873 unsigned NumOps = Op.getNumOperands();
2874 SmallVector<SDValue, 6> Ops;
2875 Ops.reserve(N: NumOps - 1);
2876 for (unsigned I = 1; I < NumOps; ++I) {
2877 SDValue CurrOper = Op.getOperand(i: I);
2878 if (CurrOper.getValueType() == MVT::f16) {
2879 assert((Op.getConstantOperandVal(0) == Intrinsic::s390_tdc && I == 1) &&
2880 "Unhandled intrinsic with f16 operand.");
2881 CurrOper = DAG.getFPExtendOrRound(Op: CurrOper, DL, VT: MVT::f32);
2882 }
2883 Ops.push_back(Elt: CurrOper);
2884 }
2885
2886 SDValue Intr = DAG.getNode(Opcode, DL, VTList: Op->getVTList(), Ops);
2887 return Intr.getNode();
2888}
2889
2890// CC is a comparison that will be implemented using an integer or
2891// floating-point comparison. Return the condition code mask for
2892// a branch on true. In the integer case, CCMASK_CMP_UO is set for
2893// unsigned comparisons and clear for signed ones. In the floating-point
2894// case, CCMASK_CMP_UO has its normal mask meaning (unordered).
2895static unsigned CCMaskForCondCode(ISD::CondCode CC) {
2896#define CONV(X) \
2897 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
2898 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
2899 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
2900
2901 switch (CC) {
2902 default:
2903 llvm_unreachable("Invalid integer condition!");
2904
2905 CONV(EQ);
2906 CONV(NE);
2907 CONV(GT);
2908 CONV(GE);
2909 CONV(LT);
2910 CONV(LE);
2911
2912 case ISD::SETO: return SystemZ::CCMASK_CMP_O;
2913 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO;
2914 }
2915#undef CONV
2916}
2917
2918// If C can be converted to a comparison against zero, adjust the operands
2919// as necessary.
2920static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
2921 if (C.ICmpType == SystemZICMP::UnsignedOnly)
2922 return;
2923
2924 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1.getNode());
2925 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
2926 return;
2927
2928 int64_t Value = ConstOp1->getSExtValue();
2929 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
2930 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
2931 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
2932 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
2933 C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
2934 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op1.getValueType());
2935 }
2936}
2937
2938// If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
2939// adjust the operands as necessary.
2940static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
2941 Comparison &C) {
2942 // For us to make any changes, it must a comparison between a single-use
2943 // load and a constant.
2944 if (!C.Op0.hasOneUse() ||
2945 C.Op0.getOpcode() != ISD::LOAD ||
2946 C.Op1.getOpcode() != ISD::Constant)
2947 return;
2948
2949 // We must have an 8- or 16-bit load.
2950 auto *Load = cast<LoadSDNode>(Val&: C.Op0);
2951 unsigned NumBits = Load->getMemoryVT().getSizeInBits();
2952 if ((NumBits != 8 && NumBits != 16) ||
2953 NumBits != Load->getMemoryVT().getStoreSizeInBits())
2954 return;
2955
2956 // The load must be an extending one and the constant must be within the
2957 // range of the unextended value.
2958 auto *ConstOp1 = cast<ConstantSDNode>(Val&: C.Op1);
2959 if (!ConstOp1 || ConstOp1->getValueSizeInBits(ResNo: 0) > 64)
2960 return;
2961 uint64_t Value = ConstOp1->getZExtValue();
2962 uint64_t Mask = (1 << NumBits) - 1;
2963 if (Load->getExtensionType() == ISD::SEXTLOAD) {
2964 // Make sure that ConstOp1 is in range of C.Op0.
2965 int64_t SignedValue = ConstOp1->getSExtValue();
2966 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)
2967 return;
2968 if (C.ICmpType != SystemZICMP::SignedOnly) {
2969 // Unsigned comparison between two sign-extended values is equivalent
2970 // to unsigned comparison between two zero-extended values.
2971 Value &= Mask;
2972 } else if (NumBits == 8) {
2973 // Try to treat the comparison as unsigned, so that we can use CLI.
2974 // Adjust CCMask and Value as necessary.
2975 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
2976 // Test whether the high bit of the byte is set.
2977 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
2978 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
2979 // Test whether the high bit of the byte is clear.
2980 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
2981 else
2982 // No instruction exists for this combination.
2983 return;
2984 C.ICmpType = SystemZICMP::UnsignedOnly;
2985 }
2986 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {
2987 if (Value > Mask)
2988 return;
2989 // If the constant is in range, we can use any comparison.
2990 C.ICmpType = SystemZICMP::Any;
2991 } else
2992 return;
2993
2994 // Make sure that the first operand is an i32 of the right extension type.
2995 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ?
2996 ISD::SEXTLOAD :
2997 ISD::ZEXTLOAD);
2998 if (C.Op0.getValueType() != MVT::i32 ||
2999 Load->getExtensionType() != ExtType) {
3000 C.Op0 = DAG.getExtLoad(ExtType, dl: SDLoc(Load), VT: MVT::i32, Chain: Load->getChain(),
3001 Ptr: Load->getBasePtr(), PtrInfo: Load->getPointerInfo(),
3002 MemVT: Load->getMemoryVT(), Alignment: Load->getAlign(),
3003 MMOFlags: Load->getMemOperand()->getFlags());
3004 // Update the chain uses.
3005 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 1), To: C.Op0.getValue(R: 1));
3006 }
3007
3008 // Make sure that the second operand is an i32 with the right value.
3009 if (C.Op1.getValueType() != MVT::i32 ||
3010 Value != ConstOp1->getZExtValue())
3011 C.Op1 = DAG.getConstant(Val: (uint32_t)Value, DL, VT: MVT::i32);
3012}
3013
3014// Return true if Op is either an unextended load, or a load suitable
3015// for integer register-memory comparisons of type ICmpType.
3016static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
3017 auto *Load = dyn_cast<LoadSDNode>(Val: Op.getNode());
3018 if (Load) {
3019 // There are no instructions to compare a register with a memory byte.
3020 if (Load->getMemoryVT() == MVT::i8)
3021 return false;
3022 // Otherwise decide on extension type.
3023 switch (Load->getExtensionType()) {
3024 case ISD::NON_EXTLOAD:
3025 return true;
3026 case ISD::SEXTLOAD:
3027 return ICmpType != SystemZICMP::UnsignedOnly;
3028 case ISD::ZEXTLOAD:
3029 return ICmpType != SystemZICMP::SignedOnly;
3030 default:
3031 break;
3032 }
3033 }
3034 return false;
3035}
3036
3037// Return true if it is better to swap the operands of C.
3038static bool shouldSwapCmpOperands(const Comparison &C) {
3039 // If one side of the compare is a load of the stackguard reference value,
3040 // then that load should be Op1.
3041 if (C.Op0.isMachineOpcode() &&
3042 (C.Op0.getMachineOpcode() == SystemZ::LOAD_STACK_GUARD))
3043 return true;
3044
3045 // Leave i128 and f128 comparisons alone, since they have no memory forms.
3046 if (C.Op0.getValueType() == MVT::i128)
3047 return false;
3048 if (C.Op0.getValueType() == MVT::f128)
3049 return false;
3050
3051 // Always keep a floating-point constant second, since comparisons with
3052 // zero can use LOAD TEST and comparisons with other constants make a
3053 // natural memory operand.
3054 if (isa<ConstantFPSDNode>(Val: C.Op1))
3055 return false;
3056
3057 // Never swap comparisons with zero since there are many ways to optimize
3058 // those later.
3059 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val: C.Op1);
3060 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
3061 return false;
3062
3063 // Also keep natural memory operands second if the loaded value is
3064 // only used here. Several comparisons have memory forms.
3065 if (isNaturalMemoryOperand(Op: C.Op1, ICmpType: C.ICmpType) && C.Op1.hasOneUse())
3066 return false;
3067
3068 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.
3069 // In that case we generally prefer the memory to be second.
3070 if (isNaturalMemoryOperand(Op: C.Op0, ICmpType: C.ICmpType) && C.Op0.hasOneUse()) {
3071 // The only exceptions are when the second operand is a constant and
3072 // we can use things like CHHSI.
3073 if (!ConstOp1)
3074 return true;
3075 // The unsigned memory-immediate instructions can handle 16-bit
3076 // unsigned integers.
3077 if (C.ICmpType != SystemZICMP::SignedOnly &&
3078 isUInt<16>(x: ConstOp1->getZExtValue()))
3079 return false;
3080 // The signed memory-immediate instructions can handle 16-bit
3081 // signed integers.
3082 if (C.ICmpType != SystemZICMP::UnsignedOnly &&
3083 isInt<16>(x: ConstOp1->getSExtValue()))
3084 return false;
3085 return true;
3086 }
3087
3088 // Try to promote the use of CGFR and CLGFR.
3089 unsigned Opcode0 = C.Op0.getOpcode();
3090 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND)
3091 return true;
3092 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND)
3093 return true;
3094 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::AND &&
3095 C.Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
3096 C.Op0.getConstantOperandVal(i: 1) == 0xffffffff)
3097 return true;
3098
3099 return false;
3100}
3101
3102// Check whether C tests for equality between X and Y and whether X - Y
3103// or Y - X is also computed. In that case it's better to compare the
3104// result of the subtraction against zero.
3105static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
3106 Comparison &C) {
3107 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3108 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3109 for (SDNode *N : C.Op0->users()) {
3110 if (N->getOpcode() == ISD::SUB &&
3111 ((N->getOperand(Num: 0) == C.Op0 && N->getOperand(Num: 1) == C.Op1) ||
3112 (N->getOperand(Num: 0) == C.Op1 && N->getOperand(Num: 1) == C.Op0))) {
3113 // Disable the nsw and nuw flags: the backend needs to handle
3114 // overflow as well during comparison elimination.
3115 N->dropFlags(Mask: SDNodeFlags::NoWrap);
3116 C.Op0 = SDValue(N, 0);
3117 C.Op1 = DAG.getConstant(Val: 0, DL, VT: N->getValueType(ResNo: 0));
3118 return;
3119 }
3120 }
3121 }
3122}
3123
3124// Check whether C compares a floating-point value with zero and if that
3125// floating-point value is also negated. In this case we can use the
3126// negation to set CC, so avoiding separate LOAD AND TEST and
3127// LOAD (NEGATIVE/COMPLEMENT) instructions.
3128static void adjustForFNeg(Comparison &C) {
3129 // This optimization is invalid for strict comparisons, since FNEG
3130 // does not raise any exceptions.
3131 if (C.Chain)
3132 return;
3133 auto *C1 = dyn_cast<ConstantFPSDNode>(Val&: C.Op1);
3134 if (C1 && C1->isZero()) {
3135 for (SDNode *N : C.Op0->users()) {
3136 if (N->getOpcode() == ISD::FNEG) {
3137 C.Op0 = SDValue(N, 0);
3138 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3139 return;
3140 }
3141 }
3142 }
3143}
3144
3145// Check whether C compares (shl X, 32) with 0 and whether X is
3146// also sign-extended. In that case it is better to test the result
3147// of the sign extension using LTGFR.
3148//
3149// This case is important because InstCombine transforms a comparison
3150// with (sext (trunc X)) into a comparison with (shl X, 32).
3151static void adjustForLTGFR(Comparison &C) {
3152 // Check for a comparison between (shl X, 32) and 0.
3153 if (C.Op0.getOpcode() == ISD::SHL && C.Op0.getValueType() == MVT::i64 &&
3154 C.Op1.getOpcode() == ISD::Constant && C.Op1->getAsZExtVal() == 0) {
3155 auto *C1 = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3156 if (C1 && C1->getZExtValue() == 32) {
3157 SDValue ShlOp0 = C.Op0.getOperand(i: 0);
3158 // See whether X has any SIGN_EXTEND_INREG uses.
3159 for (SDNode *N : ShlOp0->users()) {
3160 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3161 cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT() == MVT::i32) {
3162 C.Op0 = SDValue(N, 0);
3163 return;
3164 }
3165 }
3166 }
3167 }
3168}
3169
3170// If C compares the truncation of an extending load, try to compare
3171// the untruncated value instead. This exposes more opportunities to
3172// reuse CC.
3173static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
3174 Comparison &C) {
3175 if (C.Op0.getOpcode() == ISD::TRUNCATE &&
3176 C.Op0.getOperand(i: 0).getOpcode() == ISD::LOAD &&
3177 C.Op1.getOpcode() == ISD::Constant &&
3178 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
3179 C.Op1->getAsZExtVal() == 0) {
3180 auto *L = cast<LoadSDNode>(Val: C.Op0.getOperand(i: 0));
3181 if (L->getMemoryVT().getStoreSizeInBits().getFixedValue() <=
3182 C.Op0.getValueSizeInBits().getFixedValue()) {
3183 unsigned Type = L->getExtensionType();
3184 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
3185 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
3186 C.Op0 = C.Op0.getOperand(i: 0);
3187 C.Op1 = DAG.getConstant(Val: 0, DL, VT: C.Op0.getValueType());
3188 }
3189 }
3190 }
3191}
3192
3193// Adjust if a given Compare is a check of the stack guard against a stack
3194// guard instance on the stack. Specifically, this checks if:
3195// - The operands are a load of the stack guard, and a load from a stack slot
3196// - The original opcode is ICMP
3197// - ICMPType is compatible with unsigned comparison.
3198static void adjustForStackGuardCompare(SelectionDAG &DAG, const SDLoc &DL,
3199 Comparison &C) {
3200
3201 // Opcode must be ICMP.
3202 if (C.Opcode != SystemZISD::ICMP)
3203 return;
3204 // ICmpType must be Unsigned or Any.
3205 if (C.ICmpType == SystemZICMP::SignedOnly)
3206 return;
3207 // Op0 must be FrameIndex Load.
3208 if (!(ISD::isNormalLoad(N: C.Op0.getNode()) &&
3209 dyn_cast<FrameIndexSDNode>(Val: C.Op0.getOperand(i: 1))))
3210 return;
3211 // Op1 must be LOAD_STACK_GUARD.
3212 if (!C.Op1.isMachineOpcode() ||
3213 C.Op1.getMachineOpcode() != SystemZ::LOAD_STACK_GUARD)
3214 return;
3215
3216 // At this point we are sure that this is a proper CMP_STACKGUARD
3217 // case, update the opcode to reflect this.
3218 C.Opcode = SystemZISD::CMP_STACKGUARD;
3219 C.Op1 = SDValue();
3220}
3221
3222// Return true if shift operation N has an in-range constant shift value.
3223// Store it in ShiftVal if so.
3224static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
3225 auto *Shift = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
3226 if (!Shift)
3227 return false;
3228
3229 uint64_t Amount = Shift->getZExtValue();
3230 if (Amount >= N.getValueSizeInBits())
3231 return false;
3232
3233 ShiftVal = Amount;
3234 return true;
3235}
3236
3237// Check whether an AND with Mask is suitable for a TEST UNDER MASK
3238// instruction and whether the CC value is descriptive enough to handle
3239// a comparison of type Opcode between the AND result and CmpVal.
3240// CCMask says which comparison result is being tested and BitSize is
3241// the number of bits in the operands. If TEST UNDER MASK can be used,
3242// return the corresponding CC mask, otherwise return 0.
3243static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
3244 uint64_t Mask, uint64_t CmpVal,
3245 unsigned ICmpType) {
3246 assert(Mask != 0 && "ANDs with zero should have been removed by now");
3247
3248 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL.
3249 if (!SystemZ::isImmLL(Val: Mask) && !SystemZ::isImmLH(Val: Mask) &&
3250 !SystemZ::isImmHL(Val: Mask) && !SystemZ::isImmHH(Val: Mask))
3251 return 0;
3252
3253 // Work out the masks for the lowest and highest bits.
3254 uint64_t High = llvm::bit_floor(Value: Mask);
3255 uint64_t Low = uint64_t(1) << llvm::countr_zero(Val: Mask);
3256
3257 // Signed ordered comparisons are effectively unsigned if the sign
3258 // bit is dropped.
3259 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly);
3260
3261 // Check for equality comparisons with 0, or the equivalent.
3262 if (CmpVal == 0) {
3263 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3264 return SystemZ::CCMASK_TM_ALL_0;
3265 if (CCMask == SystemZ::CCMASK_CMP_NE)
3266 return SystemZ::CCMASK_TM_SOME_1;
3267 }
3268 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) {
3269 if (CCMask == SystemZ::CCMASK_CMP_LT)
3270 return SystemZ::CCMASK_TM_ALL_0;
3271 if (CCMask == SystemZ::CCMASK_CMP_GE)
3272 return SystemZ::CCMASK_TM_SOME_1;
3273 }
3274 if (EffectivelyUnsigned && CmpVal < Low) {
3275 if (CCMask == SystemZ::CCMASK_CMP_LE)
3276 return SystemZ::CCMASK_TM_ALL_0;
3277 if (CCMask == SystemZ::CCMASK_CMP_GT)
3278 return SystemZ::CCMASK_TM_SOME_1;
3279 }
3280
3281 // Check for equality comparisons with the mask, or the equivalent.
3282 if (CmpVal == Mask) {
3283 if (CCMask == SystemZ::CCMASK_CMP_EQ)
3284 return SystemZ::CCMASK_TM_ALL_1;
3285 if (CCMask == SystemZ::CCMASK_CMP_NE)
3286 return SystemZ::CCMASK_TM_SOME_0;
3287 }
3288 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {
3289 if (CCMask == SystemZ::CCMASK_CMP_GT)
3290 return SystemZ::CCMASK_TM_ALL_1;
3291 if (CCMask == SystemZ::CCMASK_CMP_LE)
3292 return SystemZ::CCMASK_TM_SOME_0;
3293 }
3294 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {
3295 if (CCMask == SystemZ::CCMASK_CMP_GE)
3296 return SystemZ::CCMASK_TM_ALL_1;
3297 if (CCMask == SystemZ::CCMASK_CMP_LT)
3298 return SystemZ::CCMASK_TM_SOME_0;
3299 }
3300
3301 // Check for ordered comparisons with the top bit.
3302 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {
3303 if (CCMask == SystemZ::CCMASK_CMP_LE)
3304 return SystemZ::CCMASK_TM_MSB_0;
3305 if (CCMask == SystemZ::CCMASK_CMP_GT)
3306 return SystemZ::CCMASK_TM_MSB_1;
3307 }
3308 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {
3309 if (CCMask == SystemZ::CCMASK_CMP_LT)
3310 return SystemZ::CCMASK_TM_MSB_0;
3311 if (CCMask == SystemZ::CCMASK_CMP_GE)
3312 return SystemZ::CCMASK_TM_MSB_1;
3313 }
3314
3315 // If there are just two bits, we can do equality checks for Low and High
3316 // as well.
3317 if (Mask == Low + High) {
3318 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
3319 return SystemZ::CCMASK_TM_MIXED_MSB_0;
3320 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
3321 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY;
3322 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
3323 return SystemZ::CCMASK_TM_MIXED_MSB_1;
3324 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
3325 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY;
3326 }
3327
3328 // Looks like we've exhausted our options.
3329 return 0;
3330}
3331
3332// See whether C can be implemented as a TEST UNDER MASK instruction.
3333// Update the arguments with the TM version if so.
3334static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
3335 Comparison &C) {
3336 // Use VECTOR TEST UNDER MASK for i128 operations.
3337 if (C.Op0.getValueType() == MVT::i128) {
3338 // We can use VTM for EQ/NE comparisons of x & y against 0.
3339 if (C.Op0.getOpcode() == ISD::AND &&
3340 (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3341 C.CCMask == SystemZ::CCMASK_CMP_NE)) {
3342 auto *Mask = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3343 if (Mask && Mask->getAPIntValue() == 0) {
3344 C.Opcode = SystemZISD::VTM;
3345 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 1));
3346 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: C.Op0.getOperand(i: 0));
3347 C.CCValid = SystemZ::CCMASK_VCMP;
3348 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3349 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3350 else
3351 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3352 }
3353 }
3354 return;
3355 }
3356
3357 // Check that we have a comparison with a constant.
3358 auto *ConstOp1 = dyn_cast<ConstantSDNode>(Val&: C.Op1);
3359 if (!ConstOp1)
3360 return;
3361 uint64_t CmpVal = ConstOp1->getZExtValue();
3362
3363 // Check whether the nonconstant input is an AND with a constant mask.
3364 Comparison NewC(C);
3365 uint64_t MaskVal;
3366 ConstantSDNode *Mask = nullptr;
3367 if (C.Op0.getOpcode() == ISD::AND) {
3368 NewC.Op0 = C.Op0.getOperand(i: 0);
3369 NewC.Op1 = C.Op0.getOperand(i: 1);
3370 Mask = dyn_cast<ConstantSDNode>(Val&: NewC.Op1);
3371 if (!Mask)
3372 return;
3373 MaskVal = Mask->getZExtValue();
3374 } else {
3375 // There is no instruction to compare with a 64-bit immediate
3376 // so use TMHH instead if possible. We need an unsigned ordered
3377 // comparison with an i64 immediate.
3378 if (NewC.Op0.getValueType() != MVT::i64 ||
3379 NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
3380 NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
3381 NewC.ICmpType == SystemZICMP::SignedOnly)
3382 return;
3383 // Convert LE and GT comparisons into LT and GE.
3384 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
3385 NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
3386 if (CmpVal == uint64_t(-1))
3387 return;
3388 CmpVal += 1;
3389 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
3390 }
3391 // If the low N bits of Op1 are zero than the low N bits of Op0 can
3392 // be masked off without changing the result.
3393 MaskVal = -(CmpVal & -CmpVal);
3394 NewC.ICmpType = SystemZICMP::UnsignedOnly;
3395 }
3396 if (!MaskVal)
3397 return;
3398
3399 // Check whether the combination of mask, comparison value and comparison
3400 // type are suitable.
3401 unsigned BitSize = NewC.Op0.getValueSizeInBits();
3402 unsigned NewCCMask, ShiftVal;
3403 if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3404 NewC.Op0.getOpcode() == ISD::SHL &&
3405 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3406 (MaskVal >> ShiftVal != 0) &&
3407 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
3408 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3409 Mask: MaskVal >> ShiftVal,
3410 CmpVal: CmpVal >> ShiftVal,
3411 ICmpType: SystemZICMP::Any))) {
3412 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3413 MaskVal >>= ShiftVal;
3414 } else if (NewC.ICmpType != SystemZICMP::SignedOnly &&
3415 NewC.Op0.getOpcode() == ISD::SRL &&
3416 isSimpleShift(N: NewC.Op0, ShiftVal) &&
3417 (MaskVal << ShiftVal != 0) &&
3418 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
3419 (NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask,
3420 Mask: MaskVal << ShiftVal,
3421 CmpVal: CmpVal << ShiftVal,
3422 ICmpType: SystemZICMP::UnsignedOnly))) {
3423 NewC.Op0 = NewC.Op0.getOperand(i: 0);
3424 MaskVal <<= ShiftVal;
3425 } else {
3426 NewCCMask = getTestUnderMaskCond(BitSize, CCMask: NewC.CCMask, Mask: MaskVal, CmpVal,
3427 ICmpType: NewC.ICmpType);
3428 if (!NewCCMask)
3429 return;
3430 }
3431
3432 // Go ahead and make the change.
3433 C.Opcode = SystemZISD::TM;
3434 C.Op0 = NewC.Op0;
3435 if (Mask && Mask->getZExtValue() == MaskVal)
3436 C.Op1 = SDValue(Mask, 0);
3437 else
3438 C.Op1 = DAG.getConstant(Val: MaskVal, DL, VT: C.Op0.getValueType());
3439 C.CCValid = SystemZ::CCMASK_TM;
3440 C.CCMask = NewCCMask;
3441}
3442
3443// Implement i128 comparison in vector registers.
3444static void adjustICmp128(SelectionDAG &DAG, const SDLoc &DL,
3445 Comparison &C) {
3446 if (C.Opcode != SystemZISD::ICMP)
3447 return;
3448 if (C.Op0.getValueType() != MVT::i128)
3449 return;
3450
3451 // Recognize vector comparison reductions.
3452 if ((C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3453 C.CCMask == SystemZ::CCMASK_CMP_NE) &&
3454 (isNullConstant(V: C.Op1) || isAllOnesConstant(V: C.Op1))) {
3455 bool CmpEq = C.CCMask == SystemZ::CCMASK_CMP_EQ;
3456 bool CmpNull = isNullConstant(V: C.Op1);
3457 SDValue Src = peekThroughBitcasts(V: C.Op0);
3458 if (Src.hasOneUse() && isBitwiseNot(V: Src)) {
3459 Src = Src.getOperand(i: 0);
3460 CmpNull = !CmpNull;
3461 }
3462 unsigned Opcode = 0;
3463 if (Src.hasOneUse()) {
3464 switch (Src.getOpcode()) {
3465 case SystemZISD::VICMPE: Opcode = SystemZISD::VICMPES; break;
3466 case SystemZISD::VICMPH: Opcode = SystemZISD::VICMPHS; break;
3467 case SystemZISD::VICMPHL: Opcode = SystemZISD::VICMPHLS; break;
3468 case SystemZISD::VFCMPE: Opcode = SystemZISD::VFCMPES; break;
3469 case SystemZISD::VFCMPH: Opcode = SystemZISD::VFCMPHS; break;
3470 case SystemZISD::VFCMPHE: Opcode = SystemZISD::VFCMPHES; break;
3471 default: break;
3472 }
3473 }
3474 if (Opcode) {
3475 C.Opcode = Opcode;
3476 C.Op0 = Src->getOperand(Num: 0);
3477 C.Op1 = Src->getOperand(Num: 1);
3478 C.CCValid = SystemZ::CCMASK_VCMP;
3479 C.CCMask = CmpNull ? SystemZ::CCMASK_VCMP_NONE : SystemZ::CCMASK_VCMP_ALL;
3480 if (!CmpEq)
3481 C.CCMask ^= C.CCValid;
3482 return;
3483 }
3484 }
3485
3486 // Everything below here is not useful if we have native i128 compares.
3487 if (DAG.getSubtarget<SystemZSubtarget>().hasVectorEnhancements3())
3488 return;
3489
3490 // (In-)Equality comparisons can be implemented via VCEQGS.
3491 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3492 C.CCMask == SystemZ::CCMASK_CMP_NE) {
3493 C.Opcode = SystemZISD::VICMPES;
3494 C.Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op0);
3495 C.Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: C.Op1);
3496 C.CCValid = SystemZ::CCMASK_VCMP;
3497 if (C.CCMask == SystemZ::CCMASK_CMP_EQ)
3498 C.CCMask = SystemZ::CCMASK_VCMP_ALL;
3499 else
3500 C.CCMask = SystemZ::CCMASK_VCMP_ALL ^ C.CCValid;
3501 return;
3502 }
3503
3504 // Normalize other comparisons to GT.
3505 bool Swap = false, Invert = false;
3506 switch (C.CCMask) {
3507 case SystemZ::CCMASK_CMP_GT: break;
3508 case SystemZ::CCMASK_CMP_LT: Swap = true; break;
3509 case SystemZ::CCMASK_CMP_LE: Invert = true; break;
3510 case SystemZ::CCMASK_CMP_GE: Swap = Invert = true; break;
3511 default: llvm_unreachable("Invalid integer condition!");
3512 }
3513 if (Swap)
3514 std::swap(a&: C.Op0, b&: C.Op1);
3515
3516 if (C.ICmpType == SystemZICMP::UnsignedOnly)
3517 C.Opcode = SystemZISD::UCMP128HI;
3518 else
3519 C.Opcode = SystemZISD::SCMP128HI;
3520 C.CCValid = SystemZ::CCMASK_ANY;
3521 C.CCMask = SystemZ::CCMASK_1;
3522
3523 if (Invert)
3524 C.CCMask ^= C.CCValid;
3525}
3526
3527// See whether the comparison argument contains a redundant AND
3528// and remove it if so. This sometimes happens due to the generic
3529// BRCOND expansion.
3530static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
3531 Comparison &C) {
3532 if (C.Op0.getOpcode() != ISD::AND)
3533 return;
3534 auto *Mask = dyn_cast<ConstantSDNode>(Val: C.Op0.getOperand(i: 1));
3535 if (!Mask || Mask->getValueSizeInBits(ResNo: 0) > 64)
3536 return;
3537 KnownBits Known = DAG.computeKnownBits(Op: C.Op0.getOperand(i: 0));
3538 if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
3539 return;
3540
3541 C.Op0 = C.Op0.getOperand(i: 0);
3542}
3543
3544// Return a Comparison that tests the condition-code result of intrinsic
3545// node Call against constant integer CC using comparison code Cond.
3546// Opcode is the opcode of the SystemZISD operation for the intrinsic
3547// and CCValid is the set of possible condition-code results.
3548static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
3549 SDValue Call, unsigned CCValid, uint64_t CC,
3550 ISD::CondCode Cond) {
3551 Comparison C(Call, SDValue(), SDValue());
3552 C.Opcode = Opcode;
3553 C.CCValid = CCValid;
3554 if (Cond == ISD::SETEQ)
3555 // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3.
3556 C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
3557 else if (Cond == ISD::SETNE)
3558 // ...and the inverse of that.
3559 C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
3560 else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
3561 // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
3562 // always true for CC>3.
3563 C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
3564 else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
3565 // ...and the inverse of that.
3566 C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
3567 else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
3568 // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
3569 // always true for CC>3.
3570 C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
3571 else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
3572 // ...and the inverse of that.
3573 C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
3574 else
3575 llvm_unreachable("Unexpected integer comparison type");
3576 C.CCMask &= CCValid;
3577 return C;
3578}
3579
3580// Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
3581static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
3582 ISD::CondCode Cond, const SDLoc &DL,
3583 SDValue Chain = SDValue(),
3584 bool IsSignaling = false) {
3585 if (CmpOp1.getOpcode() == ISD::Constant) {
3586 assert(!Chain);
3587 unsigned Opcode, CCValid;
3588 if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
3589 CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(NUses: 1, Value: 0) &&
3590 isIntrinsicWithCCAndChain(Op: CmpOp0, Opcode, CCValid))
3591 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3592 CC: CmpOp1->getAsZExtVal(), Cond);
3593 if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
3594 CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
3595 isIntrinsicWithCC(Op: CmpOp0, Opcode, CCValid))
3596 return getIntrinsicCmp(DAG, Opcode, Call: CmpOp0, CCValid,
3597 CC: CmpOp1->getAsZExtVal(), Cond);
3598 }
3599 Comparison C(CmpOp0, CmpOp1, Chain);
3600 C.CCMask = CCMaskForCondCode(CC: Cond);
3601 if (C.Op0.getValueType().isFloatingPoint()) {
3602 C.CCValid = SystemZ::CCMASK_FCMP;
3603 if (!C.Chain)
3604 C.Opcode = SystemZISD::FCMP;
3605 else if (!IsSignaling)
3606 C.Opcode = SystemZISD::STRICT_FCMP;
3607 else
3608 C.Opcode = SystemZISD::STRICT_FCMPS;
3609 adjustForFNeg(C);
3610 } else {
3611 assert(!C.Chain);
3612 C.CCValid = SystemZ::CCMASK_ICMP;
3613 C.Opcode = SystemZISD::ICMP;
3614 // Choose the type of comparison. Equality and inequality tests can
3615 // use either signed or unsigned comparisons. The choice also doesn't
3616 // matter if both sign bits are known to be clear. In those cases we
3617 // want to give the main isel code the freedom to choose whichever
3618 // form fits best.
3619 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
3620 C.CCMask == SystemZ::CCMASK_CMP_NE ||
3621 (DAG.SignBitIsZero(Op: C.Op0) && DAG.SignBitIsZero(Op: C.Op1)))
3622 C.ICmpType = SystemZICMP::Any;
3623 else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
3624 C.ICmpType = SystemZICMP::UnsignedOnly;
3625 else
3626 C.ICmpType = SystemZICMP::SignedOnly;
3627 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
3628 adjustForRedundantAnd(DAG, DL, C);
3629 adjustZeroCmp(DAG, DL, C);
3630 adjustSubwordCmp(DAG, DL, C);
3631 adjustForSubtraction(DAG, DL, C);
3632 adjustForLTGFR(C);
3633 adjustICmpTruncate(DAG, DL, C);
3634 }
3635
3636 if (shouldSwapCmpOperands(C)) {
3637 std::swap(a&: C.Op0, b&: C.Op1);
3638 C.CCMask = SystemZ::reverseCCMask(CCMask: C.CCMask);
3639 }
3640
3641 adjustForTestUnderMask(DAG, DL, C);
3642 adjustICmp128(DAG, DL, C);
3643 adjustForStackGuardCompare(DAG, DL, C);
3644 return C;
3645}
3646
3647// Emit the comparison instruction described by C.
3648static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
3649 if (!C.Op1.getNode()) {
3650 if (C.Opcode == SystemZISD::CMP_STACKGUARD)
3651 return DAG.getNode(Opcode: SystemZISD::CMP_STACKGUARD, DL, VT: MVT::i32, Operand: C.Op0);
3652 SDNode *Node;
3653 switch (C.Op0.getOpcode()) {
3654 case ISD::INTRINSIC_W_CHAIN:
3655 Node = emitIntrinsicWithCCAndChain(DAG, Op: C.Op0, Opcode: C.Opcode);
3656 return SDValue(Node, 0);
3657 case ISD::INTRINSIC_WO_CHAIN:
3658 Node = emitIntrinsicWithCC(DAG, Op: C.Op0, Opcode: C.Opcode);
3659 return SDValue(Node, Node->getNumValues() - 1);
3660 default:
3661 llvm_unreachable("Invalid comparison operands");
3662 }
3663 }
3664 if (C.Opcode == SystemZISD::ICMP)
3665 return DAG.getNode(Opcode: SystemZISD::ICMP, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3666 N3: DAG.getTargetConstant(Val: C.ICmpType, DL, VT: MVT::i32));
3667 if (C.Opcode == SystemZISD::TM) {
3668 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
3669 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
3670 return DAG.getNode(Opcode: SystemZISD::TM, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1,
3671 N3: DAG.getTargetConstant(Val: RegisterOnly, DL, VT: MVT::i32));
3672 }
3673 if (C.Opcode == SystemZISD::VICMPES ||
3674 C.Opcode == SystemZISD::VICMPHS ||
3675 C.Opcode == SystemZISD::VICMPHLS ||
3676 C.Opcode == SystemZISD::VFCMPES ||
3677 C.Opcode == SystemZISD::VFCMPHS ||
3678 C.Opcode == SystemZISD::VFCMPHES) {
3679 EVT IntVT = C.Op0.getValueType().changeVectorElementTypeToInteger();
3680 SDVTList VTs = DAG.getVTList(VT1: IntVT, VT2: MVT::i32);
3681 SDValue Val = DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Op0, N2: C.Op1);
3682 return SDValue(Val.getNode(), 1);
3683 }
3684 if (C.Chain) {
3685 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
3686 return DAG.getNode(Opcode: C.Opcode, DL, VTList: VTs, N1: C.Chain, N2: C.Op0, N3: C.Op1);
3687 }
3688 return DAG.getNode(Opcode: C.Opcode, DL, VT: MVT::i32, N1: C.Op0, N2: C.Op1);
3689}
3690
3691// Implement a 32-bit *MUL_LOHI operation by extending both operands to
3692// 64 bits. Extend is the extension type to use. Store the high part
3693// in Hi and the low part in Lo.
3694static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
3695 SDValue Op0, SDValue Op1, SDValue &Hi,
3696 SDValue &Lo) {
3697 Op0 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op0);
3698 Op1 = DAG.getNode(Opcode: Extend, DL, VT: MVT::i64, Operand: Op1);
3699 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
3700 Hi = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Mul,
3701 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3702 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Hi);
3703 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Mul);
3704}
3705
3706// Lower a binary operation that produces two VT results, one in each
3707// half of a GR128 pair. Op0 and Op1 are the VT operands to the operation,
3708// and Opcode performs the GR128 operation. Store the even register result
3709// in Even and the odd register result in Odd.
3710static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
3711 unsigned Opcode, SDValue Op0, SDValue Op1,
3712 SDValue &Even, SDValue &Odd) {
3713 SDValue Result = DAG.getNode(Opcode, DL, VT: MVT::Untyped, N1: Op0, N2: Op1);
3714 bool Is32Bit = is32Bit(VT);
3715 Even = DAG.getTargetExtractSubreg(SRIdx: SystemZ::even128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3716 Odd = DAG.getTargetExtractSubreg(SRIdx: SystemZ::odd128(Is32bit: Is32Bit), DL, VT, Operand: Result);
3717}
3718
3719// Return an i32 value that is 1 if the CC value produced by CCReg is
3720// in the mask CCMask and 0 otherwise. CC is known to have a value
3721// in CCValid, so other values can be ignored.
3722static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
3723 unsigned CCValid, unsigned CCMask) {
3724 SDValue Ops[] = {DAG.getConstant(Val: 1, DL, VT: MVT::i32),
3725 DAG.getConstant(Val: 0, DL, VT: MVT::i32),
3726 DAG.getTargetConstant(Val: CCValid, DL, VT: MVT::i32),
3727 DAG.getTargetConstant(Val: CCMask, DL, VT: MVT::i32), CCReg};
3728 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: MVT::i32, Ops);
3729}
3730
3731// Return the SystemISD vector comparison operation for CC, or 0 if it cannot
3732// be done directly. Mode is CmpMode::Int for integer comparisons, CmpMode::FP
3733// for regular floating-point comparisons, CmpMode::StrictFP for strict (quiet)
3734// floating-point comparisons, and CmpMode::SignalingFP for strict signaling
3735// floating-point comparisons.
3736enum class CmpMode { Int, FP, StrictFP, SignalingFP };
3737static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode) {
3738 switch (CC) {
3739 case ISD::SETOEQ:
3740 case ISD::SETEQ:
3741 switch (Mode) {
3742 case CmpMode::Int: return SystemZISD::VICMPE;
3743 case CmpMode::FP: return SystemZISD::VFCMPE;
3744 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPE;
3745 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPES;
3746 }
3747 llvm_unreachable("Bad mode");
3748
3749 case ISD::SETOGE:
3750 case ISD::SETGE:
3751 switch (Mode) {
3752 case CmpMode::Int: return 0;
3753 case CmpMode::FP: return SystemZISD::VFCMPHE;
3754 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPHE;
3755 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHES;
3756 }
3757 llvm_unreachable("Bad mode");
3758
3759 case ISD::SETOGT:
3760 case ISD::SETGT:
3761 switch (Mode) {
3762 case CmpMode::Int: return SystemZISD::VICMPH;
3763 case CmpMode::FP: return SystemZISD::VFCMPH;
3764 case CmpMode::StrictFP: return SystemZISD::STRICT_VFCMPH;
3765 case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHS;
3766 }
3767 llvm_unreachable("Bad mode");
3768
3769 case ISD::SETUGT:
3770 switch (Mode) {
3771 case CmpMode::Int: return SystemZISD::VICMPHL;
3772 case CmpMode::FP: return 0;
3773 case CmpMode::StrictFP: return 0;
3774 case CmpMode::SignalingFP: return 0;
3775 }
3776 llvm_unreachable("Bad mode");
3777
3778 default:
3779 return 0;
3780 }
3781}
3782
3783// Return the SystemZISD vector comparison operation for CC or its inverse,
3784// or 0 if neither can be done directly. Indicate in Invert whether the
3785// result is for the inverse of CC. Mode is as above.
3786static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode,
3787 bool &Invert) {
3788 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3789 Invert = false;
3790 return Opcode;
3791 }
3792
3793 CC = ISD::getSetCCInverse(Operation: CC, Type: Mode == CmpMode::Int ? MVT::i32 : MVT::f32);
3794 if (unsigned Opcode = getVectorComparison(CC, Mode)) {
3795 Invert = true;
3796 return Opcode;
3797 }
3798
3799 return 0;
3800}
3801
3802// Return a v2f64 that contains the extended form of elements Start and Start+1
3803// of v4f32 value Op. If Chain is nonnull, return the strict form.
3804static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
3805 SDValue Op, SDValue Chain) {
3806 int Mask[] = { Start, -1, Start + 1, -1 };
3807 Op = DAG.getVectorShuffle(VT: MVT::v4f32, dl: DL, N1: Op, N2: DAG.getUNDEF(VT: MVT::v4f32), Mask);
3808 if (Chain) {
3809 SDVTList VTs = DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other);
3810 return DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL, VTList: VTs, N1: Chain, N2: Op);
3811 }
3812 return DAG.getNode(Opcode: SystemZISD::VEXTEND, DL, VT: MVT::v2f64, Operand: Op);
3813}
3814
3815// Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
3816// producing a result of type VT. If Chain is nonnull, return the strict form.
3817SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
3818 const SDLoc &DL, EVT VT,
3819 SDValue CmpOp0,
3820 SDValue CmpOp1,
3821 SDValue Chain) const {
3822 // There is no hardware support for v4f32 (unless we have the vector
3823 // enhancements facility 1), so extend the vector into two v2f64s
3824 // and compare those.
3825 if (CmpOp0.getValueType() == MVT::v4f32 &&
3826 !Subtarget.hasVectorEnhancements1()) {
3827 SDValue H0 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp0, Chain);
3828 SDValue L0 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp0, Chain);
3829 SDValue H1 = expandV4F32ToV2F64(DAG, Start: 0, DL, Op: CmpOp1, Chain);
3830 SDValue L1 = expandV4F32ToV2F64(DAG, Start: 2, DL, Op: CmpOp1, Chain);
3831 if (Chain) {
3832 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i64, VT2: MVT::Other);
3833 SDValue HRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: H0, N3: H1);
3834 SDValue LRes = DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: L0, N3: L1);
3835 SDValue Res = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3836 SDValue Chains[6] = { H0.getValue(R: 1), L0.getValue(R: 1),
3837 H1.getValue(R: 1), L1.getValue(R: 1),
3838 HRes.getValue(R: 1), LRes.getValue(R: 1) };
3839 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3840 SDValue Ops[2] = { Res, NewChain };
3841 return DAG.getMergeValues(Ops, dl: DL);
3842 }
3843 SDValue HRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: H0, N2: H1);
3844 SDValue LRes = DAG.getNode(Opcode, DL, VT: MVT::v2i64, N1: L0, N2: L1);
3845 return DAG.getNode(Opcode: SystemZISD::PACK, DL, VT, N1: HRes, N2: LRes);
3846 }
3847 if (Chain) {
3848 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::Other);
3849 return DAG.getNode(Opcode, DL, VTList: VTs, N1: Chain, N2: CmpOp0, N3: CmpOp1);
3850 }
3851 return DAG.getNode(Opcode, DL, VT, N1: CmpOp0, N2: CmpOp1);
3852}
3853
3854// Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
3855// an integer mask of type VT. If Chain is nonnull, we have a strict
3856// floating-point comparison. If in addition IsSignaling is true, we have
3857// a strict signaling floating-point comparison.
3858SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
3859 const SDLoc &DL, EVT VT,
3860 ISD::CondCode CC,
3861 SDValue CmpOp0,
3862 SDValue CmpOp1,
3863 SDValue Chain,
3864 bool IsSignaling) const {
3865 bool IsFP = CmpOp0.getValueType().isFloatingPoint();
3866 assert (!Chain || IsFP);
3867 assert (!IsSignaling || Chain);
3868 CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
3869 Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
3870 bool Invert = false;
3871 SDValue Cmp;
3872 switch (CC) {
3873 // Handle tests for order using (or (ogt y x) (oge x y)).
3874 case ISD::SETUO:
3875 Invert = true;
3876 [[fallthrough]];
3877 case ISD::SETO: {
3878 assert(IsFP && "Unexpected integer comparison");
3879 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3880 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3881 SDValue GE = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGE, Mode),
3882 DL, VT, CmpOp0, CmpOp1, Chain);
3883 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GE);
3884 if (Chain)
3885 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
3886 N1: LT.getValue(R: 1), N2: GE.getValue(R: 1));
3887 break;
3888 }
3889
3890 // Handle <> tests using (or (ogt y x) (ogt x y)).
3891 case ISD::SETUEQ:
3892 Invert = true;
3893 [[fallthrough]];
3894 case ISD::SETONE: {
3895 assert(IsFP && "Unexpected integer comparison");
3896 SDValue LT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3897 DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3898 SDValue GT = getVectorCmp(DAG, Opcode: getVectorComparison(CC: ISD::SETOGT, Mode),
3899 DL, VT, CmpOp0, CmpOp1, Chain);
3900 Cmp = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LT, N2: GT);
3901 if (Chain)
3902 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
3903 N1: LT.getValue(R: 1), N2: GT.getValue(R: 1));
3904 break;
3905 }
3906
3907 // Otherwise a single comparison is enough. It doesn't really
3908 // matter whether we try the inversion or the swap first, since
3909 // there are no cases where both work.
3910 default:
3911 // Optimize sign-bit comparisons to signed compares.
3912 if (Mode == CmpMode::Int && (CC == ISD::SETEQ || CC == ISD::SETNE) &&
3913 ISD::isConstantSplatVectorAllZeros(N: CmpOp1.getNode())) {
3914 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
3915 APInt Mask;
3916 if (CmpOp0.getOpcode() == ISD::AND
3917 && ISD::isConstantSplatVector(N: CmpOp0.getOperand(i: 1).getNode(), SplatValue&: Mask)
3918 && Mask == APInt::getSignMask(BitWidth: EltSize)) {
3919 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
3920 CmpOp0 = CmpOp0.getOperand(i: 0);
3921 }
3922 }
3923 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3924 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
3925 else {
3926 CC = ISD::getSetCCSwappedOperands(Operation: CC);
3927 if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
3928 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0: CmpOp1, CmpOp1: CmpOp0, Chain);
3929 else
3930 llvm_unreachable("Unhandled comparison");
3931 }
3932 if (Chain)
3933 Chain = Cmp.getValue(R: 1);
3934 break;
3935 }
3936 if (Invert) {
3937 SDValue Mask =
3938 DAG.getSplatBuildVector(VT, DL, Op: DAG.getAllOnesConstant(DL, VT: MVT::i64));
3939 Cmp = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Cmp, N2: Mask);
3940 }
3941 if (Chain && Chain.getNode() != Cmp.getNode()) {
3942 SDValue Ops[2] = { Cmp, Chain };
3943 Cmp = DAG.getMergeValues(Ops, dl: DL);
3944 }
3945 return Cmp;
3946}
3947
3948SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
3949 SelectionDAG &DAG) const {
3950 SDValue CmpOp0 = Op.getOperand(i: 0);
3951 SDValue CmpOp1 = Op.getOperand(i: 1);
3952 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
3953 SDLoc DL(Op);
3954 EVT VT = Op.getValueType();
3955 if (VT.isVector())
3956 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
3957
3958 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
3959 SDValue CCReg = emitCmp(DAG, DL, C);
3960 return emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
3961}
3962
3963SDValue SystemZTargetLowering::lowerSTRICT_FSETCC(SDValue Op,
3964 SelectionDAG &DAG,
3965 bool IsSignaling) const {
3966 SDValue Chain = Op.getOperand(i: 0);
3967 SDValue CmpOp0 = Op.getOperand(i: 1);
3968 SDValue CmpOp1 = Op.getOperand(i: 2);
3969 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 3))->get();
3970 SDLoc DL(Op);
3971 EVT VT = Op.getNode()->getValueType(ResNo: 0);
3972 if (VT.isVector()) {
3973 SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
3974 Chain, IsSignaling);
3975 return Res.getValue(R: Op.getResNo());
3976 }
3977
3978 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL, Chain, IsSignaling));
3979 SDValue CCReg = emitCmp(DAG, DL, C);
3980 CCReg->setFlags(Op->getFlags());
3981 SDValue Result = emitSETCC(DAG, DL, CCReg, CCValid: C.CCValid, CCMask: C.CCMask);
3982 SDValue Ops[2] = { Result, CCReg.getValue(R: 1) };
3983 return DAG.getMergeValues(Ops, dl: DL);
3984}
3985
3986SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
3987 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
3988 SDValue CmpOp0 = Op.getOperand(i: 2);
3989 SDValue CmpOp1 = Op.getOperand(i: 3);
3990 SDValue Dest = Op.getOperand(i: 4);
3991 SDLoc DL(Op);
3992
3993 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
3994 SDValue CCReg = emitCmp(DAG, DL, C);
3995 return DAG.getNode(
3996 Opcode: SystemZISD::BR_CCMASK, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
3997 N2: DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
3998 N3: DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), N4: Dest, N5: CCReg);
3999}
4000
4001// Return true if Pos is CmpOp and Neg is the negative of CmpOp,
4002// allowing Pos and Neg to be wider than CmpOp.
4003static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
4004 return (Neg.getOpcode() == ISD::SUB &&
4005 Neg.getOperand(i: 0).getOpcode() == ISD::Constant &&
4006 Neg.getConstantOperandVal(i: 0) == 0 && Neg.getOperand(i: 1) == Pos &&
4007 (Pos == CmpOp || (Pos.getOpcode() == ISD::SIGN_EXTEND &&
4008 Pos.getOperand(i: 0) == CmpOp)));
4009}
4010
4011// Return the absolute or negative absolute of Op; IsNegative decides which.
4012static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
4013 bool IsNegative) {
4014 Op = DAG.getNode(Opcode: ISD::ABS, DL, VT: Op.getValueType(), Operand: Op);
4015 if (IsNegative)
4016 Op = DAG.getNode(Opcode: ISD::SUB, DL, VT: Op.getValueType(),
4017 N1: DAG.getConstant(Val: 0, DL, VT: Op.getValueType()), N2: Op);
4018 return Op;
4019}
4020
4021static SDValue getI128Select(SelectionDAG &DAG, const SDLoc &DL,
4022 Comparison C, SDValue TrueOp, SDValue FalseOp) {
4023 EVT VT = MVT::i128;
4024 unsigned Op;
4025
4026 if (C.CCMask == SystemZ::CCMASK_CMP_NE ||
4027 C.CCMask == SystemZ::CCMASK_CMP_GE ||
4028 C.CCMask == SystemZ::CCMASK_CMP_LE) {
4029 std::swap(a&: TrueOp, b&: FalseOp);
4030 C.CCMask ^= C.CCValid;
4031 }
4032 if (C.CCMask == SystemZ::CCMASK_CMP_LT) {
4033 std::swap(a&: C.Op0, b&: C.Op1);
4034 C.CCMask = SystemZ::CCMASK_CMP_GT;
4035 }
4036 switch (C.CCMask) {
4037 case SystemZ::CCMASK_CMP_EQ:
4038 Op = SystemZISD::VICMPE;
4039 break;
4040 case SystemZ::CCMASK_CMP_GT:
4041 if (C.ICmpType == SystemZICMP::UnsignedOnly)
4042 Op = SystemZISD::VICMPHL;
4043 else
4044 Op = SystemZISD::VICMPH;
4045 break;
4046 default:
4047 llvm_unreachable("Unhandled comparison");
4048 break;
4049 }
4050
4051 SDValue Mask = DAG.getNode(Opcode: Op, DL, VT, N1: C.Op0, N2: C.Op1);
4052 TrueOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: TrueOp, N2: Mask);
4053 FalseOp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: FalseOp, N2: DAG.getNOT(DL, Val: Mask, VT));
4054 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: TrueOp, N2: FalseOp);
4055}
4056
4057SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
4058 SelectionDAG &DAG) const {
4059 SDValue CmpOp0 = Op.getOperand(i: 0);
4060 SDValue CmpOp1 = Op.getOperand(i: 1);
4061 SDValue TrueOp = Op.getOperand(i: 2);
4062 SDValue FalseOp = Op.getOperand(i: 3);
4063 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
4064 SDLoc DL(Op);
4065
4066 // SELECT_CC involving f16 will not have the cmp-ops promoted by the
4067 // legalizer, as it will be handled according to the type of the resulting
4068 // value. Extend them here if needed.
4069 if (CmpOp0.getSimpleValueType() == MVT::f16) {
4070 CmpOp0 = DAG.getFPExtendOrRound(Op: CmpOp0, DL: SDLoc(CmpOp0), VT: MVT::f32);
4071 CmpOp1 = DAG.getFPExtendOrRound(Op: CmpOp1, DL: SDLoc(CmpOp1), VT: MVT::f32);
4072 }
4073
4074 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, Cond: CC, DL));
4075
4076 // Check for absolute and negative-absolute selections, including those
4077 // where the comparison value is sign-extended (for LPGFR and LNGFR).
4078 // This check supplements the one in DAGCombiner.
4079 if (C.Opcode == SystemZISD::ICMP && C.CCMask != SystemZ::CCMASK_CMP_EQ &&
4080 C.CCMask != SystemZ::CCMASK_CMP_NE &&
4081 C.Op1.getOpcode() == ISD::Constant &&
4082 cast<ConstantSDNode>(Val&: C.Op1)->getValueSizeInBits(ResNo: 0) <= 64 &&
4083 C.Op1->getAsZExtVal() == 0) {
4084 if (isAbsolute(CmpOp: C.Op0, Pos: TrueOp, Neg: FalseOp))
4085 return getAbsolute(DAG, DL, Op: TrueOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_LT);
4086 if (isAbsolute(CmpOp: C.Op0, Pos: FalseOp, Neg: TrueOp))
4087 return getAbsolute(DAG, DL, Op: FalseOp, IsNegative: C.CCMask & SystemZ::CCMASK_CMP_GT);
4088 }
4089
4090 if (Subtarget.hasVectorEnhancements3() &&
4091 C.Opcode == SystemZISD::ICMP &&
4092 C.Op0.getValueType() == MVT::i128 &&
4093 TrueOp.getValueType() == MVT::i128) {
4094 return getI128Select(DAG, DL, C, TrueOp, FalseOp);
4095 }
4096
4097 SDValue CCReg = emitCmp(DAG, DL, C);
4098 SDValue Ops[] = {TrueOp, FalseOp,
4099 DAG.getTargetConstant(Val: C.CCValid, DL, VT: MVT::i32),
4100 DAG.getTargetConstant(Val: C.CCMask, DL, VT: MVT::i32), CCReg};
4101
4102 return DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT: Op.getValueType(), Ops);
4103}
4104
4105SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
4106 SelectionDAG &DAG) const {
4107 SDLoc DL(Node);
4108 const GlobalValue *GV = Node->getGlobal();
4109 int64_t Offset = Node->getOffset();
4110 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4111 CodeModel::Model CM = DAG.getTarget().getCodeModel();
4112
4113 SDValue Result;
4114 if (Subtarget.isPC32DBLSymbol(GV, CM)) {
4115 if (isInt<32>(x: Offset)) {
4116 // Assign anchors at 1<<12 byte boundaries.
4117 uint64_t Anchor = Offset & ~uint64_t(0xfff);
4118 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor);
4119 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4120
4121 // The offset can be folded into the address if it is aligned to a
4122 // halfword.
4123 Offset -= Anchor;
4124 if (Offset != 0 && (Offset & 1) == 0) {
4125 SDValue Full =
4126 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: Anchor + Offset);
4127 Result = DAG.getNode(Opcode: SystemZISD::PCREL_OFFSET, DL, VT: PtrVT, N1: Full, N2: Result);
4128 Offset = 0;
4129 }
4130 } else {
4131 // Conservatively load a constant offset greater than 32 bits into a
4132 // register below.
4133 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT);
4134 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4135 }
4136 } else if (Subtarget.isTargetELF()) {
4137 Result = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: SystemZII::MO_GOT);
4138 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4139 Result = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Result,
4140 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4141 } else if (Subtarget.isTargetzOS()) {
4142 Result = getADAEntry(DAG, GV, DL, PtrVT);
4143 } else
4144 llvm_unreachable("Unexpected Subtarget");
4145
4146 // If there was a non-zero offset that we didn't fold, create an explicit
4147 // addition for it.
4148 if (Offset != 0)
4149 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4150 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4151
4152 return Result;
4153}
4154
4155SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
4156 SelectionDAG &DAG,
4157 unsigned Opcode,
4158 SDValue GOTOffset) const {
4159 SDLoc DL(Node);
4160 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4161 SDValue Chain = DAG.getEntryNode();
4162 SDValue Glue;
4163
4164 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4165 CallingConv::GHC)
4166 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4167
4168 // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12.
4169 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(VT: PtrVT);
4170 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R12D, N: GOT, Glue);
4171 Glue = Chain.getValue(R: 1);
4172 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SystemZ::R2D, N: GOTOffset, Glue);
4173 Glue = Chain.getValue(R: 1);
4174
4175 // The first call operand is the chain and the second is the TLS symbol.
4176 SmallVector<SDValue, 8> Ops;
4177 Ops.push_back(Elt: Chain);
4178 Ops.push_back(Elt: DAG.getTargetGlobalAddress(GV: Node->getGlobal(), DL,
4179 VT: Node->getValueType(ResNo: 0),
4180 offset: 0, TargetFlags: 0));
4181
4182 // Add argument registers to the end of the list so that they are
4183 // known live into the call.
4184 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R2D, VT: PtrVT));
4185 Ops.push_back(Elt: DAG.getRegister(Reg: SystemZ::R12D, VT: PtrVT));
4186
4187 // Add a register mask operand representing the call-preserved registers.
4188 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
4189 const uint32_t *Mask =
4190 TRI->getCallPreservedMask(MF: DAG.getMachineFunction(), CallingConv::C);
4191 assert(Mask && "Missing call preserved mask for calling convention");
4192 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
4193
4194 // Glue the call to the argument copies.
4195 Ops.push_back(Elt: Glue);
4196
4197 // Emit the call.
4198 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
4199 Chain = DAG.getNode(Opcode, DL, VTList: NodeTys, Ops);
4200 Glue = Chain.getValue(R: 1);
4201
4202 // Copy the return value from %r2.
4203 return DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::R2D, VT: PtrVT, Glue);
4204}
4205
4206SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
4207 SelectionDAG &DAG) const {
4208 SDValue Chain = DAG.getEntryNode();
4209 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4210
4211 // The high part of the thread pointer is in access register 0.
4212 SDValue TPHi = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A0, VT: MVT::i32);
4213 TPHi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PtrVT, Operand: TPHi);
4214
4215 // The low part of the thread pointer is in access register 1.
4216 SDValue TPLo = DAG.getCopyFromReg(Chain, dl: DL, Reg: SystemZ::A1, VT: MVT::i32);
4217 TPLo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: PtrVT, Operand: TPLo);
4218
4219 // Merge them into a single 64-bit address.
4220 SDValue TPHiShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: TPHi,
4221 N2: DAG.getConstant(Val: 32, DL, VT: PtrVT));
4222 return DAG.getNode(Opcode: ISD::OR, DL, VT: PtrVT, N1: TPHiShifted, N2: TPLo);
4223}
4224
4225SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
4226 SelectionDAG &DAG) const {
4227 if (DAG.getTarget().useEmulatedTLS())
4228 return LowerToTLSEmulatedModel(GA: Node, DAG);
4229 SDLoc DL(Node);
4230 const GlobalValue *GV = Node->getGlobal();
4231 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4232 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
4233
4234 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
4235 CallingConv::GHC)
4236 report_fatal_error(reason: "In GHC calling convention TLS is not supported");
4237
4238 SDValue TP = lowerThreadPointer(DL, DAG);
4239
4240 // Get the offset of GA from the thread pointer, based on the TLS model.
4241 SDValue Offset;
4242 switch (model) {
4243 case TLSModel::GeneralDynamic: {
4244 // Load the GOT offset of the tls_index (module ID / per-symbol offset).
4245 SystemZConstantPoolValue *CPV =
4246 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSGD);
4247
4248 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4249 Offset = DAG.getLoad(
4250 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4251 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4252
4253 // Call __tls_get_offset to retrieve the offset.
4254 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_GDCALL, GOTOffset: Offset);
4255 break;
4256 }
4257
4258 case TLSModel::LocalDynamic: {
4259 // Load the GOT offset of the module ID.
4260 SystemZConstantPoolValue *CPV =
4261 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::TLSLDM);
4262
4263 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4264 Offset = DAG.getLoad(
4265 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4266 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4267
4268 // Call __tls_get_offset to retrieve the module base offset.
4269 Offset = lowerTLSGetOffset(Node, DAG, Opcode: SystemZISD::TLS_LDCALL, GOTOffset: Offset);
4270
4271 // Note: The SystemZLDCleanupPass will remove redundant computations
4272 // of the module base offset. Count total number of local-dynamic
4273 // accesses to trigger execution of that pass.
4274 SystemZMachineFunctionInfo* MFI =
4275 DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>();
4276 MFI->incNumLocalDynamicTLSAccesses();
4277
4278 // Add the per-symbol offset.
4279 CPV = SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::DTPOFF);
4280
4281 SDValue DTPOffset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4282 DTPOffset = DAG.getLoad(
4283 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: DTPOffset,
4284 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4285
4286 Offset = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Offset, N2: DTPOffset);
4287 break;
4288 }
4289
4290 case TLSModel::InitialExec: {
4291 // Load the offset from the GOT.
4292 Offset = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0,
4293 TargetFlags: SystemZII::MO_INDNTPOFF);
4294 Offset = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Offset);
4295 Offset =
4296 DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4297 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()));
4298 break;
4299 }
4300
4301 case TLSModel::LocalExec: {
4302 // Force the offset into the constant pool and load it from there.
4303 SystemZConstantPoolValue *CPV =
4304 SystemZConstantPoolValue::Create(GV, Modifier: SystemZCP::NTPOFF);
4305
4306 Offset = DAG.getConstantPool(C: CPV, VT: PtrVT, Align: Align(8));
4307 Offset = DAG.getLoad(
4308 VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: Offset,
4309 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
4310 break;
4311 }
4312 }
4313
4314 // Add the base and offset together.
4315 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: TP, N2: Offset);
4316}
4317
4318SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
4319 SelectionDAG &DAG) const {
4320 SDLoc DL(Node);
4321 const BlockAddress *BA = Node->getBlockAddress();
4322 int64_t Offset = Node->getOffset();
4323 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4324
4325 SDValue Result = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset);
4326 Result = DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4327 return Result;
4328}
4329
4330SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
4331 SelectionDAG &DAG) const {
4332 SDLoc DL(JT);
4333 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4334 SDValue Result = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT);
4335
4336 // Use LARL to load the address of the table.
4337 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4338}
4339
4340SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
4341 SelectionDAG &DAG) const {
4342 SDLoc DL(CP);
4343 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4344
4345 SDValue Result;
4346 if (CP->isMachineConstantPoolEntry())
4347 Result =
4348 DAG.getTargetConstantPool(C: CP->getMachineCPVal(), VT: PtrVT, Align: CP->getAlign());
4349 else
4350 Result = DAG.getTargetConstantPool(C: CP->getConstVal(), VT: PtrVT, Align: CP->getAlign(),
4351 Offset: CP->getOffset());
4352
4353 // Use LARL to load the address of the constant pool entry.
4354 return DAG.getNode(Opcode: SystemZISD::PCREL_WRAPPER, DL, VT: PtrVT, Operand: Result);
4355}
4356
4357SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
4358 SelectionDAG &DAG) const {
4359 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4360 MachineFunction &MF = DAG.getMachineFunction();
4361 MachineFrameInfo &MFI = MF.getFrameInfo();
4362 MFI.setFrameAddressIsTaken(true);
4363
4364 SDLoc DL(Op);
4365 unsigned Depth = Op.getConstantOperandVal(i: 0);
4366 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4367
4368 // By definition, the frame address is the address of the back chain. (In
4369 // the case of packed stack without backchain, return the address where the
4370 // backchain would have been stored. This will either be an unused space or
4371 // contain a saved register).
4372 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
4373 SDValue BackChain = DAG.getFrameIndex(FI: BackChainIdx, VT: PtrVT);
4374
4375 if (Depth > 0) {
4376 // FIXME The frontend should detect this case.
4377 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4378 report_fatal_error(reason: "Unsupported stack frame traversal count");
4379
4380 SDValue Offset = DAG.getConstant(Val: TFL->getBackchainOffset(MF), DL, VT: PtrVT);
4381 while (Depth--) {
4382 BackChain = DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: BackChain,
4383 PtrInfo: MachinePointerInfo());
4384 BackChain = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: BackChain, N2: Offset);
4385 }
4386 }
4387
4388 return BackChain;
4389}
4390
4391SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
4392 SelectionDAG &DAG) const {
4393 MachineFunction &MF = DAG.getMachineFunction();
4394 MachineFrameInfo &MFI = MF.getFrameInfo();
4395 MFI.setReturnAddressIsTaken(true);
4396
4397 SDLoc DL(Op);
4398 unsigned Depth = Op.getConstantOperandVal(i: 0);
4399 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4400
4401 if (Depth > 0) {
4402 // FIXME The frontend should detect this case.
4403 if (!MF.getSubtarget<SystemZSubtarget>().hasBackChain())
4404 report_fatal_error(reason: "Unsupported stack frame traversal count");
4405
4406 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
4407 const auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
4408 int Offset = TFL->getReturnAddressOffset(MF);
4409 SDValue Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FrameAddr,
4410 N2: DAG.getSignedConstant(Val: Offset, DL, VT: PtrVT));
4411 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr,
4412 PtrInfo: MachinePointerInfo());
4413 }
4414
4415 // Return R14D (Elf) / R7D (XPLINK), which has the return address. Mark it an
4416 // implicit live-in.
4417 SystemZCallingConventionRegisters *CCR = Subtarget.getSpecialRegisters();
4418 Register LinkReg = MF.addLiveIn(PReg: CCR->getReturnFunctionAddressRegister(),
4419 RC: &SystemZ::GR64BitRegClass);
4420 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: LinkReg, VT: PtrVT);
4421}
4422
4423SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
4424 SelectionDAG &DAG) const {
4425 SDLoc DL(Op);
4426 SDValue In = Op.getOperand(i: 0);
4427 EVT InVT = In.getValueType();
4428 EVT ResVT = Op.getValueType();
4429
4430 // Convert loads directly. This is normally done by DAGCombiner,
4431 // but we need this case for bitcasts that are created during lowering
4432 // and which are then lowered themselves.
4433 if (auto *LoadN = dyn_cast<LoadSDNode>(Val&: In))
4434 if (ISD::isNormalLoad(N: LoadN)) {
4435 SDValue NewLoad = DAG.getLoad(VT: ResVT, dl: DL, Chain: LoadN->getChain(),
4436 Ptr: LoadN->getBasePtr(), MMO: LoadN->getMemOperand());
4437 // Update the chain uses.
4438 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LoadN, 1), To: NewLoad.getValue(R: 1));
4439 return NewLoad;
4440 }
4441
4442 if (InVT == MVT::i32 && ResVT == MVT::f32) {
4443 SDValue In64;
4444 if (Subtarget.hasHighWord()) {
4445 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL,
4446 VT: MVT::i64);
4447 In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4448 VT: MVT::i64, Operand: SDValue(U64, 0), Subreg: In);
4449 } else {
4450 In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: In);
4451 In64 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: In64,
4452 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4453 }
4454 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: In64);
4455 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32,
4456 DL, VT: MVT::f32, Operand: Out64);
4457 }
4458 if (InVT == MVT::f32 && ResVT == MVT::i32) {
4459 SDNode *U64 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
4460 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h32, DL,
4461 VT: MVT::f64, Operand: SDValue(U64, 0), Subreg: In);
4462 SDValue Out64 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
4463 if (Subtarget.hasHighWord())
4464 return DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h32, DL,
4465 VT: MVT::i32, Operand: Out64);
4466 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Out64,
4467 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
4468 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Shift);
4469 }
4470 llvm_unreachable("Unexpected bitcast combination");
4471}
4472
4473SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
4474 SelectionDAG &DAG) const {
4475
4476 if (Subtarget.isTargetXPLINK64())
4477 return lowerVASTART_XPLINK(Op, DAG);
4478 else
4479 return lowerVASTART_ELF(Op, DAG);
4480}
4481
4482SDValue SystemZTargetLowering::lowerVASTART_XPLINK(SDValue Op,
4483 SelectionDAG &DAG) const {
4484 MachineFunction &MF = DAG.getMachineFunction();
4485 SystemZMachineFunctionInfo *FuncInfo =
4486 MF.getInfo<SystemZMachineFunctionInfo>();
4487
4488 SDLoc DL(Op);
4489
4490 // vastart just stores the address of the VarArgsFrameIndex slot into the
4491 // memory location argument.
4492 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4493 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4494 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4495 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FR, Ptr: Op.getOperand(i: 1),
4496 PtrInfo: MachinePointerInfo(SV));
4497}
4498
4499SDValue SystemZTargetLowering::lowerVASTART_ELF(SDValue Op,
4500 SelectionDAG &DAG) const {
4501 MachineFunction &MF = DAG.getMachineFunction();
4502 SystemZMachineFunctionInfo *FuncInfo =
4503 MF.getInfo<SystemZMachineFunctionInfo>();
4504 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
4505
4506 SDValue Chain = Op.getOperand(i: 0);
4507 SDValue Addr = Op.getOperand(i: 1);
4508 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4509 SDLoc DL(Op);
4510
4511 // The initial values of each field.
4512 const unsigned NumFields = 4;
4513 SDValue Fields[NumFields] = {
4514 DAG.getConstant(Val: FuncInfo->getVarArgsFirstGPR(), DL, VT: PtrVT),
4515 DAG.getConstant(Val: FuncInfo->getVarArgsFirstFPR(), DL, VT: PtrVT),
4516 DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT),
4517 DAG.getFrameIndex(FI: FuncInfo->getRegSaveFrameIndex(), VT: PtrVT)
4518 };
4519
4520 // Store each field into its respective slot.
4521 SDValue MemOps[NumFields];
4522 unsigned Offset = 0;
4523 for (unsigned I = 0; I < NumFields; ++I) {
4524 SDValue FieldAddr = Addr;
4525 if (Offset != 0)
4526 FieldAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FieldAddr,
4527 N2: DAG.getIntPtrConstant(Val: Offset, DL));
4528 MemOps[I] = DAG.getStore(Chain, dl: DL, Val: Fields[I], Ptr: FieldAddr,
4529 PtrInfo: MachinePointerInfo(SV, Offset));
4530 Offset += 8;
4531 }
4532 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOps);
4533}
4534
4535SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
4536 SelectionDAG &DAG) const {
4537 SDValue Chain = Op.getOperand(i: 0);
4538 SDValue DstPtr = Op.getOperand(i: 1);
4539 SDValue SrcPtr = Op.getOperand(i: 2);
4540 const Value *DstSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 3))->getValue();
4541 const Value *SrcSV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
4542 SDLoc DL(Op);
4543
4544 uint32_t Sz =
4545 Subtarget.isTargetXPLINK64() ? getTargetMachine().getPointerSize(AS: 0) : 32;
4546 return DAG.getMemcpy(Chain, dl: DL, Dst: DstPtr, Src: SrcPtr, Size: DAG.getIntPtrConstant(Val: Sz, DL),
4547 DstAlign: Align(8), SrcAlign: Align(8), /*isVolatile*/ isVol: false,
4548 /*AlwaysInline*/ false,
4549 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(DstSV),
4550 SrcPtrInfo: MachinePointerInfo(SrcSV));
4551}
4552
4553SDValue
4554SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
4555 SelectionDAG &DAG) const {
4556 if (Subtarget.isTargetXPLINK64())
4557 return lowerDYNAMIC_STACKALLOC_XPLINK(Op, DAG);
4558 else
4559 return lowerDYNAMIC_STACKALLOC_ELF(Op, DAG);
4560}
4561
4562SDValue
4563SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op,
4564 SelectionDAG &DAG) const {
4565 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4566 MachineFunction &MF = DAG.getMachineFunction();
4567 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4568 SDValue Chain = Op.getOperand(i: 0);
4569 SDValue Size = Op.getOperand(i: 1);
4570 SDValue Align = Op.getOperand(i: 2);
4571 SDLoc DL(Op);
4572
4573 // If user has set the no alignment function attribute, ignore
4574 // alloca alignments.
4575 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4576
4577 uint64_t StackAlign = TFI->getStackAlignment();
4578 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4579 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4580
4581 SDValue NeededSpace = Size;
4582
4583 // Add extra space for alignment if needed.
4584 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4585 if (ExtraAlignSpace)
4586 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: NeededSpace,
4587 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4588
4589 bool IsSigned = false;
4590 bool DoesNotReturn = false;
4591 bool IsReturnValueUsed = false;
4592 EVT VT = Op.getValueType();
4593 SDValue AllocaCall =
4594 makeExternalCall(Chain, DAG, CalleeName: "@@ALCAXP", RetVT: VT, Ops: ArrayRef(NeededSpace),
4595 CallConv: CallingConv::C, IsSigned, DL, DoesNotReturn,
4596 IsReturnValueUsed)
4597 .first;
4598
4599 // Perform a CopyFromReg from %GPR4 (stack pointer register). Chain and Glue
4600 // to end of call in order to ensure it isn't broken up from the call
4601 // sequence.
4602 auto &Regs = Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
4603 Register SPReg = Regs.getStackPointerRegister();
4604 Chain = AllocaCall.getValue(R: 1);
4605 SDValue Glue = AllocaCall.getValue(R: 2);
4606 SDValue NewSPRegNode = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: PtrVT, Glue);
4607 Chain = NewSPRegNode.getValue(R: 1);
4608
4609 MVT PtrMVT = getPointerMemTy(DL: MF.getDataLayout());
4610 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: PtrMVT);
4611 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrMVT, N1: NewSPRegNode, N2: ArgAdjust);
4612
4613 // Dynamically realign if needed.
4614 if (ExtraAlignSpace) {
4615 Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Result,
4616 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: PtrVT));
4617 Result = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Result,
4618 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: PtrVT));
4619 }
4620
4621 SDValue Ops[2] = {Result, Chain};
4622 return DAG.getMergeValues(Ops, dl: DL);
4623}
4624
4625SDValue
4626SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(SDValue Op,
4627 SelectionDAG &DAG) const {
4628 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
4629 MachineFunction &MF = DAG.getMachineFunction();
4630 bool RealignOpt = !MF.getFunction().hasFnAttribute(Kind: "no-realign-stack");
4631 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
4632
4633 SDValue Chain = Op.getOperand(i: 0);
4634 SDValue Size = Op.getOperand(i: 1);
4635 SDValue Align = Op.getOperand(i: 2);
4636 SDLoc DL(Op);
4637
4638 // If user has set the no alignment function attribute, ignore
4639 // alloca alignments.
4640 uint64_t AlignVal = (RealignOpt ? Align->getAsZExtVal() : 0);
4641
4642 uint64_t StackAlign = TFI->getStackAlignment();
4643 uint64_t RequiredAlign = std::max(a: AlignVal, b: StackAlign);
4644 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
4645
4646 Register SPReg = getStackPointerRegisterToSaveRestore();
4647 SDValue NeededSpace = Size;
4648
4649 // Get a reference to the stack pointer.
4650 SDValue OldSP = DAG.getCopyFromReg(Chain, dl: DL, Reg: SPReg, VT: MVT::i64);
4651
4652 // If we need a backchain, save it now.
4653 SDValue Backchain;
4654 if (StoreBackchain)
4655 Backchain = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
4656 PtrInfo: MachinePointerInfo());
4657
4658 // Add extra space for alignment if needed.
4659 if (ExtraAlignSpace)
4660 NeededSpace = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NeededSpace,
4661 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4662
4663 // Get the new stack pointer value.
4664 SDValue NewSP;
4665 if (hasInlineStackProbe(MF)) {
4666 NewSP = DAG.getNode(Opcode: SystemZISD::PROBED_ALLOCA, DL,
4667 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other), N1: Chain, N2: OldSP, N3: NeededSpace);
4668 Chain = NewSP.getValue(R: 1);
4669 }
4670 else {
4671 NewSP = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: OldSP, N2: NeededSpace);
4672 // Copy the new stack pointer back.
4673 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: SPReg, N: NewSP);
4674 }
4675
4676 // The allocated data lives above the 160 bytes allocated for the standard
4677 // frame, plus any outgoing stack arguments. We don't know how much that
4678 // amounts to yet, so emit a special ADJDYNALLOC placeholder.
4679 SDValue ArgAdjust = DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4680 SDValue Result = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: NewSP, N2: ArgAdjust);
4681
4682 // Dynamically realign if needed.
4683 if (RequiredAlign > StackAlign) {
4684 Result =
4685 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Result,
4686 N2: DAG.getConstant(Val: ExtraAlignSpace, DL, VT: MVT::i64));
4687 Result =
4688 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, N1: Result,
4689 N2: DAG.getConstant(Val: ~(RequiredAlign - 1), DL, VT: MVT::i64));
4690 }
4691
4692 if (StoreBackchain)
4693 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
4694 PtrInfo: MachinePointerInfo());
4695
4696 SDValue Ops[2] = { Result, Chain };
4697 return DAG.getMergeValues(Ops, dl: DL);
4698}
4699
4700SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
4701 SDValue Op, SelectionDAG &DAG) const {
4702 SDLoc DL(Op);
4703
4704 return DAG.getNode(Opcode: SystemZISD::ADJDYNALLOC, DL, VT: MVT::i64);
4705}
4706
4707SDValue SystemZTargetLowering::lowerMULH(SDValue Op,
4708 SelectionDAG &DAG,
4709 unsigned Opcode) const {
4710 EVT VT = Op.getValueType();
4711 SDLoc DL(Op);
4712 SDValue Even, Odd;
4713
4714 // This custom expander is only used on z17 and later for 64-bit types.
4715 assert(!is32Bit(VT));
4716 assert(Subtarget.hasMiscellaneousExtensions2());
4717
4718 // SystemZISD::xMUL_LOHI returns the low result in the odd register and
4719 // the high result in the even register. Return the latter.
4720 lowerGR128Binary(DAG, DL, VT, Opcode,
4721 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even, Odd);
4722 return Even;
4723}
4724
4725SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
4726 SelectionDAG &DAG) const {
4727 EVT VT = Op.getValueType();
4728 SDLoc DL(Op);
4729 SDValue Ops[2];
4730 if (is32Bit(VT))
4731 // Just do a normal 64-bit multiplication and extract the results.
4732 // We define this so that it can be used for constant division.
4733 lowerMUL_LOHI32(DAG, DL, Extend: ISD::SIGN_EXTEND, Op0: Op.getOperand(i: 0),
4734 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4735 else if (Subtarget.hasMiscellaneousExtensions2())
4736 // SystemZISD::SMUL_LOHI returns the low result in the odd register and
4737 // the high result in the even register. ISD::SMUL_LOHI is defined to
4738 // return the low half first, so the results are in reverse order.
4739 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SMUL_LOHI,
4740 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4741 else {
4742 // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:
4743 //
4744 // (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64)
4745 //
4746 // but using the fact that the upper halves are either all zeros
4747 // or all ones:
4748 //
4749 // (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)
4750 //
4751 // and grouping the right terms together since they are quicker than the
4752 // multiplication:
4753 //
4754 // (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
4755 SDValue C63 = DAG.getConstant(Val: 63, DL, VT: MVT::i64);
4756 SDValue LL = Op.getOperand(i: 0);
4757 SDValue RL = Op.getOperand(i: 1);
4758 SDValue LH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LL, N2: C63);
4759 SDValue RH = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: RL, N2: C63);
4760 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4761 // the high result in the even register. ISD::SMUL_LOHI is defined to
4762 // return the low half first, so the results are in reverse order.
4763 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4764 Op0: LL, Op1: RL, Even&: Ops[1], Odd&: Ops[0]);
4765 SDValue NegLLTimesRH = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LL, N2: RH);
4766 SDValue NegLHTimesRL = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LH, N2: RL);
4767 SDValue NegSum = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NegLLTimesRH, N2: NegLHTimesRL);
4768 Ops[1] = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Ops[1], N2: NegSum);
4769 }
4770 return DAG.getMergeValues(Ops, dl: DL);
4771}
4772
4773SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
4774 SelectionDAG &DAG) const {
4775 EVT VT = Op.getValueType();
4776 SDLoc DL(Op);
4777 SDValue Ops[2];
4778 if (is32Bit(VT))
4779 // Just do a normal 64-bit multiplication and extract the results.
4780 // We define this so that it can be used for constant division.
4781 lowerMUL_LOHI32(DAG, DL, Extend: ISD::ZERO_EXTEND, Op0: Op.getOperand(i: 0),
4782 Op1: Op.getOperand(i: 1), Hi&: Ops[1], Lo&: Ops[0]);
4783 else
4784 // SystemZISD::UMUL_LOHI returns the low result in the odd register and
4785 // the high result in the even register. ISD::UMUL_LOHI is defined to
4786 // return the low half first, so the results are in reverse order.
4787 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UMUL_LOHI,
4788 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4789 return DAG.getMergeValues(Ops, dl: DL);
4790}
4791
4792SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
4793 SelectionDAG &DAG) const {
4794 SDValue Op0 = Op.getOperand(i: 0);
4795 SDValue Op1 = Op.getOperand(i: 1);
4796 EVT VT = Op.getValueType();
4797 SDLoc DL(Op);
4798
4799 // We use DSGF for 32-bit division. This means the first operand must
4800 // always be 64-bit, and the second operand should be 32-bit whenever
4801 // that is possible, to improve performance.
4802 if (is32Bit(VT))
4803 Op0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: Op0);
4804 else if (DAG.ComputeNumSignBits(Op: Op1) > 32)
4805 Op1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Op1);
4806
4807 // DSG(F) returns the remainder in the even register and the
4808 // quotient in the odd register.
4809 SDValue Ops[2];
4810 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::SDIVREM, Op0, Op1, Even&: Ops[1], Odd&: Ops[0]);
4811 return DAG.getMergeValues(Ops, dl: DL);
4812}
4813
4814SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
4815 SelectionDAG &DAG) const {
4816 EVT VT = Op.getValueType();
4817 SDLoc DL(Op);
4818
4819 // DL(G) returns the remainder in the even register and the
4820 // quotient in the odd register.
4821 SDValue Ops[2];
4822 lowerGR128Binary(DAG, DL, VT, Opcode: SystemZISD::UDIVREM,
4823 Op0: Op.getOperand(i: 0), Op1: Op.getOperand(i: 1), Even&: Ops[1], Odd&: Ops[0]);
4824 return DAG.getMergeValues(Ops, dl: DL);
4825}
4826
4827SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
4828 assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");
4829
4830 // Get the known-zero masks for each operand.
4831 SDValue Ops[] = {Op.getOperand(i: 0), Op.getOperand(i: 1)};
4832 KnownBits Known[2] = {DAG.computeKnownBits(Op: Ops[0]),
4833 DAG.computeKnownBits(Op: Ops[1])};
4834
4835 // See if the upper 32 bits of one operand and the lower 32 bits of the
4836 // other are known zero. They are the low and high operands respectively.
4837 uint64_t Masks[] = { Known[0].Zero.getZExtValue(),
4838 Known[1].Zero.getZExtValue() };
4839 unsigned High, Low;
4840 if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
4841 High = 1, Low = 0;
4842 else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
4843 High = 0, Low = 1;
4844 else
4845 return Op;
4846
4847 SDValue LowOp = Ops[Low];
4848 SDValue HighOp = Ops[High];
4849
4850 // If the high part is a constant, we're better off using IILH.
4851 if (HighOp.getOpcode() == ISD::Constant)
4852 return Op;
4853
4854 // If the low part is a constant that is outside the range of LHI,
4855 // then we're better off using IILF.
4856 if (LowOp.getOpcode() == ISD::Constant) {
4857 int64_t Value = int32_t(LowOp->getAsZExtVal());
4858 if (!isInt<16>(x: Value))
4859 return Op;
4860 }
4861
4862 // Check whether the high part is an AND that doesn't change the
4863 // high 32 bits and just masks out low bits. We can skip it if so.
4864 if (HighOp.getOpcode() == ISD::AND &&
4865 HighOp.getOperand(i: 1).getOpcode() == ISD::Constant) {
4866 SDValue HighOp0 = HighOp.getOperand(i: 0);
4867 uint64_t Mask = HighOp.getConstantOperandVal(i: 1);
4868 if (DAG.MaskedValueIsZero(Op: HighOp0, Mask: APInt(64, ~(Mask | 0xffffffff))))
4869 HighOp = HighOp0;
4870 }
4871
4872 // Take advantage of the fact that all GR32 operations only change the
4873 // low 32 bits by truncating Low to an i32 and inserting it directly
4874 // using a subreg. The interesting cases are those where the truncation
4875 // can be folded.
4876 SDLoc DL(Op);
4877 SDValue Low32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: LowOp);
4878 return DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_l32, DL,
4879 VT: MVT::i64, Operand: HighOp, Subreg: Low32);
4880}
4881
4882// Lower SADDO/SSUBO/UADDO/USUBO nodes.
4883SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
4884 SelectionDAG &DAG) const {
4885 SDNode *N = Op.getNode();
4886 SDValue LHS = N->getOperand(Num: 0);
4887 SDValue RHS = N->getOperand(Num: 1);
4888 SDLoc DL(N);
4889
4890 if (N->getValueType(ResNo: 0) == MVT::i128) {
4891 unsigned BaseOp = 0;
4892 unsigned FlagOp = 0;
4893 bool IsBorrow = false;
4894 switch (Op.getOpcode()) {
4895 default: llvm_unreachable("Unknown instruction!");
4896 case ISD::UADDO:
4897 BaseOp = ISD::ADD;
4898 FlagOp = SystemZISD::VACC;
4899 break;
4900 case ISD::USUBO:
4901 BaseOp = ISD::SUB;
4902 FlagOp = SystemZISD::VSCBI;
4903 IsBorrow = true;
4904 break;
4905 }
4906 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
4907 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS);
4908 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
4909 N2: DAG.getValueType(MVT::i1));
4910 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
4911 if (IsBorrow)
4912 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
4913 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
4914 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
4915 }
4916
4917 unsigned BaseOp = 0;
4918 unsigned CCValid = 0;
4919 unsigned CCMask = 0;
4920
4921 switch (Op.getOpcode()) {
4922 default: llvm_unreachable("Unknown instruction!");
4923 case ISD::SADDO:
4924 BaseOp = SystemZISD::SADDO;
4925 CCValid = SystemZ::CCMASK_ARITH;
4926 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4927 break;
4928 case ISD::SSUBO:
4929 BaseOp = SystemZISD::SSUBO;
4930 CCValid = SystemZ::CCMASK_ARITH;
4931 CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
4932 break;
4933 case ISD::UADDO:
4934 BaseOp = SystemZISD::UADDO;
4935 CCValid = SystemZ::CCMASK_LOGICAL;
4936 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
4937 break;
4938 case ISD::USUBO:
4939 BaseOp = SystemZISD::USUBO;
4940 CCValid = SystemZ::CCMASK_LOGICAL;
4941 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
4942 break;
4943 }
4944
4945 SDVTList VTs = DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::i32);
4946 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS);
4947
4948 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
4949 if (N->getValueType(ResNo: 1) == MVT::i1)
4950 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
4951
4952 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
4953}
4954
4955static bool isAddCarryChain(SDValue Carry) {
4956 while (Carry.getOpcode() == ISD::UADDO_CARRY &&
4957 Carry->getValueType(ResNo: 0) != MVT::i128)
4958 Carry = Carry.getOperand(i: 2);
4959 return Carry.getOpcode() == ISD::UADDO &&
4960 Carry->getValueType(ResNo: 0) != MVT::i128;
4961}
4962
4963static bool isSubBorrowChain(SDValue Carry) {
4964 while (Carry.getOpcode() == ISD::USUBO_CARRY &&
4965 Carry->getValueType(ResNo: 0) != MVT::i128)
4966 Carry = Carry.getOperand(i: 2);
4967 return Carry.getOpcode() == ISD::USUBO &&
4968 Carry->getValueType(ResNo: 0) != MVT::i128;
4969}
4970
4971// Lower UADDO_CARRY/USUBO_CARRY nodes.
4972SDValue SystemZTargetLowering::lowerUADDSUBO_CARRY(SDValue Op,
4973 SelectionDAG &DAG) const {
4974
4975 SDNode *N = Op.getNode();
4976 MVT VT = N->getSimpleValueType(ResNo: 0);
4977
4978 // Let legalize expand this if it isn't a legal type yet.
4979 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
4980 return SDValue();
4981
4982 SDValue LHS = N->getOperand(Num: 0);
4983 SDValue RHS = N->getOperand(Num: 1);
4984 SDValue Carry = Op.getOperand(i: 2);
4985 SDLoc DL(N);
4986
4987 if (VT == MVT::i128) {
4988 unsigned BaseOp = 0;
4989 unsigned FlagOp = 0;
4990 bool IsBorrow = false;
4991 switch (Op.getOpcode()) {
4992 default: llvm_unreachable("Unknown instruction!");
4993 case ISD::UADDO_CARRY:
4994 BaseOp = SystemZISD::VAC;
4995 FlagOp = SystemZISD::VACCC;
4996 break;
4997 case ISD::USUBO_CARRY:
4998 BaseOp = SystemZISD::VSBI;
4999 FlagOp = SystemZISD::VSBCBI;
5000 IsBorrow = true;
5001 break;
5002 }
5003 if (IsBorrow)
5004 Carry = DAG.getNode(Opcode: ISD::XOR, DL, VT: Carry.getValueType(),
5005 N1: Carry, N2: DAG.getConstant(Val: 1, DL, VT: Carry.getValueType()));
5006 Carry = DAG.getZExtOrTrunc(Op: Carry, DL, VT: MVT::i128);
5007 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
5008 SDValue Flag = DAG.getNode(Opcode: FlagOp, DL, VT: MVT::i128, N1: LHS, N2: RHS, N3: Carry);
5009 Flag = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: MVT::i128, N1: Flag,
5010 N2: DAG.getValueType(MVT::i1));
5011 Flag = DAG.getZExtOrTrunc(Op: Flag, DL, VT: N->getValueType(ResNo: 1));
5012 if (IsBorrow)
5013 Flag = DAG.getNode(Opcode: ISD::XOR, DL, VT: Flag.getValueType(),
5014 N1: Flag, N2: DAG.getConstant(Val: 1, DL, VT: Flag.getValueType()));
5015 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: Flag);
5016 }
5017
5018 unsigned BaseOp = 0;
5019 unsigned CCValid = 0;
5020 unsigned CCMask = 0;
5021
5022 switch (Op.getOpcode()) {
5023 default: llvm_unreachable("Unknown instruction!");
5024 case ISD::UADDO_CARRY:
5025 if (!isAddCarryChain(Carry))
5026 return SDValue();
5027
5028 BaseOp = SystemZISD::ADDCARRY;
5029 CCValid = SystemZ::CCMASK_LOGICAL;
5030 CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
5031 break;
5032 case ISD::USUBO_CARRY:
5033 if (!isSubBorrowChain(Carry))
5034 return SDValue();
5035
5036 BaseOp = SystemZISD::SUBCARRY;
5037 CCValid = SystemZ::CCMASK_LOGICAL;
5038 CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
5039 break;
5040 }
5041
5042 // Set the condition code from the carry flag.
5043 Carry = DAG.getNode(Opcode: SystemZISD::GET_CCMASK, DL, VT: MVT::i32, N1: Carry,
5044 N2: DAG.getConstant(Val: CCValid, DL, VT: MVT::i32),
5045 N3: DAG.getConstant(Val: CCMask, DL, VT: MVT::i32));
5046
5047 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: MVT::i32);
5048 SDValue Result = DAG.getNode(Opcode: BaseOp, DL, VTList: VTs, N1: LHS, N2: RHS, N3: Carry);
5049
5050 SDValue SetCC = emitSETCC(DAG, DL, CCReg: Result.getValue(R: 1), CCValid, CCMask);
5051 if (N->getValueType(ResNo: 1) == MVT::i1)
5052 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: SetCC);
5053
5054 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Result, N2: SetCC);
5055}
5056
5057SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
5058 SelectionDAG &DAG) const {
5059 EVT VT = Op.getValueType();
5060 SDLoc DL(Op);
5061 Op = Op.getOperand(i: 0);
5062
5063 if (VT.getScalarSizeInBits() == 128) {
5064 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i64, Operand: Op);
5065 Op = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: MVT::v2i64, Operand: Op);
5066 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v2i64, DL,
5067 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i64));
5068 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5069 return Op;
5070 }
5071
5072 // Handle vector types via VPOPCT.
5073 if (VT.isVector()) {
5074 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Op);
5075 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::v16i8, Operand: Op);
5076 switch (VT.getScalarSizeInBits()) {
5077 case 8:
5078 break;
5079 case 16: {
5080 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
5081 SDValue Shift = DAG.getConstant(Val: 8, DL, VT: MVT::i32);
5082 SDValue Tmp = DAG.getNode(Opcode: SystemZISD::VSHL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
5083 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
5084 Op = DAG.getNode(Opcode: SystemZISD::VSRL_BY_SCALAR, DL, VT, N1: Op, N2: Shift);
5085 break;
5086 }
5087 case 32: {
5088 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
5089 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5090 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5091 break;
5092 }
5093 case 64: {
5094 SDValue Tmp = DAG.getSplatBuildVector(VT: MVT::v16i8, DL,
5095 Op: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
5096 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Tmp);
5097 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT, N1: Op, N2: Tmp);
5098 break;
5099 }
5100 default:
5101 llvm_unreachable("Unexpected type");
5102 }
5103 return Op;
5104 }
5105
5106 // Get the known-zero mask for the operand.
5107 KnownBits Known = DAG.computeKnownBits(Op);
5108 unsigned NumSignificantBits = Known.getMaxValue().getActiveBits();
5109 if (NumSignificantBits == 0)
5110 return DAG.getConstant(Val: 0, DL, VT);
5111
5112 // Skip known-zero high parts of the operand.
5113 int64_t OrigBitSize = VT.getSizeInBits();
5114 int64_t BitSize = llvm::bit_ceil(Value: NumSignificantBits);
5115 BitSize = std::min(a: BitSize, b: OrigBitSize);
5116
5117 // The POPCNT instruction counts the number of bits in each byte.
5118 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op);
5119 Op = DAG.getNode(Opcode: SystemZISD::POPCNT, DL, VT: MVT::i64, Operand: Op);
5120 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
5121
5122 // Add up per-byte counts in a binary tree. All bits of Op at
5123 // position larger than BitSize remain zero throughout.
5124 for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
5125 SDValue Tmp = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Op, N2: DAG.getConstant(Val: I, DL, VT));
5126 if (BitSize != OrigBitSize)
5127 Tmp = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Tmp,
5128 N2: DAG.getConstant(Val: ((uint64_t)1 << BitSize) - 1, DL, VT));
5129 Op = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op, N2: Tmp);
5130 }
5131
5132 // Extract overall result from high byte.
5133 if (BitSize > 8)
5134 Op = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Op,
5135 N2: DAG.getConstant(Val: BitSize - 8, DL, VT));
5136
5137 return Op;
5138}
5139
5140SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
5141 SelectionDAG &DAG) const {
5142 SDLoc DL(Op);
5143 AtomicOrdering FenceOrdering =
5144 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
5145 SyncScope::ID FenceSSID =
5146 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
5147
5148 // The only fence that needs an instruction is a sequentially-consistent
5149 // cross-thread fence.
5150 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
5151 FenceSSID == SyncScope::System) {
5152 return SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL, VT: MVT::Other,
5153 Op1: Op.getOperand(i: 0)),
5154 0);
5155 }
5156
5157 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
5158 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL, VT: MVT::Other, Operand: Op.getOperand(i: 0));
5159}
5160
5161SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
5162 SelectionDAG &DAG) const {
5163 EVT RegVT = Op.getValueType();
5164 if (RegVT.getSizeInBits() == 128)
5165 return lowerATOMIC_LDST_I128(Op, DAG);
5166 return lowerLoadF16(Op, DAG);
5167}
5168
5169SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
5170 SelectionDAG &DAG) const {
5171 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5172 if (Node->getMemoryVT().getSizeInBits() == 128)
5173 return lowerATOMIC_LDST_I128(Op, DAG);
5174 return lowerStoreF16(Op, DAG);
5175}
5176
5177SDValue SystemZTargetLowering::lowerATOMIC_LDST_I128(SDValue Op,
5178 SelectionDAG &DAG) const {
5179 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5180 assert(
5181 (Node->getMemoryVT() == MVT::i128 || Node->getMemoryVT() == MVT::f128) &&
5182 "Only custom lowering i128 or f128.");
5183 // Use same code to handle both legal and non-legal i128 types.
5184 SmallVector<SDValue, 2> Results;
5185 LowerOperationWrapper(N: Node, Results, DAG);
5186 return DAG.getMergeValues(Ops: Results, dl: SDLoc(Op));
5187}
5188
5189// Prepare for a Compare And Swap for a subword operation. This needs to be
5190// done in memory with 4 bytes at natural alignment.
5191static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL,
5192 SDValue &AlignedAddr, SDValue &BitShift,
5193 SDValue &NegBitShift) {
5194 EVT PtrVT = Addr.getValueType();
5195 EVT WideVT = MVT::i32;
5196
5197 // Get the address of the containing word.
5198 AlignedAddr = DAG.getNode(Opcode: ISD::AND, DL, VT: PtrVT, N1: Addr,
5199 N2: DAG.getSignedConstant(Val: -4, DL, VT: PtrVT));
5200
5201 // Get the number of bits that the word must be rotated left in order
5202 // to bring the field to the top bits of a GR32.
5203 BitShift = DAG.getNode(Opcode: ISD::SHL, DL, VT: PtrVT, N1: Addr,
5204 N2: DAG.getConstant(Val: 3, DL, VT: PtrVT));
5205 BitShift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: WideVT, Operand: BitShift);
5206
5207 // Get the complementing shift amount, for rotating a field in the top
5208 // bits back to its proper position.
5209 NegBitShift = DAG.getNode(Opcode: ISD::SUB, DL, VT: WideVT,
5210 N1: DAG.getConstant(Val: 0, DL, VT: WideVT), N2: BitShift);
5211
5212}
5213
5214// Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first
5215// two into the fullword ATOMIC_LOADW_* operation given by Opcode.
5216SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
5217 SelectionDAG &DAG,
5218 unsigned Opcode) const {
5219 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5220
5221 // 32-bit operations need no special handling.
5222 EVT NarrowVT = Node->getMemoryVT();
5223 EVT WideVT = MVT::i32;
5224 if (NarrowVT == WideVT)
5225 return Op;
5226
5227 int64_t BitSize = NarrowVT.getSizeInBits();
5228 SDValue ChainIn = Node->getChain();
5229 SDValue Addr = Node->getBasePtr();
5230 SDValue Src2 = Node->getVal();
5231 MachineMemOperand *MMO = Node->getMemOperand();
5232 SDLoc DL(Node);
5233
5234 // Convert atomic subtracts of constants into additions.
5235 if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
5236 if (auto *Const = dyn_cast<ConstantSDNode>(Val&: Src2)) {
5237 Opcode = SystemZISD::ATOMIC_LOADW_ADD;
5238 Src2 = DAG.getSignedConstant(Val: -Const->getSExtValue(), DL,
5239 VT: Src2.getValueType());
5240 }
5241
5242 SDValue AlignedAddr, BitShift, NegBitShift;
5243 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5244
5245 // Extend the source operand to 32 bits and prepare it for the inner loop.
5246 // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
5247 // operations require the source to be shifted in advance. (This shift
5248 // can be folded if the source is constant.) For AND and NAND, the lower
5249 // bits must be set, while for other opcodes they should be left clear.
5250 if (Opcode != SystemZISD::ATOMIC_SWAPW)
5251 Src2 = DAG.getNode(Opcode: ISD::SHL, DL, VT: WideVT, N1: Src2,
5252 N2: DAG.getConstant(Val: 32 - BitSize, DL, VT: WideVT));
5253 if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
5254 Opcode == SystemZISD::ATOMIC_LOADW_NAND)
5255 Src2 = DAG.getNode(Opcode: ISD::OR, DL, VT: WideVT, N1: Src2,
5256 N2: DAG.getConstant(Val: uint32_t(-1) >> BitSize, DL, VT: WideVT));
5257
5258 // Construct the ATOMIC_LOADW_* node.
5259 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::Other);
5260 SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
5261 DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5262 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops,
5263 MemVT: NarrowVT, MMO);
5264
5265 // Rotate the result of the final CS so that the field is in the lower
5266 // bits of a GR32, then truncate it.
5267 SDValue ResultShift = DAG.getNode(Opcode: ISD::ADD, DL, VT: WideVT, N1: BitShift,
5268 N2: DAG.getConstant(Val: BitSize, DL, VT: WideVT));
5269 SDValue Result = DAG.getNode(Opcode: ISD::ROTL, DL, VT: WideVT, N1: AtomicOp, N2: ResultShift);
5270
5271 SDValue RetOps[2] = { Result, AtomicOp.getValue(R: 1) };
5272 return DAG.getMergeValues(Ops: RetOps, dl: DL);
5273}
5274
5275// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations into
5276// ATOMIC_LOADW_SUBs and convert 32- and 64-bit operations into additions.
5277SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
5278 SelectionDAG &DAG) const {
5279 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5280 EVT MemVT = Node->getMemoryVT();
5281 if (MemVT == MVT::i32 || MemVT == MVT::i64) {
5282 // A full-width operation: negate and use LAA(G).
5283 assert(Op.getValueType() == MemVT && "Mismatched VTs");
5284 assert(Subtarget.hasInterlockedAccess1() &&
5285 "Should have been expanded by AtomicExpand pass.");
5286 SDValue Src2 = Node->getVal();
5287 SDLoc DL(Src2);
5288 SDValue NegSrc2 =
5289 DAG.getNode(Opcode: ISD::SUB, DL, VT: MemVT, N1: DAG.getConstant(Val: 0, DL, VT: MemVT), N2: Src2);
5290 return DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT,
5291 Chain: Node->getChain(), Ptr: Node->getBasePtr(), Val: NegSrc2,
5292 MMO: Node->getMemOperand());
5293 }
5294
5295 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_SUB);
5296}
5297
5298// Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
5299SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
5300 SelectionDAG &DAG) const {
5301 auto *Node = cast<AtomicSDNode>(Val: Op.getNode());
5302 SDValue ChainIn = Node->getOperand(Num: 0);
5303 SDValue Addr = Node->getOperand(Num: 1);
5304 SDValue CmpVal = Node->getOperand(Num: 2);
5305 SDValue SwapVal = Node->getOperand(Num: 3);
5306 MachineMemOperand *MMO = Node->getMemOperand();
5307 SDLoc DL(Node);
5308
5309 if (Node->getMemoryVT() == MVT::i128) {
5310 // Use same code to handle both legal and non-legal i128 types.
5311 SmallVector<SDValue, 3> Results;
5312 LowerOperationWrapper(N: Node, Results, DAG);
5313 return DAG.getMergeValues(Ops: Results, dl: DL);
5314 }
5315
5316 // We have native support for 32-bit and 64-bit compare and swap, but we
5317 // still need to expand extracting the "success" result from the CC.
5318 EVT NarrowVT = Node->getMemoryVT();
5319 EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
5320 if (NarrowVT == WideVT) {
5321 SDVTList Tys = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5322 SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
5323 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP,
5324 dl: DL, VTList: Tys, Ops, MemVT: NarrowVT, MMO);
5325 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5326 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
5327
5328 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: AtomicOp.getValue(R: 0));
5329 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5330 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5331 return SDValue();
5332 }
5333
5334 // Convert 8-bit and 16-bit compare and swap to a loop, implemented
5335 // via a fullword ATOMIC_CMP_SWAPW operation.
5336 int64_t BitSize = NarrowVT.getSizeInBits();
5337
5338 SDValue AlignedAddr, BitShift, NegBitShift;
5339 getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift);
5340
5341 // Construct the ATOMIC_CMP_SWAPW node.
5342 SDVTList VTList = DAG.getVTList(VT1: WideVT, VT2: MVT::i32, VT3: MVT::Other);
5343 SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
5344 NegBitShift, DAG.getConstant(Val: BitSize, DL, VT: WideVT) };
5345 SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAPW, dl: DL,
5346 VTList, Ops, MemVT: NarrowVT, MMO);
5347 SDValue Success = emitSETCC(DAG, DL, CCReg: AtomicOp.getValue(R: 1),
5348 CCValid: SystemZ::CCMASK_ICMP, CCMask: SystemZ::CCMASK_CMP_EQ);
5349
5350 // emitAtomicCmpSwapW() will zero extend the result (original value).
5351 SDValue OrigVal = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: WideVT, N1: AtomicOp.getValue(R: 0),
5352 N2: DAG.getValueType(NarrowVT));
5353 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 0), To: OrigVal);
5354 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 1), To: Success);
5355 DAG.ReplaceAllUsesOfValueWith(From: Op.getValue(R: 2), To: AtomicOp.getValue(R: 2));
5356 return SDValue();
5357}
5358
5359MachineMemOperand::Flags
5360SystemZTargetLowering::getTargetMMOFlags(const Instruction &I) const {
5361 // Because of how we convert atomic_load and atomic_store to normal loads and
5362 // stores in the DAG, we need to ensure that the MMOs are marked volatile
5363 // since DAGCombine hasn't been updated to account for atomic, but non
5364 // volatile loads. (See D57601)
5365 if (auto *SI = dyn_cast<StoreInst>(Val: &I))
5366 if (SI->isAtomic())
5367 return MachineMemOperand::MOVolatile;
5368 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
5369 if (LI->isAtomic())
5370 return MachineMemOperand::MOVolatile;
5371 if (auto *AI = dyn_cast<AtomicRMWInst>(Val: &I))
5372 if (AI->isAtomic())
5373 return MachineMemOperand::MOVolatile;
5374 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: &I))
5375 if (AI->isAtomic())
5376 return MachineMemOperand::MOVolatile;
5377 return MachineMemOperand::MONone;
5378}
5379
5380SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
5381 SelectionDAG &DAG) const {
5382 MachineFunction &MF = DAG.getMachineFunction();
5383 auto *Regs = Subtarget.getSpecialRegisters();
5384 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5385 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5386 "in GHC calling convention");
5387 return DAG.getCopyFromReg(Chain: Op.getOperand(i: 0), dl: SDLoc(Op),
5388 Reg: Regs->getStackPointerRegister(), VT: Op.getValueType());
5389}
5390
5391SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
5392 SelectionDAG &DAG) const {
5393 MachineFunction &MF = DAG.getMachineFunction();
5394 auto *Regs = Subtarget.getSpecialRegisters();
5395 bool StoreBackchain = MF.getSubtarget<SystemZSubtarget>().hasBackChain();
5396
5397 if (MF.getFunction().getCallingConv() == CallingConv::GHC)
5398 report_fatal_error(reason: "Variable-sized stack allocations are not supported "
5399 "in GHC calling convention");
5400
5401 SDValue Chain = Op.getOperand(i: 0);
5402 SDValue NewSP = Op.getOperand(i: 1);
5403 SDValue Backchain;
5404 SDLoc DL(Op);
5405
5406 if (StoreBackchain) {
5407 SDValue OldSP = DAG.getCopyFromReg(
5408 Chain, dl: DL, Reg: Regs->getStackPointerRegister(), VT: MVT::i64);
5409 Backchain = DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: getBackchainAddress(SP: OldSP, DAG),
5410 PtrInfo: MachinePointerInfo());
5411 }
5412
5413 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Regs->getStackPointerRegister(), N: NewSP);
5414
5415 if (StoreBackchain)
5416 Chain = DAG.getStore(Chain, dl: DL, Val: Backchain, Ptr: getBackchainAddress(SP: NewSP, DAG),
5417 PtrInfo: MachinePointerInfo());
5418
5419 return Chain;
5420}
5421
5422SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
5423 SelectionDAG &DAG) const {
5424 bool IsData = Op.getConstantOperandVal(i: 4);
5425 if (!IsData)
5426 // Just preserve the chain.
5427 return Op.getOperand(i: 0);
5428
5429 SDLoc DL(Op);
5430 bool IsWrite = Op.getConstantOperandVal(i: 2);
5431 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
5432 auto *Node = cast<MemIntrinsicSDNode>(Val: Op.getNode());
5433 SDValue Ops[] = {Op.getOperand(i: 0), DAG.getTargetConstant(Val: Code, DL, VT: MVT::i32),
5434 Op.getOperand(i: 1)};
5435 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::PREFETCH, dl: DL,
5436 VTList: Node->getVTList(), Ops,
5437 MemVT: Node->getMemoryVT(), MMO: Node->getMemOperand());
5438}
5439
5440SDValue
5441SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
5442 SelectionDAG &DAG) const {
5443 unsigned Opcode, CCValid;
5444 if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) {
5445 assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
5446 SDNode *Node = emitIntrinsicWithCCAndChain(DAG, Op, Opcode);
5447 SDValue CC = getCCResult(DAG, CCReg: SDValue(Node, 0));
5448 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Op.getNode(), 0), To: CC);
5449 return SDValue();
5450 }
5451
5452 return SDValue();
5453}
5454
5455SDValue
5456SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
5457 SelectionDAG &DAG) const {
5458 unsigned Opcode, CCValid;
5459 if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
5460 SDNode *Node = emitIntrinsicWithCC(DAG, Op, Opcode);
5461 if (Op->getNumValues() == 1)
5462 return getCCResult(DAG, CCReg: SDValue(Node, 0));
5463 assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");
5464 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op), VTList: Op->getVTList(),
5465 N1: SDValue(Node, 0), N2: getCCResult(DAG, CCReg: SDValue(Node, 1)));
5466 }
5467
5468 unsigned Id = Op.getConstantOperandVal(i: 0);
5469 switch (Id) {
5470 case Intrinsic::thread_pointer:
5471 return lowerThreadPointer(DL: SDLoc(Op), DAG);
5472
5473 case Intrinsic::s390_vpdi:
5474 return DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL: SDLoc(Op), VT: Op.getValueType(),
5475 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5476
5477 case Intrinsic::s390_vperm:
5478 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL: SDLoc(Op), VT: Op.getValueType(),
5479 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5480
5481 case Intrinsic::s390_vuphb:
5482 case Intrinsic::s390_vuphh:
5483 case Intrinsic::s390_vuphf:
5484 case Intrinsic::s390_vuphg:
5485 return DAG.getNode(Opcode: SystemZISD::UNPACK_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5486 Operand: Op.getOperand(i: 1));
5487
5488 case Intrinsic::s390_vuplhb:
5489 case Intrinsic::s390_vuplhh:
5490 case Intrinsic::s390_vuplhf:
5491 case Intrinsic::s390_vuplhg:
5492 return DAG.getNode(Opcode: SystemZISD::UNPACKL_HIGH, DL: SDLoc(Op), VT: Op.getValueType(),
5493 Operand: Op.getOperand(i: 1));
5494
5495 case Intrinsic::s390_vuplb:
5496 case Intrinsic::s390_vuplhw:
5497 case Intrinsic::s390_vuplf:
5498 case Intrinsic::s390_vuplg:
5499 return DAG.getNode(Opcode: SystemZISD::UNPACK_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5500 Operand: Op.getOperand(i: 1));
5501
5502 case Intrinsic::s390_vupllb:
5503 case Intrinsic::s390_vupllh:
5504 case Intrinsic::s390_vupllf:
5505 case Intrinsic::s390_vupllg:
5506 return DAG.getNode(Opcode: SystemZISD::UNPACKL_LOW, DL: SDLoc(Op), VT: Op.getValueType(),
5507 Operand: Op.getOperand(i: 1));
5508
5509 case Intrinsic::s390_vsumb:
5510 case Intrinsic::s390_vsumh:
5511 case Intrinsic::s390_vsumgh:
5512 case Intrinsic::s390_vsumgf:
5513 case Intrinsic::s390_vsumqf:
5514 case Intrinsic::s390_vsumqg:
5515 return DAG.getNode(Opcode: SystemZISD::VSUM, DL: SDLoc(Op), VT: Op.getValueType(),
5516 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5517
5518 case Intrinsic::s390_vaq:
5519 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5520 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5521 case Intrinsic::s390_vaccb:
5522 case Intrinsic::s390_vacch:
5523 case Intrinsic::s390_vaccf:
5524 case Intrinsic::s390_vaccg:
5525 case Intrinsic::s390_vaccq:
5526 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(Op), VT: Op.getValueType(),
5527 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5528 case Intrinsic::s390_vacq:
5529 return DAG.getNode(Opcode: SystemZISD::VAC, DL: SDLoc(Op), VT: Op.getValueType(),
5530 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5531 case Intrinsic::s390_vacccq:
5532 return DAG.getNode(Opcode: SystemZISD::VACCC, DL: SDLoc(Op), VT: Op.getValueType(),
5533 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5534
5535 case Intrinsic::s390_vsq:
5536 return DAG.getNode(Opcode: ISD::SUB, DL: SDLoc(Op), VT: Op.getValueType(),
5537 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5538 case Intrinsic::s390_vscbib:
5539 case Intrinsic::s390_vscbih:
5540 case Intrinsic::s390_vscbif:
5541 case Intrinsic::s390_vscbig:
5542 case Intrinsic::s390_vscbiq:
5543 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5544 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5545 case Intrinsic::s390_vsbiq:
5546 return DAG.getNode(Opcode: SystemZISD::VSBI, DL: SDLoc(Op), VT: Op.getValueType(),
5547 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5548 case Intrinsic::s390_vsbcbiq:
5549 return DAG.getNode(Opcode: SystemZISD::VSBCBI, DL: SDLoc(Op), VT: Op.getValueType(),
5550 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5551
5552 case Intrinsic::s390_vmhb:
5553 case Intrinsic::s390_vmhh:
5554 case Intrinsic::s390_vmhf:
5555 case Intrinsic::s390_vmhg:
5556 case Intrinsic::s390_vmhq:
5557 return DAG.getNode(Opcode: ISD::MULHS, DL: SDLoc(Op), VT: Op.getValueType(),
5558 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5559 case Intrinsic::s390_vmlhb:
5560 case Intrinsic::s390_vmlhh:
5561 case Intrinsic::s390_vmlhf:
5562 case Intrinsic::s390_vmlhg:
5563 case Intrinsic::s390_vmlhq:
5564 return DAG.getNode(Opcode: ISD::MULHU, DL: SDLoc(Op), VT: Op.getValueType(),
5565 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5566
5567 case Intrinsic::s390_vmahb:
5568 case Intrinsic::s390_vmahh:
5569 case Intrinsic::s390_vmahf:
5570 case Intrinsic::s390_vmahg:
5571 case Intrinsic::s390_vmahq:
5572 return DAG.getNode(Opcode: SystemZISD::VMAH, DL: SDLoc(Op), VT: Op.getValueType(),
5573 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5574 case Intrinsic::s390_vmalhb:
5575 case Intrinsic::s390_vmalhh:
5576 case Intrinsic::s390_vmalhf:
5577 case Intrinsic::s390_vmalhg:
5578 case Intrinsic::s390_vmalhq:
5579 return DAG.getNode(Opcode: SystemZISD::VMALH, DL: SDLoc(Op), VT: Op.getValueType(),
5580 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
5581
5582 case Intrinsic::s390_vmeb:
5583 case Intrinsic::s390_vmeh:
5584 case Intrinsic::s390_vmef:
5585 case Intrinsic::s390_vmeg:
5586 return DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5587 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5588 case Intrinsic::s390_vmleb:
5589 case Intrinsic::s390_vmleh:
5590 case Intrinsic::s390_vmlef:
5591 case Intrinsic::s390_vmleg:
5592 return DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5593 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5594 case Intrinsic::s390_vmob:
5595 case Intrinsic::s390_vmoh:
5596 case Intrinsic::s390_vmof:
5597 case Intrinsic::s390_vmog:
5598 return DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5599 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5600 case Intrinsic::s390_vmlob:
5601 case Intrinsic::s390_vmloh:
5602 case Intrinsic::s390_vmlof:
5603 case Intrinsic::s390_vmlog:
5604 return DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5605 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
5606
5607 case Intrinsic::s390_vmaeb:
5608 case Intrinsic::s390_vmaeh:
5609 case Intrinsic::s390_vmaef:
5610 case Intrinsic::s390_vmaeg:
5611 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5612 N1: DAG.getNode(Opcode: SystemZISD::VME, DL: SDLoc(Op), VT: Op.getValueType(),
5613 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5614 N2: Op.getOperand(i: 3));
5615 case Intrinsic::s390_vmaleb:
5616 case Intrinsic::s390_vmaleh:
5617 case Intrinsic::s390_vmalef:
5618 case Intrinsic::s390_vmaleg:
5619 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5620 N1: DAG.getNode(Opcode: SystemZISD::VMLE, DL: SDLoc(Op), VT: Op.getValueType(),
5621 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5622 N2: Op.getOperand(i: 3));
5623 case Intrinsic::s390_vmaob:
5624 case Intrinsic::s390_vmaoh:
5625 case Intrinsic::s390_vmaof:
5626 case Intrinsic::s390_vmaog:
5627 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5628 N1: DAG.getNode(Opcode: SystemZISD::VMO, DL: SDLoc(Op), VT: Op.getValueType(),
5629 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5630 N2: Op.getOperand(i: 3));
5631 case Intrinsic::s390_vmalob:
5632 case Intrinsic::s390_vmaloh:
5633 case Intrinsic::s390_vmalof:
5634 case Intrinsic::s390_vmalog:
5635 return DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(Op), VT: Op.getValueType(),
5636 N1: DAG.getNode(Opcode: SystemZISD::VMLO, DL: SDLoc(Op), VT: Op.getValueType(),
5637 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2)),
5638 N2: Op.getOperand(i: 3));
5639 }
5640
5641 return SDValue();
5642}
5643
5644namespace {
5645// Says that SystemZISD operation Opcode can be used to perform the equivalent
5646// of a VPERM with permute vector Bytes. If Opcode takes three operands,
5647// Operand is the constant third operand, otherwise it is the number of
5648// bytes in each element of the result.
5649struct Permute {
5650 unsigned Opcode;
5651 unsigned Operand;
5652 unsigned char Bytes[SystemZ::VectorBytes];
5653};
5654}
5655
5656static const Permute PermuteForms[] = {
5657 // VMRHG
5658 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 8,
5659 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
5660 // VMRHF
5661 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 4,
5662 .Bytes: { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
5663 // VMRHH
5664 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 2,
5665 .Bytes: { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
5666 // VMRHB
5667 { .Opcode: SystemZISD::MERGE_HIGH, .Operand: 1,
5668 .Bytes: { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
5669 // VMRLG
5670 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 8,
5671 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
5672 // VMRLF
5673 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 4,
5674 .Bytes: { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
5675 // VMRLH
5676 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 2,
5677 .Bytes: { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
5678 // VMRLB
5679 { .Opcode: SystemZISD::MERGE_LOW, .Operand: 1,
5680 .Bytes: { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
5681 // VPKG
5682 { .Opcode: SystemZISD::PACK, .Operand: 4,
5683 .Bytes: { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
5684 // VPKF
5685 { .Opcode: SystemZISD::PACK, .Operand: 2,
5686 .Bytes: { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
5687 // VPKH
5688 { .Opcode: SystemZISD::PACK, .Operand: 1,
5689 .Bytes: { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
5690 // VPDI V1, V2, 4 (low half of V1, high half of V2)
5691 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 4,
5692 .Bytes: { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
5693 // VPDI V1, V2, 1 (high half of V1, low half of V2)
5694 { .Opcode: SystemZISD::PERMUTE_DWORDS, .Operand: 1,
5695 .Bytes: { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
5696};
5697
5698// Called after matching a vector shuffle against a particular pattern.
5699// Both the original shuffle and the pattern have two vector operands.
5700// OpNos[0] is the operand of the original shuffle that should be used for
5701// operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
5702// OpNos[1] is the same for operand 1 of the pattern. Resolve these -1s and
5703// set OpNo0 and OpNo1 to the shuffle operands that should actually be used
5704// for operands 0 and 1 of the pattern.
5705static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
5706 if (OpNos[0] < 0) {
5707 if (OpNos[1] < 0)
5708 return false;
5709 OpNo0 = OpNo1 = OpNos[1];
5710 } else if (OpNos[1] < 0) {
5711 OpNo0 = OpNo1 = OpNos[0];
5712 } else {
5713 OpNo0 = OpNos[0];
5714 OpNo1 = OpNos[1];
5715 }
5716 return true;
5717}
5718
5719// Bytes is a VPERM-like permute vector, except that -1 is used for
5720// undefined bytes. Return true if the VPERM can be implemented using P.
5721// When returning true set OpNo0 to the VPERM operand that should be
5722// used for operand 0 of P and likewise OpNo1 for operand 1 of P.
5723//
5724// For example, if swapping the VPERM operands allows P to match, OpNo0
5725// will be 1 and OpNo1 will be 0. If instead Bytes only refers to one
5726// operand, but rewriting it to use two duplicated operands allows it to
5727// match P, then OpNo0 and OpNo1 will be the same.
5728static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
5729 unsigned &OpNo0, unsigned &OpNo1) {
5730 int OpNos[] = { -1, -1 };
5731 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5732 int Elt = Bytes[I];
5733 if (Elt >= 0) {
5734 // Make sure that the two permute vectors use the same suboperand
5735 // byte number. Only the operand numbers (the high bits) are
5736 // allowed to differ.
5737 if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
5738 return false;
5739 int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
5740 int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
5741 // Make sure that the operand mappings are consistent with previous
5742 // elements.
5743 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5744 return false;
5745 OpNos[ModelOpNo] = RealOpNo;
5746 }
5747 }
5748 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5749}
5750
5751// As above, but search for a matching permute.
5752static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
5753 unsigned &OpNo0, unsigned &OpNo1) {
5754 for (auto &P : PermuteForms)
5755 if (matchPermute(Bytes, P, OpNo0, OpNo1))
5756 return &P;
5757 return nullptr;
5758}
5759
5760// Bytes is a VPERM-like permute vector, except that -1 is used for
5761// undefined bytes. This permute is an operand of an outer permute.
5762// See whether redistributing the -1 bytes gives a shuffle that can be
5763// implemented using P. If so, set Transform to a VPERM-like permute vector
5764// that, when applied to the result of P, gives the original permute in Bytes.
5765static bool matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5766 const Permute &P,
5767 SmallVectorImpl<int> &Transform) {
5768 unsigned To = 0;
5769 for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
5770 int Elt = Bytes[From];
5771 if (Elt < 0)
5772 // Byte number From of the result is undefined.
5773 Transform[From] = -1;
5774 else {
5775 while (P.Bytes[To] != Elt) {
5776 To += 1;
5777 if (To == SystemZ::VectorBytes)
5778 return false;
5779 }
5780 Transform[From] = To;
5781 }
5782 }
5783 return true;
5784}
5785
5786// As above, but search for a matching permute.
5787static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
5788 SmallVectorImpl<int> &Transform) {
5789 for (auto &P : PermuteForms)
5790 if (matchDoublePermute(Bytes, P, Transform))
5791 return &P;
5792 return nullptr;
5793}
5794
5795// Convert the mask of the given shuffle op into a byte-level mask,
5796// as if it had type vNi8.
5797static bool getVPermMask(SDValue ShuffleOp,
5798 SmallVectorImpl<int> &Bytes) {
5799 EVT VT = ShuffleOp.getValueType();
5800 unsigned NumElements = VT.getVectorNumElements();
5801 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
5802
5803 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: ShuffleOp)) {
5804 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5805 for (unsigned I = 0; I < NumElements; ++I) {
5806 int Index = VSN->getMaskElt(Idx: I);
5807 if (Index >= 0)
5808 for (unsigned J = 0; J < BytesPerElement; ++J)
5809 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5810 }
5811 return true;
5812 }
5813 if (SystemZISD::SPLAT == ShuffleOp.getOpcode() &&
5814 isa<ConstantSDNode>(Val: ShuffleOp.getOperand(i: 1))) {
5815 unsigned Index = ShuffleOp.getConstantOperandVal(i: 1);
5816 Bytes.resize(N: NumElements * BytesPerElement, NV: -1);
5817 for (unsigned I = 0; I < NumElements; ++I)
5818 for (unsigned J = 0; J < BytesPerElement; ++J)
5819 Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
5820 return true;
5821 }
5822 return false;
5823}
5824
5825// Bytes is a VPERM-like permute vector, except that -1 is used for
5826// undefined bytes. See whether bytes [Start, Start + BytesPerElement) of
5827// the result come from a contiguous sequence of bytes from one input.
5828// Set Base to the selector for the first byte if so.
5829static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
5830 unsigned BytesPerElement, int &Base) {
5831 Base = -1;
5832 for (unsigned I = 0; I < BytesPerElement; ++I) {
5833 if (Bytes[Start + I] >= 0) {
5834 unsigned Elem = Bytes[Start + I];
5835 if (Base < 0) {
5836 Base = Elem - I;
5837 // Make sure the bytes would come from one input operand.
5838 if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
5839 return false;
5840 } else if (unsigned(Base) != Elem - I)
5841 return false;
5842 }
5843 }
5844 return true;
5845}
5846
5847// Bytes is a VPERM-like permute vector, except that -1 is used for
5848// undefined bytes. Return true if it can be performed using VSLDB.
5849// When returning true, set StartIndex to the shift amount and OpNo0
5850// and OpNo1 to the VPERM operands that should be used as the first
5851// and second shift operand respectively.
5852static bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes,
5853 unsigned &StartIndex, unsigned &OpNo0,
5854 unsigned &OpNo1) {
5855 int OpNos[] = { -1, -1 };
5856 int Shift = -1;
5857 for (unsigned I = 0; I < 16; ++I) {
5858 int Index = Bytes[I];
5859 if (Index >= 0) {
5860 int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
5861 int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
5862 int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
5863 if (Shift < 0)
5864 Shift = ExpectedShift;
5865 else if (Shift != ExpectedShift)
5866 return false;
5867 // Make sure that the operand mappings are consistent with previous
5868 // elements.
5869 if (OpNos[ModelOpNo] == 1 - RealOpNo)
5870 return false;
5871 OpNos[ModelOpNo] = RealOpNo;
5872 }
5873 }
5874 StartIndex = Shift;
5875 return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
5876}
5877
5878// Create a node that performs P on operands Op0 and Op1, casting the
5879// operands to the appropriate type. The type of the result is determined by P.
5880static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5881 const Permute &P, SDValue Op0, SDValue Op1) {
5882 // VPDI (PERMUTE_DWORDS) always operates on v2i64s. The input
5883 // elements of a PACK are twice as wide as the outputs.
5884 unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
5885 P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
5886 P.Operand);
5887 // Cast both operands to the appropriate type.
5888 MVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBytes * 8),
5889 NumElements: SystemZ::VectorBytes / InBytes);
5890 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op0);
5891 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op1);
5892 SDValue Op;
5893 if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
5894 SDValue Op2 = DAG.getTargetConstant(Val: P.Operand, DL, VT: MVT::i32);
5895 Op = DAG.getNode(Opcode: SystemZISD::PERMUTE_DWORDS, DL, VT: InVT, N1: Op0, N2: Op1, N3: Op2);
5896 } else if (P.Opcode == SystemZISD::PACK) {
5897 MVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: P.Operand * 8),
5898 NumElements: SystemZ::VectorBytes / P.Operand);
5899 Op = DAG.getNode(Opcode: SystemZISD::PACK, DL, VT: OutVT, N1: Op0, N2: Op1);
5900 } else {
5901 Op = DAG.getNode(Opcode: P.Opcode, DL, VT: InVT, N1: Op0, N2: Op1);
5902 }
5903 return Op;
5904}
5905
5906static bool isZeroVector(SDValue N) {
5907 if (N->getOpcode() == ISD::BITCAST)
5908 N = N->getOperand(Num: 0);
5909 if (N->getOpcode() == ISD::SPLAT_VECTOR)
5910 if (auto *Op = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0)))
5911 return Op->getZExtValue() == 0;
5912 return ISD::isBuildVectorAllZeros(N: N.getNode());
5913}
5914
5915// Return the index of the zero/undef vector, or UINT32_MAX if not found.
5916static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num) {
5917 for (unsigned I = 0; I < Num ; I++)
5918 if (isZeroVector(N: Ops[I]))
5919 return I;
5920 return UINT32_MAX;
5921}
5922
5923// Bytes is a VPERM-like permute vector, except that -1 is used for
5924// undefined bytes. Implement it on operands Ops[0] and Ops[1] using
5925// VSLDB or VPERM.
5926static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
5927 SDValue *Ops,
5928 const SmallVectorImpl<int> &Bytes) {
5929 for (unsigned I = 0; I < 2; ++I)
5930 Ops[I] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: Ops[I]);
5931
5932 // First see whether VSLDB can be used.
5933 unsigned StartIndex, OpNo0, OpNo1;
5934 if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
5935 return DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE, DL, VT: MVT::v16i8, N1: Ops[OpNo0],
5936 N2: Ops[OpNo1],
5937 N3: DAG.getTargetConstant(Val: StartIndex, DL, VT: MVT::i32));
5938
5939 // Fall back on VPERM. Construct an SDNode for the permute vector. Try to
5940 // eliminate a zero vector by reusing any zero index in the permute vector.
5941 unsigned ZeroVecIdx = findZeroVectorIdx(Ops: &Ops[0], Num: 2);
5942 if (ZeroVecIdx != UINT32_MAX) {
5943 bool MaskFirst = true;
5944 int ZeroIdx = -1;
5945 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5946 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5947 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
5948 if (OpNo == ZeroVecIdx && I == 0) {
5949 // If the first byte is zero, use mask as first operand.
5950 ZeroIdx = 0;
5951 break;
5952 }
5953 if (OpNo != ZeroVecIdx && Byte == 0) {
5954 // If mask contains a zero, use it by placing that vector first.
5955 ZeroIdx = I + SystemZ::VectorBytes;
5956 MaskFirst = false;
5957 break;
5958 }
5959 }
5960 if (ZeroIdx != -1) {
5961 SDValue IndexNodes[SystemZ::VectorBytes];
5962 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
5963 if (Bytes[I] >= 0) {
5964 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
5965 unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
5966 if (OpNo == ZeroVecIdx)
5967 IndexNodes[I] = DAG.getConstant(Val: ZeroIdx, DL, VT: MVT::i32);
5968 else {
5969 unsigned BIdx = MaskFirst ? Byte + SystemZ::VectorBytes : Byte;
5970 IndexNodes[I] = DAG.getConstant(Val: BIdx, DL, VT: MVT::i32);
5971 }
5972 } else
5973 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
5974 }
5975 SDValue Mask = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
5976 SDValue Src = ZeroVecIdx == 0 ? Ops[1] : Ops[0];
5977 if (MaskFirst)
5978 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Mask, N2: Src,
5979 N3: Mask);
5980 else
5981 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Src, N2: Mask,
5982 N3: Mask);
5983 }
5984 }
5985
5986 SDValue IndexNodes[SystemZ::VectorBytes];
5987 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
5988 if (Bytes[I] >= 0)
5989 IndexNodes[I] = DAG.getConstant(Val: Bytes[I], DL, VT: MVT::i32);
5990 else
5991 IndexNodes[I] = DAG.getUNDEF(VT: MVT::i32);
5992 SDValue Op2 = DAG.getBuildVector(VT: MVT::v16i8, DL, Ops: IndexNodes);
5993 return DAG.getNode(Opcode: SystemZISD::PERMUTE, DL, VT: MVT::v16i8, N1: Ops[0],
5994 N2: (!Ops[1].isUndef() ? Ops[1] : Ops[0]), N3: Op2);
5995}
5996
5997namespace {
5998// Describes a general N-operand vector shuffle.
5999struct GeneralShuffle {
6000 GeneralShuffle(EVT vt)
6001 : VT(vt), UnpackFromEltSize(UINT_MAX), UnpackLow(false) {}
6002 void addUndef();
6003 bool add(SDValue, unsigned);
6004 SDValue getNode(SelectionDAG &, const SDLoc &);
6005 void tryPrepareForUnpack();
6006 bool unpackWasPrepared() { return UnpackFromEltSize <= 4; }
6007 SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
6008
6009 // The operands of the shuffle.
6010 SmallVector<SDValue, SystemZ::VectorBytes> Ops;
6011
6012 // Index I is -1 if byte I of the result is undefined. Otherwise the
6013 // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
6014 // Bytes[I] / SystemZ::VectorBytes.
6015 SmallVector<int, SystemZ::VectorBytes> Bytes;
6016
6017 // The type of the shuffle result.
6018 EVT VT;
6019
6020 // Holds a value of 1, 2 or 4 if a final unpack has been prepared for.
6021 unsigned UnpackFromEltSize;
6022 // True if the final unpack uses the low half.
6023 bool UnpackLow;
6024};
6025} // namespace
6026
6027// Add an extra undefined element to the shuffle.
6028void GeneralShuffle::addUndef() {
6029 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
6030 for (unsigned I = 0; I < BytesPerElement; ++I)
6031 Bytes.push_back(Elt: -1);
6032}
6033
6034// Add an extra element to the shuffle, taking it from element Elem of Op.
6035// A null Op indicates a vector input whose value will be calculated later;
6036// there is at most one such input per shuffle and it always has the same
6037// type as the result. Aborts and returns false if the source vector elements
6038// of an EXTRACT_VECTOR_ELT are smaller than the destination elements. Per
6039// LLVM they become implicitly extended, but this is rare and not optimized.
6040bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
6041 unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
6042
6043 // The source vector can have wider elements than the result,
6044 // either through an explicit TRUNCATE or because of type legalization.
6045 // We want the least significant part.
6046 EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
6047 unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
6048
6049 // Return false if the source elements are smaller than their destination
6050 // elements.
6051 if (FromBytesPerElement < BytesPerElement)
6052 return false;
6053
6054 unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
6055 (FromBytesPerElement - BytesPerElement));
6056
6057 // Look through things like shuffles and bitcasts.
6058 while (Op.getNode()) {
6059 if (Op.getOpcode() == ISD::BITCAST)
6060 Op = Op.getOperand(i: 0);
6061 else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
6062 // See whether the bytes we need come from a contiguous part of one
6063 // operand.
6064 SmallVector<int, SystemZ::VectorBytes> OpBytes;
6065 if (!getVPermMask(ShuffleOp: Op, Bytes&: OpBytes))
6066 break;
6067 int NewByte;
6068 if (!getShuffleInput(Bytes: OpBytes, Start: Byte, BytesPerElement, Base&: NewByte))
6069 break;
6070 if (NewByte < 0) {
6071 addUndef();
6072 return true;
6073 }
6074 Op = Op.getOperand(i: unsigned(NewByte) / SystemZ::VectorBytes);
6075 Byte = unsigned(NewByte) % SystemZ::VectorBytes;
6076 } else if (Op.isUndef()) {
6077 addUndef();
6078 return true;
6079 } else
6080 break;
6081 }
6082
6083 // Make sure that the source of the extraction is in Ops.
6084 unsigned OpNo = 0;
6085 for (; OpNo < Ops.size(); ++OpNo)
6086 if (Ops[OpNo] == Op)
6087 break;
6088 if (OpNo == Ops.size())
6089 Ops.push_back(Elt: Op);
6090
6091 // Add the element to Bytes.
6092 unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
6093 for (unsigned I = 0; I < BytesPerElement; ++I)
6094 Bytes.push_back(Elt: Base + I);
6095
6096 return true;
6097}
6098
6099// Return SDNodes for the completed shuffle.
6100SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
6101 assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
6102
6103 if (Ops.size() == 0)
6104 return DAG.getUNDEF(VT);
6105
6106 // Use a single unpack if possible as the last operation.
6107 tryPrepareForUnpack();
6108
6109 // Make sure that there are at least two shuffle operands.
6110 if (Ops.size() == 1)
6111 Ops.push_back(Elt: DAG.getUNDEF(VT: MVT::v16i8));
6112
6113 // Create a tree of shuffles, deferring root node until after the loop.
6114 // Try to redistribute the undefined elements of non-root nodes so that
6115 // the non-root shuffles match something like a pack or merge, then adjust
6116 // the parent node's permute vector to compensate for the new order.
6117 // Among other things, this copes with vectors like <2 x i16> that were
6118 // padded with undefined elements during type legalization.
6119 //
6120 // In the best case this redistribution will lead to the whole tree
6121 // using packs and merges. It should rarely be a loss in other cases.
6122 unsigned Stride = 1;
6123 for (; Stride * 2 < Ops.size(); Stride *= 2) {
6124 for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
6125 SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
6126
6127 // Create a mask for just these two operands.
6128 SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes);
6129 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6130 unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
6131 unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
6132 if (OpNo == I)
6133 NewBytes[J] = Byte;
6134 else if (OpNo == I + Stride)
6135 NewBytes[J] = SystemZ::VectorBytes + Byte;
6136 else
6137 NewBytes[J] = -1;
6138 }
6139 // See if it would be better to reorganize NewMask to avoid using VPERM.
6140 SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes);
6141 if (const Permute *P = matchDoublePermute(Bytes: NewBytes, Transform&: NewBytesMap)) {
6142 Ops[I] = getPermuteNode(DAG, DL, P: *P, Op0: SubOps[0], Op1: SubOps[1]);
6143 // Applying NewBytesMap to Ops[I] gets back to NewBytes.
6144 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
6145 if (NewBytes[J] >= 0) {
6146 assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
6147 "Invalid double permute");
6148 Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
6149 } else
6150 assert(NewBytesMap[J] < 0 && "Invalid double permute");
6151 }
6152 } else {
6153 // Just use NewBytes on the operands.
6154 Ops[I] = getGeneralPermuteNode(DAG, DL, Ops: SubOps, Bytes: NewBytes);
6155 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
6156 if (NewBytes[J] >= 0)
6157 Bytes[J] = I * SystemZ::VectorBytes + J;
6158 }
6159 }
6160 }
6161
6162 // Now we just have 2 inputs. Put the second operand in Ops[1].
6163 if (Stride > 1) {
6164 Ops[1] = Ops[Stride];
6165 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6166 if (Bytes[I] >= int(SystemZ::VectorBytes))
6167 Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
6168 }
6169
6170 // Look for an instruction that can do the permute without resorting
6171 // to VPERM.
6172 unsigned OpNo0, OpNo1;
6173 SDValue Op;
6174 if (unpackWasPrepared() && Ops[1].isUndef())
6175 Op = Ops[0];
6176 else if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
6177 Op = getPermuteNode(DAG, DL, P: *P, Op0: Ops[OpNo0], Op1: Ops[OpNo1]);
6178 else
6179 Op = getGeneralPermuteNode(DAG, DL, Ops: &Ops[0], Bytes);
6180
6181 Op = insertUnpackIfPrepared(DAG, DL, Op);
6182
6183 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6184}
6185
6186#ifndef NDEBUG
6187static void dumpBytes(const SmallVectorImpl<int> &Bytes, std::string Msg) {
6188 dbgs() << Msg.c_str() << " { ";
6189 for (unsigned I = 0; I < Bytes.size(); I++)
6190 dbgs() << Bytes[I] << " ";
6191 dbgs() << "}\n";
6192}
6193#endif
6194
6195// If the Bytes vector matches an unpack operation, prepare to do the unpack
6196// after all else by removing the zero vector and the effect of the unpack on
6197// Bytes.
6198void GeneralShuffle::tryPrepareForUnpack() {
6199 uint32_t ZeroVecOpNo = findZeroVectorIdx(Ops: &Ops[0], Num: Ops.size());
6200 if (ZeroVecOpNo == UINT32_MAX || Ops.size() == 1)
6201 return;
6202
6203 // Only do this if removing the zero vector reduces the depth, otherwise
6204 // the critical path will increase with the final unpack.
6205 if (Ops.size() > 2 &&
6206 Log2_32_Ceil(Value: Ops.size()) == Log2_32_Ceil(Value: Ops.size() - 1))
6207 return;
6208
6209 // Find an unpack that would allow removing the zero vector from Ops.
6210 UnpackFromEltSize = 1;
6211 for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
6212 bool MatchUnpack = true;
6213 SmallVector<int, SystemZ::VectorBytes> SrcBytes;
6214 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes; Elt++) {
6215 unsigned ToEltSize = UnpackFromEltSize * 2;
6216 bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
6217 if (!IsZextByte)
6218 SrcBytes.push_back(Elt: Bytes[Elt]);
6219 if (Bytes[Elt] != -1) {
6220 unsigned OpNo = unsigned(Bytes[Elt]) / SystemZ::VectorBytes;
6221 if (IsZextByte != (OpNo == ZeroVecOpNo)) {
6222 MatchUnpack = false;
6223 break;
6224 }
6225 }
6226 }
6227 if (MatchUnpack) {
6228 if (Ops.size() == 2) {
6229 // Don't use unpack if a single source operand needs rearrangement.
6230 bool CanUseUnpackLow = true, CanUseUnpackHigh = true;
6231 for (unsigned i = 0; i < SystemZ::VectorBytes / 2; i++) {
6232 if (SrcBytes[i] == -1)
6233 continue;
6234 if (SrcBytes[i] % 16 != int(i))
6235 CanUseUnpackHigh = false;
6236 if (SrcBytes[i] % 16 != int(i + SystemZ::VectorBytes / 2))
6237 CanUseUnpackLow = false;
6238 if (!CanUseUnpackLow && !CanUseUnpackHigh) {
6239 UnpackFromEltSize = UINT_MAX;
6240 return;
6241 }
6242 }
6243 if (!CanUseUnpackHigh)
6244 UnpackLow = true;
6245 }
6246 break;
6247 }
6248 }
6249 if (UnpackFromEltSize > 4)
6250 return;
6251
6252 LLVM_DEBUG(dbgs() << "Preparing for final unpack of element size "
6253 << UnpackFromEltSize << ". Zero vector is Op#" << ZeroVecOpNo
6254 << ".\n";
6255 dumpBytes(Bytes, "Original Bytes vector:"););
6256
6257 // Apply the unpack in reverse to the Bytes array.
6258 unsigned B = 0;
6259 if (UnpackLow) {
6260 while (B < SystemZ::VectorBytes / 2)
6261 Bytes[B++] = -1;
6262 }
6263 for (unsigned Elt = 0; Elt < SystemZ::VectorBytes;) {
6264 Elt += UnpackFromEltSize;
6265 for (unsigned i = 0; i < UnpackFromEltSize; i++, Elt++, B++)
6266 Bytes[B] = Bytes[Elt];
6267 }
6268 if (!UnpackLow) {
6269 while (B < SystemZ::VectorBytes)
6270 Bytes[B++] = -1;
6271 }
6272
6273 // Remove the zero vector from Ops
6274 Ops.erase(CI: &Ops[ZeroVecOpNo]);
6275 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
6276 if (Bytes[I] >= 0) {
6277 unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
6278 if (OpNo > ZeroVecOpNo)
6279 Bytes[I] -= SystemZ::VectorBytes;
6280 }
6281
6282 LLVM_DEBUG(dumpBytes(Bytes, "Resulting Bytes vector, zero vector removed:");
6283 dbgs() << "\n";);
6284}
6285
6286SDValue GeneralShuffle::insertUnpackIfPrepared(SelectionDAG &DAG,
6287 const SDLoc &DL,
6288 SDValue Op) {
6289 if (!unpackWasPrepared())
6290 return Op;
6291 unsigned InBits = UnpackFromEltSize * 8;
6292 EVT InVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: InBits),
6293 NumElements: SystemZ::VectorBits / InBits);
6294 SDValue PackedOp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: InVT, Operand: Op);
6295 unsigned OutBits = InBits * 2;
6296 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: OutBits),
6297 NumElements: SystemZ::VectorBits / OutBits);
6298 return DAG.getNode(Opcode: UnpackLow ? SystemZISD::UNPACKL_LOW
6299 : SystemZISD::UNPACKL_HIGH,
6300 DL, VT: OutVT, Operand: PackedOp);
6301}
6302
6303// Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
6304static bool isScalarToVector(SDValue Op) {
6305 for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
6306 if (!Op.getOperand(i: I).isUndef())
6307 return false;
6308 return true;
6309}
6310
6311// Return a vector of type VT that contains Value in the first element.
6312// The other elements don't matter.
6313static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6314 SDValue Value) {
6315 // If we have a constant, replicate it to all elements and let the
6316 // BUILD_VECTOR lowering take care of it.
6317 if (Value.getOpcode() == ISD::Constant ||
6318 Value.getOpcode() == ISD::ConstantFP) {
6319 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value);
6320 return DAG.getBuildVector(VT, DL, Ops);
6321 }
6322 if (Value.isUndef())
6323 return DAG.getUNDEF(VT);
6324 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: Value);
6325}
6326
6327// Return a vector of type VT in which Op0 is in element 0 and Op1 is in
6328// element 1. Used for cases in which replication is cheap.
6329static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6330 SDValue Op0, SDValue Op1) {
6331 if (Op0.isUndef()) {
6332 if (Op1.isUndef())
6333 return DAG.getUNDEF(VT);
6334 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op1);
6335 }
6336 if (Op1.isUndef())
6337 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0);
6338 return DAG.getNode(Opcode: SystemZISD::MERGE_HIGH, DL, VT,
6339 N1: buildScalarToVector(DAG, DL, VT, Value: Op0),
6340 N2: buildScalarToVector(DAG, DL, VT, Value: Op1));
6341}
6342
6343// Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
6344// vector for them.
6345static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
6346 SDValue Op1) {
6347 if (Op0.isUndef() && Op1.isUndef())
6348 return DAG.getUNDEF(VT: MVT::v2i64);
6349 // If one of the two inputs is undefined then replicate the other one,
6350 // in order to avoid using another register unnecessarily.
6351 if (Op0.isUndef())
6352 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6353 else if (Op1.isUndef())
6354 Op0 = Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6355 else {
6356 Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
6357 Op1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op1);
6358 }
6359 return DAG.getNode(Opcode: SystemZISD::JOIN_DWORDS, DL, VT: MVT::v2i64, N1: Op0, N2: Op1);
6360}
6361
6362// If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
6363// better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
6364// the non-EXTRACT_VECTOR_ELT elements. See if the given BUILD_VECTOR
6365// would benefit from this representation and return it if so.
6366static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
6367 BuildVectorSDNode *BVN) {
6368 EVT VT = BVN->getValueType(ResNo: 0);
6369 unsigned NumElements = VT.getVectorNumElements();
6370
6371 // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
6372 // on byte vectors. If there are non-EXTRACT_VECTOR_ELT elements that still
6373 // need a BUILD_VECTOR, add an additional placeholder operand for that
6374 // BUILD_VECTOR and store its operands in ResidueOps.
6375 GeneralShuffle GS(VT);
6376 SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
6377 bool FoundOne = false;
6378 for (unsigned I = 0; I < NumElements; ++I) {
6379 SDValue Op = BVN->getOperand(Num: I);
6380 if (Op.getOpcode() == ISD::TRUNCATE)
6381 Op = Op.getOperand(i: 0);
6382 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
6383 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
6384 unsigned Elem = Op.getConstantOperandVal(i: 1);
6385 if (!GS.add(Op: Op.getOperand(i: 0), Elem))
6386 return SDValue();
6387 FoundOne = true;
6388 } else if (Op.isUndef()) {
6389 GS.addUndef();
6390 } else {
6391 if (!GS.add(Op: SDValue(), Elem: ResidueOps.size()))
6392 return SDValue();
6393 ResidueOps.push_back(Elt: BVN->getOperand(Num: I));
6394 }
6395 }
6396
6397 // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
6398 if (!FoundOne)
6399 return SDValue();
6400
6401 // Create the BUILD_VECTOR for the remaining elements, if any.
6402 if (!ResidueOps.empty()) {
6403 while (ResidueOps.size() < NumElements)
6404 ResidueOps.push_back(Elt: DAG.getUNDEF(VT: ResidueOps[0].getValueType()));
6405 for (auto &Op : GS.Ops) {
6406 if (!Op.getNode()) {
6407 Op = DAG.getBuildVector(VT, DL: SDLoc(BVN), Ops: ResidueOps);
6408 break;
6409 }
6410 }
6411 }
6412 return GS.getNode(DAG, DL: SDLoc(BVN));
6413}
6414
6415bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
6416 if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Val&: Op)->isUnindexed())
6417 return true;
6418 if (auto *AL = dyn_cast<AtomicSDNode>(Val&: Op))
6419 if (AL->getOpcode() == ISD::ATOMIC_LOAD)
6420 return true;
6421 if (Subtarget.hasVectorEnhancements2() && Op.getOpcode() == SystemZISD::LRV)
6422 return true;
6423 return false;
6424}
6425
6426static SDValue mergeHighParts(SelectionDAG &DAG, const SDLoc &DL,
6427 unsigned MergedBits, EVT VT, SDValue Op0,
6428 SDValue Op1) {
6429 MVT IntVecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: MergedBits),
6430 NumElements: SystemZ::VectorBits / MergedBits);
6431 assert(VT.getSizeInBits() == 128 && IntVecVT.getSizeInBits() == 128 &&
6432 "Handling full vectors only.");
6433 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0);
6434 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op1);
6435 SDValue Op = DAG.getNode(Opcode: SystemZISD::MERGE_HIGH, DL, VT: IntVecVT, N1: Op0, N2: Op1);
6436 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
6437}
6438
6439static SDValue buildFPVecFromScalars4(SelectionDAG &DAG, const SDLoc &DL,
6440 EVT VT, SmallVectorImpl<SDValue> &Elems,
6441 unsigned Pos) {
6442 SDValue Op01 = buildMergeScalars(DAG, DL, VT, Op0: Elems[Pos + 0], Op1: Elems[Pos + 1]);
6443 SDValue Op23 = buildMergeScalars(DAG, DL, VT, Op0: Elems[Pos + 2], Op1: Elems[Pos + 3]);
6444 // Avoid unnecessary undefs by reusing the other operand.
6445 if (Op01.isUndef()) {
6446 if (Op23.isUndef())
6447 return Op01;
6448 Op01 = Op23;
6449 } else if (Op23.isUndef())
6450 Op23 = Op01;
6451 // Merging identical replications is a no-op.
6452 if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
6453 return Op01;
6454 unsigned MergedBits = VT.getSimpleVT().getScalarSizeInBits() * 2;
6455 return mergeHighParts(DAG, DL, MergedBits, VT, Op0: Op01, Op1: Op23);
6456}
6457
6458// Combine GPR scalar values Elems into a vector of type VT.
6459SDValue
6460SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
6461 SmallVectorImpl<SDValue> &Elems) const {
6462 // See whether there is a single replicated value.
6463 SDValue Single;
6464 unsigned int NumElements = Elems.size();
6465 unsigned int Count = 0;
6466 for (auto Elem : Elems) {
6467 if (!Elem.isUndef()) {
6468 if (!Single.getNode())
6469 Single = Elem;
6470 else if (Elem != Single) {
6471 Single = SDValue();
6472 break;
6473 }
6474 Count += 1;
6475 }
6476 }
6477 // There are three cases here:
6478 //
6479 // - if the only defined element is a loaded one, the best sequence
6480 // is a replicating load.
6481 //
6482 // - otherwise, if the only defined element is an i64 value, we will
6483 // end up with the same VLVGP sequence regardless of whether we short-cut
6484 // for replication or fall through to the later code.
6485 //
6486 // - otherwise, if the only defined element is an i32 or smaller value,
6487 // we would need 2 instructions to replicate it: VLVGP followed by VREPx.
6488 // This is only a win if the single defined element is used more than once.
6489 // In other cases we're better off using a single VLVGx.
6490 if (Single.getNode() && (Count > 1 || isVectorElementLoad(Op: Single)))
6491 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Single);
6492
6493 // If all elements are loads, use VLREP/VLEs (below).
6494 bool AllLoads = true;
6495 for (auto Elem : Elems)
6496 if (!isVectorElementLoad(Op: Elem)) {
6497 AllLoads = false;
6498 break;
6499 }
6500
6501 // The best way of building a v2i64 from two i64s is to use VLVGP.
6502 if (VT == MVT::v2i64 && !AllLoads)
6503 return joinDwords(DAG, DL, Op0: Elems[0], Op1: Elems[1]);
6504
6505 // Use a 64-bit merge high to combine two doubles.
6506 if (VT == MVT::v2f64 && !AllLoads)
6507 return buildMergeScalars(DAG, DL, VT, Op0: Elems[0], Op1: Elems[1]);
6508
6509 // Build v4f32 values directly from the FPRs:
6510 //
6511 // <Axxx> <Bxxx> <Cxxxx> <Dxxx>
6512 // V V VMRHF
6513 // <ABxx> <CDxx>
6514 // V VMRHG
6515 // <ABCD>
6516 if (VT == MVT::v4f32 && !AllLoads)
6517 return buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 0);
6518
6519 // Same for v8f16.
6520 if (VT == MVT::v8f16 && !AllLoads) {
6521 SDValue Op0123 = buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 0);
6522 SDValue Op4567 = buildFPVecFromScalars4(DAG, DL, VT, Elems, Pos: 4);
6523 // Avoid unnecessary undefs by reusing the other operand.
6524 if (Op0123.isUndef())
6525 Op0123 = Op4567;
6526 else if (Op4567.isUndef())
6527 Op4567 = Op0123;
6528 // Merging identical replications is a no-op.
6529 if (Op0123.getOpcode() == SystemZISD::REPLICATE && Op0123 == Op4567)
6530 return Op0123;
6531 return mergeHighParts(DAG, DL, MergedBits: 64, VT, Op0: Op0123, Op1: Op4567);
6532 }
6533
6534 // Collect the constant terms.
6535 SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
6536 SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false);
6537
6538 unsigned NumConstants = 0;
6539 for (unsigned I = 0; I < NumElements; ++I) {
6540 SDValue Elem = Elems[I];
6541 if (Elem.getOpcode() == ISD::Constant ||
6542 Elem.getOpcode() == ISD::ConstantFP) {
6543 NumConstants += 1;
6544 Constants[I] = Elem;
6545 Done[I] = true;
6546 }
6547 }
6548 // If there was at least one constant, fill in the other elements of
6549 // Constants with undefs to get a full vector constant and use that
6550 // as the starting point.
6551 SDValue Result;
6552 SDValue ReplicatedVal;
6553 if (NumConstants > 0) {
6554 for (unsigned I = 0; I < NumElements; ++I)
6555 if (!Constants[I].getNode())
6556 Constants[I] = DAG.getUNDEF(VT: Elems[I].getValueType());
6557 Result = DAG.getBuildVector(VT, DL, Ops: Constants);
6558 } else {
6559 // Otherwise try to use VLREP or VLVGP to start the sequence in order to
6560 // avoid a false dependency on any previous contents of the vector
6561 // register.
6562
6563 // Use a VLREP if at least one element is a load. Make sure to replicate
6564 // the load with the most elements having its value.
6565 std::map<const SDNode*, unsigned> UseCounts;
6566 SDNode *LoadMaxUses = nullptr;
6567 for (unsigned I = 0; I < NumElements; ++I)
6568 if (isVectorElementLoad(Op: Elems[I])) {
6569 SDNode *Ld = Elems[I].getNode();
6570 unsigned Count = ++UseCounts[Ld];
6571 if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < Count)
6572 LoadMaxUses = Ld;
6573 }
6574 if (LoadMaxUses != nullptr) {
6575 ReplicatedVal = SDValue(LoadMaxUses, 0);
6576 Result = DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: ReplicatedVal);
6577 } else {
6578 // Try to use VLVGP.
6579 unsigned I1 = NumElements / 2 - 1;
6580 unsigned I2 = NumElements - 1;
6581 bool Def1 = !Elems[I1].isUndef();
6582 bool Def2 = !Elems[I2].isUndef();
6583 if (Def1 || Def2) {
6584 SDValue Elem1 = Elems[Def1 ? I1 : I2];
6585 SDValue Elem2 = Elems[Def2 ? I2 : I1];
6586 Result = DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
6587 Operand: joinDwords(DAG, DL, Op0: Elem1, Op1: Elem2));
6588 Done[I1] = true;
6589 Done[I2] = true;
6590 } else
6591 Result = DAG.getUNDEF(VT);
6592 }
6593 }
6594
6595 // Use VLVGx to insert the other elements.
6596 for (unsigned I = 0; I < NumElements; ++I)
6597 if (!Done[I] && !Elems[I].isUndef() && Elems[I] != ReplicatedVal)
6598 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT, N1: Result, N2: Elems[I],
6599 N3: DAG.getConstant(Val: I, DL, VT: MVT::i32));
6600 return Result;
6601}
6602
6603SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
6604 SelectionDAG &DAG) const {
6605 auto *BVN = cast<BuildVectorSDNode>(Val: Op.getNode());
6606 SDLoc DL(Op);
6607 EVT VT = Op.getValueType();
6608
6609 if (BVN->isConstant()) {
6610 if (SystemZVectorConstantInfo(BVN).isVectorConstantLegal(Subtarget))
6611 return Op;
6612
6613 // Fall back to loading it from memory.
6614 return SDValue();
6615 }
6616
6617 // See if we should use shuffles to construct the vector from other vectors.
6618 if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
6619 return Res;
6620
6621 // Detect SCALAR_TO_VECTOR conversions.
6622 if (isOperationLegal(Op: ISD::SCALAR_TO_VECTOR, VT) && isScalarToVector(Op))
6623 return buildScalarToVector(DAG, DL, VT, Value: Op.getOperand(i: 0));
6624
6625 // Otherwise use buildVector to build the vector up from GPRs.
6626 unsigned NumElements = Op.getNumOperands();
6627 SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
6628 for (unsigned I = 0; I < NumElements; ++I)
6629 Ops[I] = Op.getOperand(i: I);
6630 return buildVector(DAG, DL, VT, Elems&: Ops);
6631}
6632
6633SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6634 SelectionDAG &DAG) const {
6635 auto *VSN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
6636 SDLoc DL(Op);
6637 EVT VT = Op.getValueType();
6638 unsigned NumElements = VT.getVectorNumElements();
6639
6640 if (VSN->isSplat()) {
6641 SDValue Op0 = Op.getOperand(i: 0);
6642 unsigned Index = VSN->getSplatIndex();
6643 assert(Index < VT.getVectorNumElements() &&
6644 "Splat index should be defined and in first operand");
6645 // See whether the value we're splatting is directly available as a scalar.
6646 if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6647 Op0.getOpcode() == ISD::BUILD_VECTOR)
6648 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL, VT, Operand: Op0.getOperand(i: Index));
6649 // Otherwise keep it as a vector-to-vector operation.
6650 return DAG.getNode(Opcode: SystemZISD::SPLAT, DL, VT, N1: Op.getOperand(i: 0),
6651 N2: DAG.getTargetConstant(Val: Index, DL, VT: MVT::i32));
6652 }
6653
6654 GeneralShuffle GS(VT);
6655 for (unsigned I = 0; I < NumElements; ++I) {
6656 int Elt = VSN->getMaskElt(Idx: I);
6657 if (Elt < 0)
6658 GS.addUndef();
6659 else if (!GS.add(Op: Op.getOperand(i: unsigned(Elt) / NumElements),
6660 Elem: unsigned(Elt) % NumElements))
6661 return SDValue();
6662 }
6663 return GS.getNode(DAG, DL: SDLoc(VSN));
6664}
6665
6666SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
6667 SelectionDAG &DAG) const {
6668 SDLoc DL(Op);
6669 // Just insert the scalar into element 0 of an undefined vector.
6670 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL,
6671 VT: Op.getValueType(), N1: DAG.getUNDEF(VT: Op.getValueType()),
6672 N2: Op.getOperand(i: 0), N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
6673}
6674
6675// Shift the lower 2 bytes of Op to the left in order to insert into the
6676// upper 2 bytes of the FP register.
6677static SDValue convertToF16(SDValue Op, SelectionDAG &DAG) {
6678 assert(Op.getSimpleValueType() == MVT::i64 &&
6679 "Expexted to convert i64 to f16.");
6680 SDLoc DL(Op);
6681 SDValue Shft = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: Op,
6682 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i64));
6683 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Shft);
6684 SDValue F16Val =
6685 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f16, Operand: BCast);
6686 return F16Val;
6687}
6688
6689// Extract Op into GPR and shift the 2 f16 bytes to the right.
6690static SDValue convertFromF16(SDValue Op, SDLoc DL, SelectionDAG &DAG) {
6691 assert(Op.getSimpleValueType() == MVT::f16 &&
6692 "Expected to convert f16 to i64.");
6693 SDNode *U32 = DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::f64);
6694 SDValue In64 = DAG.getTargetInsertSubreg(SRIdx: SystemZ::subreg_h16, DL, VT: MVT::f64,
6695 Operand: SDValue(U32, 0), Subreg: Op);
6696 SDValue BCast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: In64);
6697 SDValue Shft = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: BCast,
6698 N2: DAG.getConstant(Val: 48, DL, VT: MVT::i32));
6699 return Shft;
6700}
6701
6702SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
6703 SelectionDAG &DAG) const {
6704 // Handle insertions of floating-point values.
6705 SDLoc DL(Op);
6706 SDValue Op0 = Op.getOperand(i: 0);
6707 SDValue Op1 = Op.getOperand(i: 1);
6708 SDValue Op2 = Op.getOperand(i: 2);
6709 EVT VT = Op.getValueType();
6710
6711 // Insertions into constant indices of a v2f64 can be done using VPDI.
6712 // However, if the inserted value is a bitcast or a constant then it's
6713 // better to use GPRs, as below.
6714 if (VT == MVT::v2f64 &&
6715 Op1.getOpcode() != ISD::BITCAST &&
6716 Op1.getOpcode() != ISD::ConstantFP &&
6717 Op2.getOpcode() == ISD::Constant) {
6718 uint64_t Index = Op2->getAsZExtVal();
6719 unsigned Mask = VT.getVectorNumElements() - 1;
6720 if (Index <= Mask)
6721 return Op;
6722 }
6723
6724 // Otherwise bitcast to the equivalent integer form and insert via a GPR.
6725 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits());
6726 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VT.getVectorNumElements());
6727 SDValue IntOp1 =
6728 VT == MVT::v8f16
6729 ? DAG.getZExtOrTrunc(Op: convertFromF16(Op: Op1, DL, DAG), DL, VT: MVT::i32)
6730 : DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: Op1);
6731 SDValue Res =
6732 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: IntVecVT,
6733 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0), N2: IntOp1, N3: Op2);
6734 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
6735}
6736
6737SDValue
6738SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
6739 SelectionDAG &DAG) const {
6740 // Handle extractions of floating-point values.
6741 SDLoc DL(Op);
6742 SDValue Op0 = Op.getOperand(i: 0);
6743 SDValue Op1 = Op.getOperand(i: 1);
6744 EVT VT = Op.getValueType();
6745 EVT VecVT = Op0.getValueType();
6746
6747 // Extractions of constant indices can be done directly.
6748 if (auto *CIndexN = dyn_cast<ConstantSDNode>(Val&: Op1)) {
6749 uint64_t Index = CIndexN->getZExtValue();
6750 unsigned Mask = VecVT.getVectorNumElements() - 1;
6751 if (Index <= Mask)
6752 return Op;
6753 }
6754
6755 // Otherwise bitcast to the equivalent integer form and extract via a GPR.
6756 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getSizeInBits());
6757 MVT IntVecVT = MVT::getVectorVT(VT: IntVT, NumElements: VecVT.getVectorNumElements());
6758 MVT ExtrVT = IntVT == MVT::i16 ? MVT::i32 : IntVT;
6759 SDValue Extr = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ExtrVT,
6760 N1: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVecVT, Operand: Op0), N2: Op1);
6761 if (VT == MVT::f16)
6762 return convertToF16(Op: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Extr), DAG);
6763 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Extr);
6764}
6765
6766SDValue SystemZTargetLowering::
6767lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6768 SDValue PackedOp = Op.getOperand(i: 0);
6769 EVT OutVT = Op.getValueType();
6770 EVT InVT = PackedOp.getValueType();
6771 unsigned ToBits = OutVT.getScalarSizeInBits();
6772 unsigned FromBits = InVT.getScalarSizeInBits();
6773 unsigned StartOffset = 0;
6774
6775 // If the input is a VECTOR_SHUFFLE, there are a number of important
6776 // cases where we can directly implement the sign-extension of the
6777 // original input lanes of the shuffle.
6778 if (PackedOp.getOpcode() == ISD::VECTOR_SHUFFLE) {
6779 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: PackedOp.getNode());
6780 ArrayRef<int> ShuffleMask = SVN->getMask();
6781 int OutNumElts = OutVT.getVectorNumElements();
6782
6783 // Recognize the special case where the sign-extension can be done
6784 // by the VSEG instruction. Handled via the default expander.
6785 if (ToBits == 64 && OutNumElts == 2) {
6786 int NumElem = ToBits / FromBits;
6787 if (ShuffleMask[0] == NumElem - 1 && ShuffleMask[1] == 2 * NumElem - 1)
6788 return SDValue();
6789 }
6790
6791 // Recognize the special case where we can fold the shuffle by
6792 // replacing some of the UNPACK_HIGH with UNPACK_LOW.
6793 int StartOffsetCandidate = -1;
6794 for (int Elt = 0; Elt < OutNumElts; Elt++) {
6795 if (ShuffleMask[Elt] == -1)
6796 continue;
6797 if (ShuffleMask[Elt] % OutNumElts == Elt) {
6798 if (StartOffsetCandidate == -1)
6799 StartOffsetCandidate = ShuffleMask[Elt] - Elt;
6800 if (StartOffsetCandidate == ShuffleMask[Elt] - Elt)
6801 continue;
6802 }
6803 StartOffsetCandidate = -1;
6804 break;
6805 }
6806 if (StartOffsetCandidate != -1) {
6807 StartOffset = StartOffsetCandidate;
6808 PackedOp = PackedOp.getOperand(i: 0);
6809 }
6810 }
6811
6812 do {
6813 FromBits *= 2;
6814 unsigned OutNumElts = SystemZ::VectorBits / FromBits;
6815 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: FromBits), NumElements: OutNumElts);
6816 unsigned Opcode = SystemZISD::UNPACK_HIGH;
6817 if (StartOffset >= OutNumElts) {
6818 Opcode = SystemZISD::UNPACK_LOW;
6819 StartOffset -= OutNumElts;
6820 }
6821 PackedOp = DAG.getNode(Opcode, DL: SDLoc(PackedOp), VT: OutVT, Operand: PackedOp);
6822 } while (FromBits != ToBits);
6823 return PackedOp;
6824}
6825
6826// Lower a ZERO_EXTEND_VECTOR_INREG to a vector shuffle with a zero vector.
6827SDValue SystemZTargetLowering::
6828lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
6829 SDValue PackedOp = Op.getOperand(i: 0);
6830 SDLoc DL(Op);
6831 EVT OutVT = Op.getValueType();
6832 EVT InVT = PackedOp.getValueType();
6833 unsigned InNumElts = InVT.getVectorNumElements();
6834 unsigned OutNumElts = OutVT.getVectorNumElements();
6835 unsigned NumInPerOut = InNumElts / OutNumElts;
6836
6837 SDValue ZeroVec =
6838 DAG.getSplatVector(VT: InVT, DL, Op: DAG.getConstant(Val: 0, DL, VT: InVT.getScalarType()));
6839
6840 SmallVector<int, 16> Mask(InNumElts);
6841 unsigned ZeroVecElt = InNumElts;
6842 for (unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
6843 unsigned MaskElt = PackedElt * NumInPerOut;
6844 unsigned End = MaskElt + NumInPerOut - 1;
6845 for (; MaskElt < End; MaskElt++)
6846 Mask[MaskElt] = ZeroVecElt++;
6847 Mask[MaskElt] = PackedElt;
6848 }
6849 SDValue Shuf = DAG.getVectorShuffle(VT: InVT, dl: DL, N1: PackedOp, N2: ZeroVec, Mask);
6850 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: OutVT, Operand: Shuf);
6851}
6852
6853SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
6854 unsigned ByScalar) const {
6855 // Look for cases where a vector shift can use the *_BY_SCALAR form.
6856 SDValue Op0 = Op.getOperand(i: 0);
6857 SDValue Op1 = Op.getOperand(i: 1);
6858 SDLoc DL(Op);
6859 EVT VT = Op.getValueType();
6860 unsigned ElemBitSize = VT.getScalarSizeInBits();
6861
6862 // See whether the shift vector is a splat represented as BUILD_VECTOR.
6863 if (auto *BVN = dyn_cast<BuildVectorSDNode>(Val&: Op1)) {
6864 APInt SplatBits, SplatUndef;
6865 unsigned SplatBitSize;
6866 bool HasAnyUndefs;
6867 // Check for constant splats. Use ElemBitSize as the minimum element
6868 // width and reject splats that need wider elements.
6869 if (BVN->isConstantSplat(SplatValue&: SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6870 MinSplatBits: ElemBitSize, isBigEndian: true) &&
6871 SplatBitSize == ElemBitSize) {
6872 SDValue Shift = DAG.getConstant(Val: SplatBits.getZExtValue() & 0xfff,
6873 DL, VT: MVT::i32);
6874 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6875 }
6876 // Check for variable splats.
6877 BitVector UndefElements;
6878 SDValue Splat = BVN->getSplatValue(UndefElements: &UndefElements);
6879 if (Splat) {
6880 // Since i32 is the smallest legal type, we either need a no-op
6881 // or a truncation.
6882 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Splat);
6883 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6884 }
6885 }
6886
6887 // See whether the shift vector is a splat represented as SHUFFLE_VECTOR,
6888 // and the shift amount is directly available in a GPR.
6889 if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Val&: Op1)) {
6890 if (VSN->isSplat()) {
6891 SDValue VSNOp0 = VSN->getOperand(Num: 0);
6892 unsigned Index = VSN->getSplatIndex();
6893 assert(Index < VT.getVectorNumElements() &&
6894 "Splat index should be defined and in first operand");
6895 if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
6896 VSNOp0.getOpcode() == ISD::BUILD_VECTOR) {
6897 // Since i32 is the smallest legal type, we either need a no-op
6898 // or a truncation.
6899 SDValue Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32,
6900 Operand: VSNOp0.getOperand(i: Index));
6901 return DAG.getNode(Opcode: ByScalar, DL, VT, N1: Op0, N2: Shift);
6902 }
6903 }
6904 }
6905
6906 // Otherwise just treat the current form as legal.
6907 return Op;
6908}
6909
6910SDValue SystemZTargetLowering::lowerFSHL(SDValue Op, SelectionDAG &DAG) const {
6911 SDLoc DL(Op);
6912
6913 // i128 FSHL with a constant amount that is a multiple of 8 can be
6914 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
6915 // facility, FSHL with a constant amount less than 8 can be implemented
6916 // via SHL_DOUBLE_BIT, and FSHL with other constant amounts by a
6917 // combination of the two.
6918 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
6919 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
6920 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
6921 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
6922 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
6923 if (ShiftAmt > 120) {
6924 // For N in 121..128, fshl N == fshr (128 - N), and for 1 <= N < 8
6925 // SHR_DOUBLE_BIT emits fewer instructions.
6926 SDValue Val =
6927 DAG.getNode(Opcode: SystemZISD::SHR_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Op0, N2: Op1,
6928 N3: DAG.getTargetConstant(Val: 128 - ShiftAmt, DL, VT: MVT::i32));
6929 return DAG.getBitcast(VT: MVT::i128, V: Val);
6930 }
6931 SmallVector<int, 16> Mask(16);
6932 for (unsigned Elt = 0; Elt < 16; Elt++)
6933 Mask[Elt] = (ShiftAmt >> 3) + Elt;
6934 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
6935 if ((ShiftAmt & 7) == 0)
6936 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
6937 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op1, N2: Op1, Mask);
6938 SDValue Val =
6939 DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf1, N2: Shuf2,
6940 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
6941 return DAG.getBitcast(VT: MVT::i128, V: Val);
6942 }
6943 }
6944
6945 return SDValue();
6946}
6947
6948SDValue SystemZTargetLowering::lowerFSHR(SDValue Op, SelectionDAG &DAG) const {
6949 SDLoc DL(Op);
6950
6951 // i128 FSHR with a constant amount that is a multiple of 8 can be
6952 // implemented via VECTOR_SHUFFLE. If we have the vector-enhancements-2
6953 // facility, FSHR with a constant amount less than 8 can be implemented
6954 // via SHR_DOUBLE_BIT, and FSHR with other constant amounts by a
6955 // combination of the two.
6956 if (auto *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2))) {
6957 uint64_t ShiftAmt = ShiftAmtNode->getZExtValue() & 127;
6958 if ((ShiftAmt & 7) == 0 || Subtarget.hasVectorEnhancements2()) {
6959 SDValue Op0 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 0));
6960 SDValue Op1 = DAG.getBitcast(VT: MVT::v16i8, V: Op.getOperand(i: 1));
6961 if (ShiftAmt > 120) {
6962 // For N in 121..128, fshr N == fshl (128 - N), and for 1 <= N < 8
6963 // SHL_DOUBLE_BIT emits fewer instructions.
6964 SDValue Val =
6965 DAG.getNode(Opcode: SystemZISD::SHL_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Op0, N2: Op1,
6966 N3: DAG.getTargetConstant(Val: 128 - ShiftAmt, DL, VT: MVT::i32));
6967 return DAG.getBitcast(VT: MVT::i128, V: Val);
6968 }
6969 SmallVector<int, 16> Mask(16);
6970 for (unsigned Elt = 0; Elt < 16; Elt++)
6971 Mask[Elt] = 16 - (ShiftAmt >> 3) + Elt;
6972 SDValue Shuf1 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op1, Mask);
6973 if ((ShiftAmt & 7) == 0)
6974 return DAG.getBitcast(VT: MVT::i128, V: Shuf1);
6975 SDValue Shuf2 = DAG.getVectorShuffle(VT: MVT::v16i8, dl: DL, N1: Op0, N2: Op0, Mask);
6976 SDValue Val =
6977 DAG.getNode(Opcode: SystemZISD::SHR_DOUBLE_BIT, DL, VT: MVT::v16i8, N1: Shuf2, N2: Shuf1,
6978 N3: DAG.getTargetConstant(Val: ShiftAmt & 7, DL, VT: MVT::i32));
6979 return DAG.getBitcast(VT: MVT::i128, V: Val);
6980 }
6981 }
6982
6983 return SDValue();
6984}
6985
6986static SDValue lowerAddrSpaceCast(SDValue Op, SelectionDAG &DAG) {
6987 SDLoc DL(Op);
6988 SDValue Src = Op.getOperand(i: 0);
6989 MVT DstVT = Op.getSimpleValueType();
6990
6991 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Val: Op.getNode());
6992 unsigned SrcAS = N->getSrcAddressSpace();
6993
6994 assert(SrcAS != N->getDestAddressSpace() &&
6995 "addrspacecast must be between different address spaces");
6996
6997 // addrspacecast [0 <- 1] : Assinging a ptr32 value to a 64-bit pointer.
6998 // addrspacecast [1 <- 0] : Assigining a 64-bit pointer to a ptr32 value.
6999 if (SrcAS == SYSTEMZAS::PTR32 && DstVT == MVT::i64) {
7000 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Src,
7001 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
7002 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
7003 } else if (DstVT == MVT::i32) {
7004 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: Src);
7005 Op = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Op,
7006 N2: DAG.getConstant(Val: 0x7fffffff, DL, VT: MVT::i32));
7007 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: DstVT, Operand: Op);
7008 } else {
7009 report_fatal_error(reason: "Bad address space in addrspacecast");
7010 }
7011 return Op;
7012}
7013
7014SDValue SystemZTargetLowering::lowerFP_EXTEND(SDValue Op,
7015 SelectionDAG &DAG) const {
7016 SDValue In = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0);
7017 if (In.getSimpleValueType() != MVT::f16)
7018 return Op; // Legal
7019 return SDValue(); // Let legalizer emit the libcall.
7020}
7021
7022SDValue SystemZTargetLowering::useLibCall(SelectionDAG &DAG, RTLIB::Libcall LC,
7023 MVT VT, SDValue Arg, SDLoc DL,
7024 SDValue Chain, bool IsStrict) const {
7025 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected request for libcall!");
7026 MakeLibCallOptions CallOptions;
7027 SDValue Result;
7028 std::tie(args&: Result, args&: Chain) =
7029 makeLibCall(DAG, LC, RetVT: VT, Ops: Arg, CallOptions, dl: DL, Chain);
7030 return IsStrict ? DAG.getMergeValues(Ops: {Result, Chain}, dl: DL) : Result;
7031}
7032
7033SDValue SystemZTargetLowering::lower_FP_TO_INT(SDValue Op,
7034 SelectionDAG &DAG) const {
7035 bool IsSigned = (Op->getOpcode() == ISD::FP_TO_SINT ||
7036 Op->getOpcode() == ISD::STRICT_FP_TO_SINT);
7037 bool IsStrict = Op->isStrictFPOpcode();
7038 SDLoc DL(Op);
7039 MVT VT = Op.getSimpleValueType();
7040 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
7041 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
7042 EVT InVT = InOp.getValueType();
7043
7044 // FP to unsigned is not directly supported on z10. Promoting an i32
7045 // result to (signed) i64 doesn't generate an inexact condition (fp
7046 // exception) for values that are outside the i32 range but in the i64
7047 // range, so use the default expansion.
7048 if (!Subtarget.hasFPExtension() && !IsSigned)
7049 // Expand i32/i64. F16 values will be recognized to fit and extended.
7050 return SDValue();
7051
7052 // Conversion from f16 is done via f32.
7053 if (InOp.getSimpleValueType() == MVT::f16) {
7054 SmallVector<SDValue, 2> Results;
7055 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
7056 return DAG.getMergeValues(Ops: Results, dl: DL);
7057 }
7058
7059 if (VT == MVT::i128) {
7060 RTLIB::Libcall LC =
7061 IsSigned ? RTLIB::getFPTOSINT(OpVT: InVT, RetVT: VT) : RTLIB::getFPTOUINT(OpVT: InVT, RetVT: VT);
7062 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
7063 }
7064
7065 return Op; // Legal
7066}
7067
7068SDValue SystemZTargetLowering::lower_INT_TO_FP(SDValue Op,
7069 SelectionDAG &DAG) const {
7070 bool IsSigned = (Op->getOpcode() == ISD::SINT_TO_FP ||
7071 Op->getOpcode() == ISD::STRICT_SINT_TO_FP);
7072 bool IsStrict = Op->isStrictFPOpcode();
7073 SDLoc DL(Op);
7074 MVT VT = Op.getSimpleValueType();
7075 SDValue InOp = Op.getOperand(i: IsStrict ? 1 : 0);
7076 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
7077 EVT InVT = InOp.getValueType();
7078
7079 // Conversion to f16 is done via f32.
7080 if (VT == MVT::f16) {
7081 SmallVector<SDValue, 2> Results;
7082 LowerOperationWrapper(N: Op.getNode(), Results, DAG);
7083 return DAG.getMergeValues(Ops: Results, dl: DL);
7084 }
7085
7086 // Unsigned to fp is not directly supported on z10.
7087 if (!Subtarget.hasFPExtension() && !IsSigned)
7088 return SDValue(); // Expand i64.
7089
7090 if (InVT == MVT::i128) {
7091 RTLIB::Libcall LC =
7092 IsSigned ? RTLIB::getSINTTOFP(OpVT: InVT, RetVT: VT) : RTLIB::getUINTTOFP(OpVT: InVT, RetVT: VT);
7093 return useLibCall(DAG, LC, VT, Arg: InOp, DL, Chain, IsStrict);
7094 }
7095
7096 return Op; // Legal
7097}
7098
7099// Lower an f16 LOAD in case of no vector support.
7100SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
7101 SelectionDAG &DAG) const {
7102 EVT RegVT = Op.getValueType();
7103 assert(RegVT == MVT::f16 && "Expected to lower an f16 load.");
7104 (void)RegVT;
7105
7106 // Load as integer.
7107 SDLoc DL(Op);
7108 SDValue NewLd;
7109 if (auto *AtomicLd = dyn_cast<AtomicSDNode>(Val: Op.getNode())) {
7110 assert(EVT(RegVT) == AtomicLd->getMemoryVT() && "Unhandled f16 load");
7111 NewLd = DAG.getAtomicLoad(ExtType: ISD::EXTLOAD, dl: DL, MemVT: MVT::i16, VT: MVT::i64,
7112 Chain: AtomicLd->getChain(), Ptr: AtomicLd->getBasePtr(),
7113 MMO: AtomicLd->getMemOperand());
7114 } else {
7115 LoadSDNode *Ld = cast<LoadSDNode>(Val: Op.getNode());
7116 assert(EVT(RegVT) == Ld->getMemoryVT() && "Unhandled f16 load");
7117 NewLd = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i64, Chain: Ld->getChain(),
7118 Ptr: Ld->getBasePtr(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i16,
7119 Alignment: Ld->getBaseAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
7120 }
7121 SDValue F16Val = convertToF16(Op: NewLd, DAG);
7122 return DAG.getMergeValues(Ops: {F16Val, NewLd.getValue(R: 1)}, dl: DL);
7123}
7124
7125// Lower an f16 STORE in case of no vector support.
7126SDValue SystemZTargetLowering::lowerStoreF16(SDValue Op,
7127 SelectionDAG &DAG) const {
7128 SDLoc DL(Op);
7129 SDValue Shft = convertFromF16(Op: Op->getOperand(Num: 1), DL, DAG);
7130
7131 if (auto *AtomicSt = dyn_cast<AtomicSDNode>(Val: Op.getNode()))
7132 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MVT::i16, Chain: AtomicSt->getChain(),
7133 Ptr: Shft, Val: AtomicSt->getBasePtr(),
7134 MMO: AtomicSt->getMemOperand());
7135
7136 StoreSDNode *St = cast<StoreSDNode>(Val: Op.getNode());
7137 return DAG.getTruncStore(Chain: St->getChain(), dl: DL, Val: Shft, Ptr: St->getBasePtr(), SVT: MVT::i16,
7138 MMO: St->getMemOperand());
7139}
7140
7141SDValue SystemZTargetLowering::lowerIS_FPCLASS(SDValue Op,
7142 SelectionDAG &DAG) const {
7143 SDLoc DL(Op);
7144 MVT ResultVT = Op.getSimpleValueType();
7145 SDValue Arg = Op.getOperand(i: 0);
7146 unsigned Check = Op.getConstantOperandVal(i: 1);
7147
7148 unsigned TDCMask = 0;
7149 if (Check & fcSNan)
7150 TDCMask |= SystemZ::TDCMASK_SNAN_PLUS | SystemZ::TDCMASK_SNAN_MINUS;
7151 if (Check & fcQNan)
7152 TDCMask |= SystemZ::TDCMASK_QNAN_PLUS | SystemZ::TDCMASK_QNAN_MINUS;
7153 if (Check & fcPosInf)
7154 TDCMask |= SystemZ::TDCMASK_INFINITY_PLUS;
7155 if (Check & fcNegInf)
7156 TDCMask |= SystemZ::TDCMASK_INFINITY_MINUS;
7157 if (Check & fcPosNormal)
7158 TDCMask |= SystemZ::TDCMASK_NORMAL_PLUS;
7159 if (Check & fcNegNormal)
7160 TDCMask |= SystemZ::TDCMASK_NORMAL_MINUS;
7161 if (Check & fcPosSubnormal)
7162 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_PLUS;
7163 if (Check & fcNegSubnormal)
7164 TDCMask |= SystemZ::TDCMASK_SUBNORMAL_MINUS;
7165 if (Check & fcPosZero)
7166 TDCMask |= SystemZ::TDCMASK_ZERO_PLUS;
7167 if (Check & fcNegZero)
7168 TDCMask |= SystemZ::TDCMASK_ZERO_MINUS;
7169 SDValue TDCMaskV = DAG.getConstant(Val: TDCMask, DL, VT: MVT::i64);
7170
7171 SDValue Intr = DAG.getNode(Opcode: SystemZISD::TDC, DL, VT: ResultVT, N1: Arg, N2: TDCMaskV);
7172 return getCCResult(DAG, CCReg: Intr);
7173}
7174
7175SDValue SystemZTargetLowering::lowerREADCYCLECOUNTER(SDValue Op,
7176 SelectionDAG &DAG) const {
7177 SDLoc DL(Op);
7178 SDValue Chain = Op.getOperand(i: 0);
7179
7180 // STCKF only supports a memory operand, so we have to use a temporary.
7181 SDValue StackPtr = DAG.CreateStackTemporary(VT: MVT::i64);
7182 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
7183 MachinePointerInfo MPI =
7184 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
7185
7186 // Use STCFK to store the TOD clock into the temporary.
7187 SDValue StoreOps[] = {Chain, StackPtr};
7188 Chain = DAG.getMemIntrinsicNode(
7189 Opcode: SystemZISD::STCKF, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps, MemVT: MVT::i64,
7190 PtrInfo: MPI, Alignment: MaybeAlign(), Flags: MachineMemOperand::MOStore);
7191
7192 // And read it back from there.
7193 return DAG.getLoad(VT: MVT::i64, dl: DL, Chain, Ptr: StackPtr, PtrInfo: MPI);
7194}
7195
7196SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
7197 SelectionDAG &DAG) const {
7198 switch (Op.getOpcode()) {
7199 case ISD::FRAMEADDR:
7200 return lowerFRAMEADDR(Op, DAG);
7201 case ISD::RETURNADDR:
7202 return lowerRETURNADDR(Op, DAG);
7203 case ISD::BR_CC:
7204 return lowerBR_CC(Op, DAG);
7205 case ISD::SELECT_CC:
7206 return lowerSELECT_CC(Op, DAG);
7207 case ISD::SETCC:
7208 return lowerSETCC(Op, DAG);
7209 case ISD::STRICT_FSETCC:
7210 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: false);
7211 case ISD::STRICT_FSETCCS:
7212 return lowerSTRICT_FSETCC(Op, DAG, IsSignaling: true);
7213 case ISD::GlobalAddress:
7214 return lowerGlobalAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7215 case ISD::GlobalTLSAddress:
7216 return lowerGlobalTLSAddress(Node: cast<GlobalAddressSDNode>(Val&: Op), DAG);
7217 case ISD::BlockAddress:
7218 return lowerBlockAddress(Node: cast<BlockAddressSDNode>(Val&: Op), DAG);
7219 case ISD::JumpTable:
7220 return lowerJumpTable(JT: cast<JumpTableSDNode>(Val&: Op), DAG);
7221 case ISD::ConstantPool:
7222 return lowerConstantPool(CP: cast<ConstantPoolSDNode>(Val&: Op), DAG);
7223 case ISD::BITCAST:
7224 return lowerBITCAST(Op, DAG);
7225 case ISD::VASTART:
7226 return lowerVASTART(Op, DAG);
7227 case ISD::VACOPY:
7228 return lowerVACOPY(Op, DAG);
7229 case ISD::DYNAMIC_STACKALLOC:
7230 return lowerDYNAMIC_STACKALLOC(Op, DAG);
7231 case ISD::GET_DYNAMIC_AREA_OFFSET:
7232 return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
7233 case ISD::MULHS:
7234 return lowerMULH(Op, DAG, Opcode: SystemZISD::SMUL_LOHI);
7235 case ISD::MULHU:
7236 return lowerMULH(Op, DAG, Opcode: SystemZISD::UMUL_LOHI);
7237 case ISD::SMUL_LOHI:
7238 return lowerSMUL_LOHI(Op, DAG);
7239 case ISD::UMUL_LOHI:
7240 return lowerUMUL_LOHI(Op, DAG);
7241 case ISD::SDIVREM:
7242 return lowerSDIVREM(Op, DAG);
7243 case ISD::UDIVREM:
7244 return lowerUDIVREM(Op, DAG);
7245 case ISD::SADDO:
7246 case ISD::SSUBO:
7247 case ISD::UADDO:
7248 case ISD::USUBO:
7249 return lowerXALUO(Op, DAG);
7250 case ISD::UADDO_CARRY:
7251 case ISD::USUBO_CARRY:
7252 return lowerUADDSUBO_CARRY(Op, DAG);
7253 case ISD::OR:
7254 return lowerOR(Op, DAG);
7255 case ISD::CTPOP:
7256 return lowerCTPOP(Op, DAG);
7257 case ISD::VECREDUCE_ADD:
7258 return lowerVECREDUCE_ADD(Op, DAG);
7259 case ISD::ATOMIC_FENCE:
7260 return lowerATOMIC_FENCE(Op, DAG);
7261 case ISD::ATOMIC_SWAP:
7262 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_SWAPW);
7263 case ISD::ATOMIC_STORE:
7264 return lowerATOMIC_STORE(Op, DAG);
7265 case ISD::ATOMIC_LOAD:
7266 return lowerATOMIC_LOAD(Op, DAG);
7267 case ISD::ATOMIC_LOAD_ADD:
7268 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_ADD);
7269 case ISD::ATOMIC_LOAD_SUB:
7270 return lowerATOMIC_LOAD_SUB(Op, DAG);
7271 case ISD::ATOMIC_LOAD_AND:
7272 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_AND);
7273 case ISD::ATOMIC_LOAD_OR:
7274 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_OR);
7275 case ISD::ATOMIC_LOAD_XOR:
7276 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_XOR);
7277 case ISD::ATOMIC_LOAD_NAND:
7278 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_NAND);
7279 case ISD::ATOMIC_LOAD_MIN:
7280 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MIN);
7281 case ISD::ATOMIC_LOAD_MAX:
7282 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_MAX);
7283 case ISD::ATOMIC_LOAD_UMIN:
7284 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMIN);
7285 case ISD::ATOMIC_LOAD_UMAX:
7286 return lowerATOMIC_LOAD_OP(Op, DAG, Opcode: SystemZISD::ATOMIC_LOADW_UMAX);
7287 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
7288 return lowerATOMIC_CMP_SWAP(Op, DAG);
7289 case ISD::STACKSAVE:
7290 return lowerSTACKSAVE(Op, DAG);
7291 case ISD::STACKRESTORE:
7292 return lowerSTACKRESTORE(Op, DAG);
7293 case ISD::PREFETCH:
7294 return lowerPREFETCH(Op, DAG);
7295 case ISD::INTRINSIC_W_CHAIN:
7296 return lowerINTRINSIC_W_CHAIN(Op, DAG);
7297 case ISD::INTRINSIC_WO_CHAIN:
7298 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
7299 case ISD::BUILD_VECTOR:
7300 return lowerBUILD_VECTOR(Op, DAG);
7301 case ISD::VECTOR_SHUFFLE:
7302 return lowerVECTOR_SHUFFLE(Op, DAG);
7303 case ISD::SCALAR_TO_VECTOR:
7304 return lowerSCALAR_TO_VECTOR(Op, DAG);
7305 case ISD::INSERT_VECTOR_ELT:
7306 return lowerINSERT_VECTOR_ELT(Op, DAG);
7307 case ISD::EXTRACT_VECTOR_ELT:
7308 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7309 case ISD::SIGN_EXTEND_VECTOR_INREG:
7310 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
7311 case ISD::ZERO_EXTEND_VECTOR_INREG:
7312 return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
7313 case ISD::SHL:
7314 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSHL_BY_SCALAR);
7315 case ISD::SRL:
7316 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRL_BY_SCALAR);
7317 case ISD::SRA:
7318 return lowerShift(Op, DAG, ByScalar: SystemZISD::VSRA_BY_SCALAR);
7319 case ISD::ADDRSPACECAST:
7320 return lowerAddrSpaceCast(Op, DAG);
7321 case ISD::ROTL:
7322 return lowerShift(Op, DAG, ByScalar: SystemZISD::VROTL_BY_SCALAR);
7323 case ISD::FSHL:
7324 return lowerFSHL(Op, DAG);
7325 case ISD::FSHR:
7326 return lowerFSHR(Op, DAG);
7327 case ISD::FP_EXTEND:
7328 case ISD::STRICT_FP_EXTEND:
7329 return lowerFP_EXTEND(Op, DAG);
7330 case ISD::FP_TO_UINT:
7331 case ISD::FP_TO_SINT:
7332 case ISD::STRICT_FP_TO_UINT:
7333 case ISD::STRICT_FP_TO_SINT:
7334 return lower_FP_TO_INT(Op, DAG);
7335 case ISD::UINT_TO_FP:
7336 case ISD::SINT_TO_FP:
7337 case ISD::STRICT_UINT_TO_FP:
7338 case ISD::STRICT_SINT_TO_FP:
7339 return lower_INT_TO_FP(Op, DAG);
7340 case ISD::LOAD:
7341 return lowerLoadF16(Op, DAG);
7342 case ISD::STORE:
7343 return lowerStoreF16(Op, DAG);
7344 case ISD::IS_FPCLASS:
7345 return lowerIS_FPCLASS(Op, DAG);
7346 case ISD::GET_ROUNDING:
7347 return lowerGET_ROUNDING(Op, DAG);
7348 case ISD::READCYCLECOUNTER:
7349 return lowerREADCYCLECOUNTER(Op, DAG);
7350 case ISD::EH_SJLJ_SETJMP:
7351 case ISD::EH_SJLJ_LONGJMP:
7352 // These operations are legal on our platform, but we cannot actually
7353 // set the operation action to Legal as common code would treat this
7354 // as equivalent to Expand. Instead, we keep the operation action to
7355 // Custom and just leave them unchanged here.
7356 return Op;
7357
7358 default:
7359 llvm_unreachable("Unexpected node to lower");
7360 }
7361}
7362
7363static SDValue expandBitCastI128ToF128(SelectionDAG &DAG, SDValue Src,
7364 const SDLoc &SL) {
7365 // If i128 is legal, just use a normal bitcast.
7366 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7367 return DAG.getBitcast(VT: MVT::f128, V: Src);
7368
7369 // Otherwise, f128 must live in FP128, so do a partwise move.
7370 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7371 &SystemZ::FP128BitRegClass);
7372
7373 SDValue Hi, Lo;
7374 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Src, DL: SL, LoVT: MVT::i64, HiVT: MVT::i64);
7375
7376 Hi = DAG.getBitcast(VT: MVT::f64, V: Hi);
7377 Lo = DAG.getBitcast(VT: MVT::f64, V: Lo);
7378
7379 SDNode *Pair = DAG.getMachineNode(
7380 Opcode: SystemZ::REG_SEQUENCE, dl: SL, VT: MVT::f128,
7381 Ops: {DAG.getTargetConstant(Val: SystemZ::FP128BitRegClassID, DL: SL, VT: MVT::i32), Lo,
7382 DAG.getTargetConstant(Val: SystemZ::subreg_l64, DL: SL, VT: MVT::i32), Hi,
7383 DAG.getTargetConstant(Val: SystemZ::subreg_h64, DL: SL, VT: MVT::i32)});
7384 return SDValue(Pair, 0);
7385}
7386
7387static SDValue expandBitCastF128ToI128(SelectionDAG &DAG, SDValue Src,
7388 const SDLoc &SL) {
7389 // If i128 is legal, just use a normal bitcast.
7390 if (DAG.getTargetLoweringInfo().isTypeLegal(VT: MVT::i128))
7391 return DAG.getBitcast(VT: MVT::i128, V: Src);
7392
7393 // Otherwise, f128 must live in FP128, so do a partwise move.
7394 assert(DAG.getTargetLoweringInfo().getRepRegClassFor(MVT::f128) ==
7395 &SystemZ::FP128BitRegClass);
7396
7397 SDValue LoFP =
7398 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_l64, DL: SL, VT: MVT::f64, Operand: Src);
7399 SDValue HiFP =
7400 DAG.getTargetExtractSubreg(SRIdx: SystemZ::subreg_h64, DL: SL, VT: MVT::f64, Operand: Src);
7401 SDValue Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: LoFP);
7402 SDValue Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: HiFP);
7403
7404 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i128, N1: Lo, N2: Hi);
7405}
7406
7407// Lower operations with invalid operand or result types.
7408void
7409SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
7410 SmallVectorImpl<SDValue> &Results,
7411 SelectionDAG &DAG) const {
7412 switch (N->getOpcode()) {
7413 case ISD::ATOMIC_LOAD: {
7414 SDLoc DL(N);
7415 SDVTList Tys = DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::Other);
7416 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1) };
7417 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7418 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_LOAD_128,
7419 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7420
7421 SDValue Lowered = lowerGR128ToI128(DAG, In: Res);
7422 if (N->getValueType(ResNo: 0) == MVT::f128)
7423 Lowered = expandBitCastI128ToF128(DAG, Src: Lowered, SL: DL);
7424 Results.push_back(Elt: Lowered);
7425 Results.push_back(Elt: Res.getValue(R: 1));
7426 break;
7427 }
7428 case ISD::ATOMIC_STORE: {
7429 SDLoc DL(N);
7430 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
7431 SDValue Val = N->getOperand(Num: 1);
7432 if (Val.getValueType() == MVT::f128)
7433 Val = expandBitCastF128ToI128(DAG, Src: Val, SL: DL);
7434 Val = lowerI128ToGR128(DAG, In: Val);
7435
7436 SDValue Ops[] = {N->getOperand(Num: 0), Val, N->getOperand(Num: 2)};
7437 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7438 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_STORE_128,
7439 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7440 // We have to enforce sequential consistency by performing a
7441 // serialization operation after the store.
7442 if (cast<AtomicSDNode>(Val: N)->getSuccessOrdering() ==
7443 AtomicOrdering::SequentiallyConsistent)
7444 Res = SDValue(DAG.getMachineNode(Opcode: SystemZ::Serialize, dl: DL,
7445 VT: MVT::Other, Op1: Res), 0);
7446 Results.push_back(Elt: Res);
7447 break;
7448 }
7449 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
7450 SDLoc DL(N);
7451 SDVTList Tys = DAG.getVTList(VT1: MVT::Untyped, VT2: MVT::i32, VT3: MVT::Other);
7452 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1),
7453 lowerI128ToGR128(DAG, In: N->getOperand(Num: 2)),
7454 lowerI128ToGR128(DAG, In: N->getOperand(Num: 3)) };
7455 MachineMemOperand *MMO = cast<AtomicSDNode>(Val: N)->getMemOperand();
7456 SDValue Res = DAG.getMemIntrinsicNode(Opcode: SystemZISD::ATOMIC_CMP_SWAP_128,
7457 dl: DL, VTList: Tys, Ops, MemVT: MVT::i128, MMO);
7458 SDValue Success = emitSETCC(DAG, DL, CCReg: Res.getValue(R: 1),
7459 CCValid: SystemZ::CCMASK_CS, CCMask: SystemZ::CCMASK_CS_EQ);
7460 Success = DAG.getZExtOrTrunc(Op: Success, DL, VT: N->getValueType(ResNo: 1));
7461 Results.push_back(Elt: lowerGR128ToI128(DAG, In: Res));
7462 Results.push_back(Elt: Success);
7463 Results.push_back(Elt: Res.getValue(R: 2));
7464 break;
7465 }
7466 case ISD::BITCAST: {
7467 if (useSoftFloat())
7468 return;
7469 SDLoc DL(N);
7470 SDValue Src = N->getOperand(Num: 0);
7471 EVT SrcVT = Src.getValueType();
7472 EVT ResVT = N->getValueType(ResNo: 0);
7473 if (ResVT == MVT::i128 && SrcVT == MVT::f128)
7474 Results.push_back(Elt: expandBitCastF128ToI128(DAG, Src, SL: DL));
7475 else if (SrcVT == MVT::i16 && ResVT == MVT::f16) {
7476 if (Subtarget.hasVector()) {
7477 SDValue In32 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Src);
7478 Results.push_back(Elt: SDValue(
7479 DAG.getMachineNode(Opcode: SystemZ::LEFR_16, dl: DL, VT: MVT::f16, Op1: In32), 0));
7480 } else {
7481 SDValue In64 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Src);
7482 Results.push_back(Elt: convertToF16(Op: In64, DAG));
7483 }
7484 } else if (SrcVT == MVT::f16 && ResVT == MVT::i16) {
7485 SDValue ExtractedVal =
7486 Subtarget.hasVector()
7487 ? SDValue(DAG.getMachineNode(Opcode: SystemZ::LFER_16, dl: DL, VT: MVT::i32, Op1: Src),
7488 0)
7489 : convertFromF16(Op: Src, DL, DAG);
7490 Results.push_back(Elt: DAG.getZExtOrTrunc(Op: ExtractedVal, DL, VT: ResVT));
7491 }
7492 break;
7493 }
7494 case ISD::UINT_TO_FP:
7495 case ISD::SINT_TO_FP:
7496 case ISD::STRICT_UINT_TO_FP:
7497 case ISD::STRICT_SINT_TO_FP: {
7498 if (useSoftFloat())
7499 return;
7500 bool IsStrict = N->isStrictFPOpcode();
7501 SDLoc DL(N);
7502 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7503 EVT ResVT = N->getValueType(ResNo: 0);
7504 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7505 if (ResVT == MVT::f16) {
7506 if (!IsStrict) {
7507 SDValue OpF32 = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::f32, Operand: InOp);
7508 Results.push_back(Elt: DAG.getFPExtendOrRound(Op: OpF32, DL, VT: MVT::f16));
7509 } else {
7510 SDValue OpF32 =
7511 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
7512 Ops: {Chain, InOp});
7513 SDValue F16Res;
7514 std::tie(args&: F16Res, args&: Chain) = DAG.getStrictFPExtendOrRound(
7515 Op: OpF32, Chain: OpF32.getValue(R: 1), DL, VT: MVT::f16);
7516 Results.push_back(Elt: F16Res);
7517 Results.push_back(Elt: Chain);
7518 }
7519 }
7520 break;
7521 }
7522 case ISD::FP_TO_UINT:
7523 case ISD::FP_TO_SINT:
7524 case ISD::STRICT_FP_TO_UINT:
7525 case ISD::STRICT_FP_TO_SINT: {
7526 if (useSoftFloat())
7527 return;
7528 bool IsStrict = N->isStrictFPOpcode();
7529 SDLoc DL(N);
7530 EVT ResVT = N->getValueType(ResNo: 0);
7531 SDValue InOp = N->getOperand(Num: IsStrict ? 1 : 0);
7532 EVT InVT = InOp->getValueType(ResNo: 0);
7533 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : DAG.getEntryNode();
7534 if (InVT == MVT::f16) {
7535 if (!IsStrict) {
7536 SDValue InF32 = DAG.getFPExtendOrRound(Op: InOp, DL, VT: MVT::f32);
7537 Results.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL, VT: ResVT, Operand: InF32));
7538 } else {
7539 SDValue InF32;
7540 std::tie(args&: InF32, args&: Chain) =
7541 DAG.getStrictFPExtendOrRound(Op: InOp, Chain, DL, VT: MVT::f32);
7542 SDValue OpF32 =
7543 DAG.getNode(Opcode: N->getOpcode(), DL, VTList: DAG.getVTList(VT1: ResVT, VT2: MVT::Other),
7544 Ops: {Chain, InF32});
7545 Results.push_back(Elt: OpF32);
7546 Results.push_back(Elt: OpF32.getValue(R: 1));
7547 }
7548 }
7549 break;
7550 }
7551 default:
7552 llvm_unreachable("Unexpected node to lower");
7553 }
7554}
7555
7556void
7557SystemZTargetLowering::ReplaceNodeResults(SDNode *N,
7558 SmallVectorImpl<SDValue> &Results,
7559 SelectionDAG &DAG) const {
7560 return LowerOperationWrapper(N, Results, DAG);
7561}
7562
7563// Return true if VT is a vector whose elements are a whole number of bytes
7564// in width. Also check for presence of vector support.
7565bool SystemZTargetLowering::canTreatAsByteVector(EVT VT) const {
7566 if (!Subtarget.hasVector())
7567 return false;
7568
7569 return VT.isVector() && VT.getScalarSizeInBits() % 8 == 0 && VT.isSimple();
7570}
7571
7572// Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT
7573// producing a result of type ResVT. Op is a possibly bitcast version
7574// of the input vector and Index is the index (based on type VecVT) that
7575// should be extracted. Return the new extraction if a simplification
7576// was possible or if Force is true.
7577SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
7578 EVT VecVT, SDValue Op,
7579 unsigned Index,
7580 DAGCombinerInfo &DCI,
7581 bool Force) const {
7582 SelectionDAG &DAG = DCI.DAG;
7583
7584 // The number of bytes being extracted.
7585 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7586
7587 for (;;) {
7588 unsigned Opcode = Op.getOpcode();
7589 if (Opcode == ISD::BITCAST)
7590 // Look through bitcasts.
7591 Op = Op.getOperand(i: 0);
7592 else if ((Opcode == ISD::VECTOR_SHUFFLE || Opcode == SystemZISD::SPLAT) &&
7593 canTreatAsByteVector(VT: Op.getValueType())) {
7594 // Get a VPERM-like permute mask and see whether the bytes covered
7595 // by the extracted element are a contiguous sequence from one
7596 // source operand.
7597 SmallVector<int, SystemZ::VectorBytes> Bytes;
7598 if (!getVPermMask(ShuffleOp: Op, Bytes))
7599 break;
7600 int First;
7601 if (!getShuffleInput(Bytes, Start: Index * BytesPerElement,
7602 BytesPerElement, Base&: First))
7603 break;
7604 if (First < 0)
7605 return DAG.getUNDEF(VT: ResVT);
7606 // Make sure the contiguous sequence starts at a multiple of the
7607 // original element size.
7608 unsigned Byte = unsigned(First) % Bytes.size();
7609 if (Byte % BytesPerElement != 0)
7610 break;
7611 // We can get the extracted value directly from an input.
7612 Index = Byte / BytesPerElement;
7613 Op = Op.getOperand(i: unsigned(First) / Bytes.size());
7614 Force = true;
7615 } else if (Opcode == ISD::BUILD_VECTOR &&
7616 canTreatAsByteVector(VT: Op.getValueType())) {
7617 // We can only optimize this case if the BUILD_VECTOR elements are
7618 // at least as wide as the extracted value.
7619 EVT OpVT = Op.getValueType();
7620 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7621 if (OpBytesPerElement < BytesPerElement)
7622 break;
7623 // Make sure that the least-significant bit of the extracted value
7624 // is the least significant bit of an input.
7625 unsigned End = (Index + 1) * BytesPerElement;
7626 if (End % OpBytesPerElement != 0)
7627 break;
7628 // We're extracting the low part of one operand of the BUILD_VECTOR.
7629 Op = Op.getOperand(i: End / OpBytesPerElement - 1);
7630 if (!Op.getValueType().isInteger()) {
7631 EVT VT = MVT::getIntegerVT(BitWidth: Op.getValueSizeInBits());
7632 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Op);
7633 DCI.AddToWorklist(N: Op.getNode());
7634 }
7635 EVT VT = MVT::getIntegerVT(BitWidth: ResVT.getSizeInBits());
7636 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
7637 if (VT != ResVT) {
7638 DCI.AddToWorklist(N: Op.getNode());
7639 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ResVT, Operand: Op);
7640 }
7641 return Op;
7642 } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
7643 Opcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
7644 Opcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
7645 canTreatAsByteVector(VT: Op.getValueType()) &&
7646 canTreatAsByteVector(VT: Op.getOperand(i: 0).getValueType())) {
7647 // Make sure that only the unextended bits are significant.
7648 EVT ExtVT = Op.getValueType();
7649 EVT OpVT = Op.getOperand(i: 0).getValueType();
7650 unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize();
7651 unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
7652 unsigned Byte = Index * BytesPerElement;
7653 unsigned SubByte = Byte % ExtBytesPerElement;
7654 unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
7655 if (SubByte < MinSubByte ||
7656 SubByte + BytesPerElement > ExtBytesPerElement)
7657 break;
7658 // Get the byte offset of the unextended element
7659 Byte = Byte / ExtBytesPerElement * OpBytesPerElement;
7660 // ...then add the byte offset relative to that element.
7661 Byte += SubByte - MinSubByte;
7662 if (Byte % BytesPerElement != 0)
7663 break;
7664 Op = Op.getOperand(i: 0);
7665 Index = Byte / BytesPerElement;
7666 Force = true;
7667 } else
7668 break;
7669 }
7670 if (Force) {
7671 if (Op.getValueType() != VecVT) {
7672 Op = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VecVT, Operand: Op);
7673 DCI.AddToWorklist(N: Op.getNode());
7674 }
7675 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ResVT, N1: Op,
7676 N2: DAG.getConstant(Val: Index, DL, VT: MVT::i32));
7677 }
7678 return SDValue();
7679}
7680
7681// Optimize vector operations in scalar value Op on the basis that Op
7682// is truncated to TruncVT.
7683SDValue SystemZTargetLowering::combineTruncateExtract(
7684 const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
7685 // If we have (trunc (extract_vector_elt X, Y)), try to turn it into
7686 // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements
7687 // of type TruncVT.
7688 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7689 TruncVT.getSizeInBits() % 8 == 0) {
7690 SDValue Vec = Op.getOperand(i: 0);
7691 EVT VecVT = Vec.getValueType();
7692 if (canTreatAsByteVector(VT: VecVT)) {
7693 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
7694 unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
7695 unsigned TruncBytes = TruncVT.getStoreSize();
7696 if (BytesPerElement % TruncBytes == 0) {
7697 // Calculate the value of Y' in the above description. We are
7698 // splitting the original elements into Scale equal-sized pieces
7699 // and for truncation purposes want the last (least-significant)
7700 // of these pieces for IndexN. This is easiest to do by calculating
7701 // the start index of the following element and then subtracting 1.
7702 unsigned Scale = BytesPerElement / TruncBytes;
7703 unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
7704
7705 // Defer the creation of the bitcast from X to combineExtract,
7706 // which might be able to optimize the extraction.
7707 VecVT = EVT::getVectorVT(Context&: *DCI.DAG.getContext(),
7708 VT: MVT::getIntegerVT(BitWidth: TruncBytes * 8),
7709 NumElements: VecVT.getStoreSize() / TruncBytes);
7710 EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
7711 return combineExtract(DL, ResVT, VecVT, Op: Vec, Index: NewIndex, DCI, Force: true);
7712 }
7713 }
7714 }
7715 }
7716 return SDValue();
7717}
7718
7719SDValue SystemZTargetLowering::combineZERO_EXTEND(
7720 SDNode *N, DAGCombinerInfo &DCI) const {
7721 // Convert (zext (select_ccmask C1, C2)) into (select_ccmask C1', C2')
7722 SelectionDAG &DAG = DCI.DAG;
7723 SDValue N0 = N->getOperand(Num: 0);
7724 EVT VT = N->getValueType(ResNo: 0);
7725 if (N0.getOpcode() == SystemZISD::SELECT_CCMASK) {
7726 auto *TrueOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0));
7727 auto *FalseOp = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7728 if (TrueOp && FalseOp) {
7729 SDLoc DL(N0);
7730 SDValue Ops[] = { DAG.getConstant(Val: TrueOp->getZExtValue(), DL, VT),
7731 DAG.getConstant(Val: FalseOp->getZExtValue(), DL, VT),
7732 N0.getOperand(i: 2), N0.getOperand(i: 3), N0.getOperand(i: 4) };
7733 SDValue NewSelect = DAG.getNode(Opcode: SystemZISD::SELECT_CCMASK, DL, VT, Ops);
7734 // If N0 has multiple uses, change other uses as well.
7735 if (!N0.hasOneUse()) {
7736 SDValue TruncSelect =
7737 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N0.getValueType(), Operand: NewSelect);
7738 DCI.CombineTo(N: N0.getNode(), Res: TruncSelect);
7739 }
7740 return NewSelect;
7741 }
7742 }
7743 // Convert (zext (xor (trunc X), C)) into (xor (trunc X), C') if the size
7744 // of the result is smaller than the size of X and all the truncated bits
7745 // of X are already zero.
7746 if (N0.getOpcode() == ISD::XOR &&
7747 N0.hasOneUse() && N0.getOperand(i: 0).hasOneUse() &&
7748 N0.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
7749 N0.getOperand(i: 1).getOpcode() == ISD::Constant) {
7750 SDValue X = N0.getOperand(i: 0).getOperand(i: 0);
7751 if (VT.isScalarInteger() && VT.getSizeInBits() < X.getValueSizeInBits()) {
7752 KnownBits Known = DAG.computeKnownBits(Op: X);
7753 APInt TruncatedBits = APInt::getBitsSet(numBits: X.getValueSizeInBits(),
7754 loBit: N0.getValueSizeInBits(),
7755 hiBit: VT.getSizeInBits());
7756 if (TruncatedBits.isSubsetOf(RHS: Known.Zero)) {
7757 X = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(X), VT, Operand: X);
7758 APInt Mask = N0.getConstantOperandAPInt(i: 1).zext(width: VT.getSizeInBits());
7759 return DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N0), VT,
7760 N1: X, N2: DAG.getConstant(Val: Mask, DL: SDLoc(N0), VT));
7761 }
7762 }
7763 }
7764 // Recognize patterns for VECTOR SUBTRACT COMPUTE BORROW INDICATION
7765 // and VECTOR ADD COMPUTE CARRY for i128:
7766 // (zext (setcc_uge X Y)) --> (VSCBI X Y)
7767 // (zext (setcc_ule Y X)) --> (VSCBI X Y)
7768 // (zext (setcc_ult (add X Y) X/Y) -> (VACC X Y)
7769 // (zext (setcc_ugt X/Y (add X Y)) -> (VACC X Y)
7770 // For vector types, these patterns are recognized in the .td file.
7771 if (N0.getOpcode() == ISD::SETCC && isTypeLegal(VT) && VT == MVT::i128 &&
7772 N0.getOperand(i: 0).getValueType() == VT) {
7773 SDValue Op0 = N0.getOperand(i: 0);
7774 SDValue Op1 = N0.getOperand(i: 1);
7775 const ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
7776 switch (CC) {
7777 case ISD::SETULE:
7778 std::swap(a&: Op0, b&: Op1);
7779 [[fallthrough]];
7780 case ISD::SETUGE:
7781 return DAG.getNode(Opcode: SystemZISD::VSCBI, DL: SDLoc(N0), VT, N1: Op0, N2: Op1);
7782 case ISD::SETUGT:
7783 std::swap(a&: Op0, b&: Op1);
7784 [[fallthrough]];
7785 case ISD::SETULT:
7786 if (Op0->hasOneUse() && Op0->getOpcode() == ISD::ADD &&
7787 (Op0->getOperand(Num: 0) == Op1 || Op0->getOperand(Num: 1) == Op1))
7788 return DAG.getNode(Opcode: SystemZISD::VACC, DL: SDLoc(N0), VT, N1: Op0->getOperand(Num: 0),
7789 N2: Op0->getOperand(Num: 1));
7790 break;
7791 default:
7792 break;
7793 }
7794 }
7795
7796 return SDValue();
7797}
7798
7799SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
7800 SDNode *N, DAGCombinerInfo &DCI) const {
7801 // Convert (sext_in_reg (setcc LHS, RHS, COND), i1)
7802 // and (sext_in_reg (any_extend (setcc LHS, RHS, COND)), i1)
7803 // into (select_cc LHS, RHS, -1, 0, COND)
7804 SelectionDAG &DAG = DCI.DAG;
7805 SDValue N0 = N->getOperand(Num: 0);
7806 EVT VT = N->getValueType(ResNo: 0);
7807 EVT EVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
7808 if (N0.hasOneUse() && N0.getOpcode() == ISD::ANY_EXTEND)
7809 N0 = N0.getOperand(i: 0);
7810 if (EVT == MVT::i1 && N0.hasOneUse() && N0.getOpcode() == ISD::SETCC) {
7811 SDLoc DL(N0);
7812 SDValue Ops[] = { N0.getOperand(i: 0), N0.getOperand(i: 1),
7813 DAG.getAllOnesConstant(DL, VT),
7814 DAG.getConstant(Val: 0, DL, VT), N0.getOperand(i: 2) };
7815 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT, Ops);
7816 }
7817 return SDValue();
7818}
7819
7820SDValue SystemZTargetLowering::combineSIGN_EXTEND(
7821 SDNode *N, DAGCombinerInfo &DCI) const {
7822 // Convert (sext (ashr (shl X, C1), C2)) to
7823 // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as
7824 // cheap as narrower ones.
7825 SelectionDAG &DAG = DCI.DAG;
7826 SDValue N0 = N->getOperand(Num: 0);
7827 EVT VT = N->getValueType(ResNo: 0);
7828 if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) {
7829 auto *SraAmt = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
7830 SDValue Inner = N0.getOperand(i: 0);
7831 if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) {
7832 if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Val: Inner.getOperand(i: 1))) {
7833 unsigned Extra = (VT.getSizeInBits() - N0.getValueSizeInBits());
7834 unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
7835 unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
7836 EVT ShiftVT = N0.getOperand(i: 1).getValueType();
7837 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(Inner), VT,
7838 Operand: Inner.getOperand(i: 0));
7839 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(Inner), VT, N1: Ext,
7840 N2: DAG.getConstant(Val: NewShlAmt, DL: SDLoc(Inner),
7841 VT: ShiftVT));
7842 return DAG.getNode(Opcode: ISD::SRA, DL: SDLoc(N0), VT, N1: Shl,
7843 N2: DAG.getConstant(Val: NewSraAmt, DL: SDLoc(N0), VT: ShiftVT));
7844 }
7845 }
7846 }
7847
7848 return SDValue();
7849}
7850
7851SDValue SystemZTargetLowering::combineMERGE(
7852 SDNode *N, DAGCombinerInfo &DCI) const {
7853 SelectionDAG &DAG = DCI.DAG;
7854 unsigned Opcode = N->getOpcode();
7855 SDValue Op0 = N->getOperand(Num: 0);
7856 SDValue Op1 = N->getOperand(Num: 1);
7857 if (Op0.getOpcode() == ISD::BITCAST)
7858 Op0 = Op0.getOperand(i: 0);
7859 if (ISD::isBuildVectorAllZeros(N: Op0.getNode())) {
7860 // (z_merge_* 0, 0) -> 0. This is mostly useful for using VLLEZF
7861 // for v4f32.
7862 if (Op1 == N->getOperand(Num: 0))
7863 return Op1;
7864 // (z_merge_? 0, X) -> (z_unpackl_? 0, X).
7865 EVT VT = Op1.getValueType();
7866 unsigned ElemBytes = VT.getVectorElementType().getStoreSize();
7867 if (ElemBytes <= 4) {
7868 Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
7869 SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
7870 EVT InVT = VT.changeVectorElementTypeToInteger();
7871 EVT OutVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ElemBytes * 16),
7872 NumElements: SystemZ::VectorBytes / ElemBytes / 2);
7873 if (VT != InVT) {
7874 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: InVT, Operand: Op1);
7875 DCI.AddToWorklist(N: Op1.getNode());
7876 }
7877 SDValue Op = DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: Op1);
7878 DCI.AddToWorklist(N: Op.getNode());
7879 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT, Operand: Op);
7880 }
7881 }
7882 return SDValue();
7883}
7884
7885static bool isI128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
7886 SDNode *&HiPart) {
7887 LoPart = HiPart = nullptr;
7888
7889 // Scan through all users.
7890 for (SDUse &Use : LD->uses()) {
7891 // Skip the uses of the chain.
7892 if (Use.getResNo() != 0)
7893 continue;
7894
7895 // Verify every user is a TRUNCATE to i64 of the low or high half.
7896 SDNode *User = Use.getUser();
7897 bool IsLoPart = true;
7898 if (User->getOpcode() == ISD::SRL &&
7899 User->getOperand(Num: 1).getOpcode() == ISD::Constant &&
7900 User->getConstantOperandVal(Num: 1) == 64 && User->hasOneUse()) {
7901 User = *User->user_begin();
7902 IsLoPart = false;
7903 }
7904 if (User->getOpcode() != ISD::TRUNCATE || User->getValueType(ResNo: 0) != MVT::i64)
7905 return false;
7906
7907 if (IsLoPart) {
7908 if (LoPart)
7909 return false;
7910 LoPart = User;
7911 } else {
7912 if (HiPart)
7913 return false;
7914 HiPart = User;
7915 }
7916 }
7917 return true;
7918}
7919
7920static bool isF128MovedToParts(LoadSDNode *LD, SDNode *&LoPart,
7921 SDNode *&HiPart) {
7922 LoPart = HiPart = nullptr;
7923
7924 // Scan through all users.
7925 for (SDUse &Use : LD->uses()) {
7926 // Skip the uses of the chain.
7927 if (Use.getResNo() != 0)
7928 continue;
7929
7930 // Verify every user is an EXTRACT_SUBREG of the low or high half.
7931 SDNode *User = Use.getUser();
7932 if (!User->hasOneUse() || !User->isMachineOpcode() ||
7933 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
7934 return false;
7935
7936 switch (User->getConstantOperandVal(Num: 1)) {
7937 case SystemZ::subreg_l64:
7938 if (LoPart)
7939 return false;
7940 LoPart = User;
7941 break;
7942 case SystemZ::subreg_h64:
7943 if (HiPart)
7944 return false;
7945 HiPart = User;
7946 break;
7947 default:
7948 return false;
7949 }
7950 }
7951 return true;
7952}
7953
7954SDValue SystemZTargetLowering::combineLOAD(
7955 SDNode *N, DAGCombinerInfo &DCI) const {
7956 SelectionDAG &DAG = DCI.DAG;
7957 EVT LdVT = N->getValueType(ResNo: 0);
7958 if (auto *LN = dyn_cast<LoadSDNode>(Val: N)) {
7959 if (LN->getAddressSpace() == SYSTEMZAS::PTR32) {
7960 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7961 MVT LoadNodeVT = LN->getBasePtr().getSimpleValueType();
7962 if (PtrVT != LoadNodeVT) {
7963 SDLoc DL(LN);
7964 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(
7965 dl: DL, VT: PtrVT, Ptr: LN->getBasePtr(), SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
7966 return DAG.getExtLoad(ExtType: LN->getExtensionType(), dl: DL, VT: LN->getValueType(ResNo: 0),
7967 Chain: LN->getChain(), Ptr: AddrSpaceCast, MemVT: LN->getMemoryVT(),
7968 MMO: LN->getMemOperand());
7969 }
7970 }
7971 }
7972 SDLoc DL(N);
7973
7974 // Replace a 128-bit load that is used solely to move its value into GPRs
7975 // by separate loads of both halves.
7976 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
7977 if (LD->isSimple() && ISD::isNormalLoad(N: LD)) {
7978 SDNode *LoPart, *HiPart;
7979 if ((LdVT == MVT::i128 && isI128MovedToParts(LD, LoPart, HiPart)) ||
7980 (LdVT == MVT::f128 && isF128MovedToParts(LD, LoPart, HiPart))) {
7981 // Rewrite each extraction as an independent load.
7982 SmallVector<SDValue, 2> ArgChains;
7983 if (HiPart) {
7984 SDValue EltLoad = DAG.getLoad(
7985 VT: HiPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(), Ptr: LD->getBasePtr(),
7986 PtrInfo: LD->getPointerInfo(), Alignment: LD->getBaseAlign(),
7987 MMOFlags: LD->getMemOperand()->getFlags(), AAInfo: LD->getAAInfo());
7988
7989 DCI.CombineTo(N: HiPart, Res: EltLoad, AddTo: true);
7990 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
7991 }
7992 if (LoPart) {
7993 SDValue EltLoad = DAG.getLoad(
7994 VT: LoPart->getValueType(ResNo: 0), dl: DL, Chain: LD->getChain(),
7995 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: LD->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
7996 PtrInfo: LD->getPointerInfo().getWithOffset(O: 8), Alignment: LD->getBaseAlign(),
7997 MMOFlags: LD->getMemOperand()->getFlags(), AAInfo: LD->getAAInfo());
7998
7999 DCI.CombineTo(N: LoPart, Res: EltLoad, AddTo: true);
8000 ArgChains.push_back(Elt: EltLoad.getValue(R: 1));
8001 }
8002
8003 // Collect all chains via TokenFactor.
8004 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ArgChains);
8005 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: Chain);
8006 DCI.AddToWorklist(N: Chain.getNode());
8007 return SDValue(N, 0);
8008 }
8009 }
8010
8011 if (LdVT.isVector() || LdVT.isInteger())
8012 return SDValue();
8013 // Transform a scalar load that is REPLICATEd as well as having other
8014 // use(s) to the form where the other use(s) use the first element of the
8015 // REPLICATE instead of the load. Otherwise instruction selection will not
8016 // produce a VLREP. Avoid extracting to a GPR, so only do this for floating
8017 // point loads.
8018
8019 SDValue Replicate;
8020 SmallVector<SDNode*, 8> OtherUses;
8021 for (SDUse &Use : N->uses()) {
8022 if (Use.getUser()->getOpcode() == SystemZISD::REPLICATE) {
8023 if (Replicate)
8024 return SDValue(); // Should never happen
8025 Replicate = SDValue(Use.getUser(), 0);
8026 } else if (Use.getResNo() == 0)
8027 OtherUses.push_back(Elt: Use.getUser());
8028 }
8029 if (!Replicate || OtherUses.empty())
8030 return SDValue();
8031
8032 SDValue Extract0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: LdVT,
8033 N1: Replicate, N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
8034 // Update uses of the loaded Value while preserving old chains.
8035 for (SDNode *U : OtherUses) {
8036 SmallVector<SDValue, 8> Ops;
8037 for (SDValue Op : U->ops())
8038 Ops.push_back(Elt: (Op.getNode() == N && Op.getResNo() == 0) ? Extract0 : Op);
8039 DAG.UpdateNodeOperands(N: U, Ops);
8040 }
8041 return SDValue(N, 0);
8042}
8043
8044bool SystemZTargetLowering::canLoadStoreByteSwapped(EVT VT) const {
8045 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64)
8046 return true;
8047 if (Subtarget.hasVectorEnhancements2())
8048 if (VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v2i64 || VT == MVT::i128)
8049 return true;
8050 return false;
8051}
8052
8053static bool isVectorElementSwap(ArrayRef<int> M, EVT VT) {
8054 if (!VT.isVector() || !VT.isSimple() ||
8055 VT.getSizeInBits() != 128 ||
8056 VT.getScalarSizeInBits() % 8 != 0)
8057 return false;
8058
8059 unsigned NumElts = VT.getVectorNumElements();
8060 for (unsigned i = 0; i < NumElts; ++i) {
8061 if (M[i] < 0) continue; // ignore UNDEF indices
8062 if ((unsigned) M[i] != NumElts - 1 - i)
8063 return false;
8064 }
8065
8066 return true;
8067}
8068
8069static bool isOnlyUsedByStores(SDValue StoredVal, SelectionDAG &DAG) {
8070 for (auto *U : StoredVal->users()) {
8071 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: U)) {
8072 EVT CurrMemVT = ST->getMemoryVT().getScalarType();
8073 if (CurrMemVT.isRound() && CurrMemVT.getStoreSize() <= 16)
8074 continue;
8075 } else if (isa<BuildVectorSDNode>(Val: U)) {
8076 SDValue BuildVector = SDValue(U, 0);
8077 if (DAG.isSplatValue(V: BuildVector, AllowUndefs: true/*AllowUndefs*/) &&
8078 isOnlyUsedByStores(StoredVal: BuildVector, DAG))
8079 continue;
8080 }
8081 return false;
8082 }
8083 return true;
8084}
8085
8086static bool isI128MovedFromParts(SDValue Val, SDValue &LoPart,
8087 SDValue &HiPart) {
8088 if (Val.getOpcode() != ISD::OR || !Val.getNode()->hasOneUse())
8089 return false;
8090
8091 SDValue Op0 = Val.getOperand(i: 0);
8092 SDValue Op1 = Val.getOperand(i: 1);
8093
8094 if (Op0.getOpcode() == ISD::SHL)
8095 std::swap(a&: Op0, b&: Op1);
8096 if (Op1.getOpcode() != ISD::SHL || !Op1.getNode()->hasOneUse() ||
8097 Op1.getOperand(i: 1).getOpcode() != ISD::Constant ||
8098 Op1.getConstantOperandVal(i: 1) != 64)
8099 return false;
8100 Op1 = Op1.getOperand(i: 0);
8101
8102 if (Op0.getOpcode() != ISD::ZERO_EXTEND || !Op0.getNode()->hasOneUse() ||
8103 Op0.getOperand(i: 0).getValueType() != MVT::i64)
8104 return false;
8105 if (Op1.getOpcode() != ISD::ANY_EXTEND || !Op1.getNode()->hasOneUse() ||
8106 Op1.getOperand(i: 0).getValueType() != MVT::i64)
8107 return false;
8108
8109 LoPart = Op0.getOperand(i: 0);
8110 HiPart = Op1.getOperand(i: 0);
8111 return true;
8112}
8113
8114static bool isF128MovedFromParts(SDValue Val, SDValue &LoPart,
8115 SDValue &HiPart) {
8116 if (!Val.getNode()->hasOneUse() || !Val.isMachineOpcode() ||
8117 Val.getMachineOpcode() != TargetOpcode::REG_SEQUENCE)
8118 return false;
8119
8120 if (Val->getNumOperands() != 5 ||
8121 Val->getOperand(Num: 0)->getAsZExtVal() != SystemZ::FP128BitRegClassID ||
8122 Val->getOperand(Num: 2)->getAsZExtVal() != SystemZ::subreg_l64 ||
8123 Val->getOperand(Num: 4)->getAsZExtVal() != SystemZ::subreg_h64)
8124 return false;
8125
8126 LoPart = Val->getOperand(Num: 1);
8127 HiPart = Val->getOperand(Num: 3);
8128 return true;
8129}
8130
8131SDValue SystemZTargetLowering::combineSTORE(
8132 SDNode *N, DAGCombinerInfo &DCI) const {
8133 SelectionDAG &DAG = DCI.DAG;
8134 auto *SN = cast<StoreSDNode>(Val: N);
8135 auto &Op1 = N->getOperand(Num: 1);
8136 EVT MemVT = SN->getMemoryVT();
8137
8138 if (SN->getAddressSpace() == SYSTEMZAS::PTR32) {
8139 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8140 MVT StoreNodeVT = SN->getBasePtr().getSimpleValueType();
8141 if (PtrVT != StoreNodeVT) {
8142 SDLoc DL(SN);
8143 SDValue AddrSpaceCast = DAG.getAddrSpaceCast(dl: DL, VT: PtrVT, Ptr: SN->getBasePtr(),
8144 SrcAS: SYSTEMZAS::PTR32, DestAS: 0);
8145 return DAG.getStore(Chain: SN->getChain(), dl: DL, Val: SN->getValue(), Ptr: AddrSpaceCast,
8146 PtrInfo: SN->getPointerInfo(), Alignment: SN->getBaseAlign(),
8147 MMOFlags: SN->getMemOperand()->getFlags(), AAInfo: SN->getAAInfo());
8148 }
8149 }
8150
8151 // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better
8152 // for the extraction to be done on a vMiN value, so that we can use VSTE.
8153 // If X has wider elements then convert it to:
8154 // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z).
8155 if (MemVT.isInteger() && SN->isTruncatingStore()) {
8156 if (SDValue Value =
8157 combineTruncateExtract(DL: SDLoc(N), TruncVT: MemVT, Op: SN->getValue(), DCI)) {
8158 DCI.AddToWorklist(N: Value.getNode());
8159
8160 // Rewrite the store with the new form of stored value.
8161 return DAG.getTruncStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: Value,
8162 Ptr: SN->getBasePtr(), SVT: SN->getMemoryVT(),
8163 MMO: SN->getMemOperand());
8164 }
8165 }
8166
8167 // combine STORE (LOAD_STACK_GUARD) into MOV_STACKGUARD_DAG
8168 if (Op1->isMachineOpcode() &&
8169 (Op1->getMachineOpcode() == SystemZ::LOAD_STACK_GUARD)) {
8170 // Obtain the frame index the store was targeting.
8171 int FI = cast<FrameIndexSDNode>(Val: SN->getOperand(Num: 2))->getIndex();
8172 // Prepare operands of the MOV_STACKGUARD ISD Node - Chain and FrameIndex.
8173 SDValue Ops[] = {SN->getChain(), DAG.getTargetFrameIndex(FI, VT: MVT::i64)};
8174 return DAG.getNode(Opcode: SystemZISD::MOV_STACKGUARD, DL: SDLoc(SN), VT: MVT::Other, Ops);
8175 }
8176
8177 // Combine STORE (BSWAP) into STRVH/STRV/STRVG/VSTBR
8178 if (!SN->isTruncatingStore() &&
8179 Op1.getOpcode() == ISD::BSWAP &&
8180 Op1.getNode()->hasOneUse() &&
8181 canLoadStoreByteSwapped(VT: Op1.getValueType())) {
8182
8183 SDValue BSwapOp = Op1.getOperand(i: 0);
8184
8185 if (BSwapOp.getValueType() == MVT::i16)
8186 BSwapOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT: MVT::i32, Operand: BSwapOp);
8187
8188 SDValue Ops[] = {
8189 N->getOperand(Num: 0), BSwapOp, N->getOperand(Num: 2)
8190 };
8191
8192 return
8193 DAG.getMemIntrinsicNode(Opcode: SystemZISD::STRV, dl: SDLoc(N), VTList: DAG.getVTList(VT: MVT::Other),
8194 Ops, MemVT, MMO: SN->getMemOperand());
8195 }
8196 // Combine STORE (element-swap) into VSTER
8197 if (!SN->isTruncatingStore() &&
8198 Op1.getOpcode() == ISD::VECTOR_SHUFFLE &&
8199 Op1.getNode()->hasOneUse() &&
8200 Subtarget.hasVectorEnhancements2()) {
8201 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op1.getNode());
8202 ArrayRef<int> ShuffleMask = SVN->getMask();
8203 if (isVectorElementSwap(M: ShuffleMask, VT: Op1.getValueType())) {
8204 SDValue Ops[] = {
8205 N->getOperand(Num: 0), Op1.getOperand(i: 0), N->getOperand(Num: 2)
8206 };
8207
8208 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::VSTER, dl: SDLoc(N),
8209 VTList: DAG.getVTList(VT: MVT::Other),
8210 Ops, MemVT, MMO: SN->getMemOperand());
8211 }
8212 }
8213
8214 // Combine STORE (READCYCLECOUNTER) into STCKF.
8215 if (!SN->isTruncatingStore() &&
8216 Op1.getOpcode() == ISD::READCYCLECOUNTER &&
8217 Op1.hasOneUse() &&
8218 N->getOperand(Num: 0).reachesChainWithoutSideEffects(Dest: SDValue(Op1.getNode(), 1))) {
8219 SDValue Ops[] = { Op1.getOperand(i: 0), N->getOperand(Num: 2) };
8220 return DAG.getMemIntrinsicNode(Opcode: SystemZISD::STCKF, dl: SDLoc(N),
8221 VTList: DAG.getVTList(VT: MVT::Other),
8222 Ops, MemVT, MMO: SN->getMemOperand());
8223 }
8224
8225 // Transform a store of a 128-bit value moved from parts into two stores.
8226 if (SN->isSimple() && ISD::isNormalStore(N: SN)) {
8227 SDValue LoPart, HiPart;
8228 if ((MemVT == MVT::i128 && isI128MovedFromParts(Val: Op1, LoPart, HiPart)) ||
8229 (MemVT == MVT::f128 && isF128MovedFromParts(Val: Op1, LoPart, HiPart))) {
8230 SDLoc DL(SN);
8231 SDValue Chain0 = DAG.getStore(
8232 Chain: SN->getChain(), dl: DL, Val: HiPart, Ptr: SN->getBasePtr(), PtrInfo: SN->getPointerInfo(),
8233 Alignment: SN->getBaseAlign(), MMOFlags: SN->getMemOperand()->getFlags(), AAInfo: SN->getAAInfo());
8234 SDValue Chain1 = DAG.getStore(
8235 Chain: SN->getChain(), dl: DL, Val: LoPart,
8236 Ptr: DAG.getObjectPtrOffset(SL: DL, Ptr: SN->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: 8)),
8237 PtrInfo: SN->getPointerInfo().getWithOffset(O: 8), Alignment: SN->getBaseAlign(),
8238 MMOFlags: SN->getMemOperand()->getFlags(), AAInfo: SN->getAAInfo());
8239
8240 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Chain0, N2: Chain1);
8241 }
8242 }
8243
8244 // Replicate a reg or immediate with VREP instead of scalar multiply or
8245 // immediate load. It seems best to do this during the first DAGCombine as
8246 // it is straight-forward to handle the zero-extend node in the initial
8247 // DAG, and also not worry about the keeping the new MemVT legal (e.g. when
8248 // extracting an i16 element from a v16i8 vector).
8249 if (Subtarget.hasVector() && DCI.Level == BeforeLegalizeTypes &&
8250 isOnlyUsedByStores(StoredVal: Op1, DAG)) {
8251 SDValue Word = SDValue();
8252 EVT WordVT;
8253
8254 // Find a replicated immediate and return it if found in Word and its
8255 // type in WordVT.
8256 auto FindReplicatedImm = [&](ConstantSDNode *C, unsigned TotBytes) {
8257 // Some constants are better handled with a scalar store.
8258 if (C->getAPIntValue().getBitWidth() > 64 || C->isAllOnes() ||
8259 isInt<16>(x: C->getSExtValue()) || MemVT.getStoreSize() <= 2)
8260 return;
8261
8262 APInt Val = C->getAPIntValue();
8263 // Truncate Val in case of a truncating store.
8264 if (!llvm::isUIntN(N: TotBytes * 8, x: Val.getZExtValue())) {
8265 assert(SN->isTruncatingStore() &&
8266 "Non-truncating store and immediate value does not fit?");
8267 Val = Val.trunc(width: TotBytes * 8);
8268 }
8269
8270 SystemZVectorConstantInfo VCI(APInt(TotBytes * 8, Val.getZExtValue()));
8271 if (VCI.isVectorConstantLegal(Subtarget) &&
8272 VCI.Opcode == SystemZISD::REPLICATE) {
8273 Word = DAG.getConstant(Val: VCI.OpVals[0], DL: SDLoc(SN), VT: MVT::i32);
8274 WordVT = VCI.VecVT.getScalarType();
8275 }
8276 };
8277
8278 // Find a replicated register and return it if found in Word and its type
8279 // in WordVT.
8280 auto FindReplicatedReg = [&](SDValue MulOp) {
8281 EVT MulVT = MulOp.getValueType();
8282 if (MulOp->getOpcode() == ISD::MUL &&
8283 (MulVT == MVT::i16 || MulVT == MVT::i32 || MulVT == MVT::i64)) {
8284 // Find a zero extended value and its type.
8285 SDValue LHS = MulOp->getOperand(Num: 0);
8286 if (LHS->getOpcode() == ISD::ZERO_EXTEND)
8287 WordVT = LHS->getOperand(Num: 0).getValueType();
8288 else if (LHS->getOpcode() == ISD::AssertZext)
8289 WordVT = cast<VTSDNode>(Val: LHS->getOperand(Num: 1))->getVT();
8290 else
8291 return;
8292 // Find a replicating constant, e.g. 0x00010001.
8293 if (auto *C = dyn_cast<ConstantSDNode>(Val: MulOp->getOperand(Num: 1))) {
8294 SystemZVectorConstantInfo VCI(
8295 APInt(MulVT.getSizeInBits(), C->getZExtValue()));
8296 if (VCI.isVectorConstantLegal(Subtarget) &&
8297 VCI.Opcode == SystemZISD::REPLICATE && VCI.OpVals[0] == 1 &&
8298 WordVT == VCI.VecVT.getScalarType())
8299 Word = DAG.getZExtOrTrunc(Op: LHS->getOperand(Num: 0), DL: SDLoc(SN), VT: WordVT);
8300 }
8301 }
8302 };
8303
8304 if (isa<BuildVectorSDNode>(Val: Op1) &&
8305 DAG.isSplatValue(V: Op1, AllowUndefs: true/*AllowUndefs*/)) {
8306 SDValue SplatVal = Op1->getOperand(Num: 0);
8307 if (auto *C = dyn_cast<ConstantSDNode>(Val&: SplatVal))
8308 FindReplicatedImm(C, SplatVal.getValueType().getStoreSize());
8309 else
8310 FindReplicatedReg(SplatVal);
8311 } else {
8312 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op1))
8313 FindReplicatedImm(C, MemVT.getStoreSize());
8314 else
8315 FindReplicatedReg(Op1);
8316 }
8317
8318 if (Word != SDValue()) {
8319 assert(MemVT.getSizeInBits() % WordVT.getSizeInBits() == 0 &&
8320 "Bad type handling");
8321 unsigned NumElts = MemVT.getSizeInBits() / WordVT.getSizeInBits();
8322 EVT SplatVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WordVT, NumElements: NumElts);
8323 SDValue SplatVal = DAG.getSplatVector(VT: SplatVT, DL: SDLoc(SN), Op: Word);
8324 return DAG.getStore(Chain: SN->getChain(), dl: SDLoc(SN), Val: SplatVal,
8325 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
8326 }
8327 }
8328
8329 return SDValue();
8330}
8331
8332SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
8333 SDNode *N, DAGCombinerInfo &DCI) const {
8334 SelectionDAG &DAG = DCI.DAG;
8335 // Combine element-swap (LOAD) into VLER
8336 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8337 N->getOperand(Num: 0).hasOneUse() &&
8338 Subtarget.hasVectorEnhancements2()) {
8339 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
8340 ArrayRef<int> ShuffleMask = SVN->getMask();
8341 if (isVectorElementSwap(M: ShuffleMask, VT: N->getValueType(ResNo: 0))) {
8342 SDValue Load = N->getOperand(Num: 0);
8343 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8344
8345 // Create the element-swapping load.
8346 SDValue Ops[] = {
8347 LD->getChain(), // Chain
8348 LD->getBasePtr() // Ptr
8349 };
8350 SDValue ESLoad =
8351 DAG.getMemIntrinsicNode(Opcode: SystemZISD::VLER, dl: SDLoc(N),
8352 VTList: DAG.getVTList(VT1: LD->getValueType(ResNo: 0), VT2: MVT::Other),
8353 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8354
8355 // First, combine the VECTOR_SHUFFLE away. This makes the value produced
8356 // by the load dead.
8357 DCI.CombineTo(N, Res: ESLoad);
8358
8359 // Next, combine the load away, we give it a bogus result value but a real
8360 // chain result. The result value is dead because the shuffle is dead.
8361 DCI.CombineTo(N: Load.getNode(), Res0: ESLoad, Res1: ESLoad.getValue(R: 1));
8362
8363 // Return N so it doesn't get rechecked!
8364 return SDValue(N, 0);
8365 }
8366 }
8367
8368 return SDValue();
8369}
8370
8371SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
8372 SDNode *N, DAGCombinerInfo &DCI) const {
8373 SelectionDAG &DAG = DCI.DAG;
8374
8375 if (!Subtarget.hasVector())
8376 return SDValue();
8377
8378 // Look through bitcasts that retain the number of vector elements.
8379 SDValue Op = N->getOperand(Num: 0);
8380 if (Op.getOpcode() == ISD::BITCAST &&
8381 Op.getValueType().isVector() &&
8382 Op.getOperand(i: 0).getValueType().isVector() &&
8383 Op.getValueType().getVectorNumElements() ==
8384 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8385 Op = Op.getOperand(i: 0);
8386
8387 // Pull BSWAP out of a vector extraction.
8388 if (Op.getOpcode() == ISD::BSWAP && Op.hasOneUse()) {
8389 EVT VecVT = Op.getValueType();
8390 EVT EltVT = VecVT.getVectorElementType();
8391 Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N), VT: EltVT,
8392 N1: Op.getOperand(i: 0), N2: N->getOperand(Num: 1));
8393 DCI.AddToWorklist(N: Op.getNode());
8394 Op = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Op);
8395 if (EltVT != N->getValueType(ResNo: 0)) {
8396 DCI.AddToWorklist(N: Op.getNode());
8397 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
8398 }
8399 return Op;
8400 }
8401
8402 // Try to simplify a vector extraction.
8403 if (auto *IndexN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
8404 SDValue Op0 = N->getOperand(Num: 0);
8405 EVT VecVT = Op0.getValueType();
8406 if (canTreatAsByteVector(VT: VecVT))
8407 return combineExtract(DL: SDLoc(N), ResVT: N->getValueType(ResNo: 0), VecVT, Op: Op0,
8408 Index: IndexN->getZExtValue(), DCI, Force: false);
8409 }
8410 return SDValue();
8411}
8412
8413SDValue SystemZTargetLowering::combineJOIN_DWORDS(
8414 SDNode *N, DAGCombinerInfo &DCI) const {
8415 SelectionDAG &DAG = DCI.DAG;
8416 // (join_dwords X, X) == (replicate X)
8417 if (N->getOperand(Num: 0) == N->getOperand(Num: 1))
8418 return DAG.getNode(Opcode: SystemZISD::REPLICATE, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
8419 Operand: N->getOperand(Num: 0));
8420 return SDValue();
8421}
8422
8423static SDValue MergeInputChains(SDNode *N1, SDNode *N2) {
8424 SDValue Chain1 = N1->getOperand(Num: 0);
8425 SDValue Chain2 = N2->getOperand(Num: 0);
8426
8427 // Trivial case: both nodes take the same chain.
8428 if (Chain1 == Chain2)
8429 return Chain1;
8430
8431 // FIXME - we could handle more complex cases via TokenFactor,
8432 // assuming we can verify that this would not create a cycle.
8433 return SDValue();
8434}
8435
8436SDValue SystemZTargetLowering::combineFP_ROUND(
8437 SDNode *N, DAGCombinerInfo &DCI) const {
8438
8439 if (!Subtarget.hasVector())
8440 return SDValue();
8441
8442 // (fpround (extract_vector_elt X 0))
8443 // (fpround (extract_vector_elt X 1)) ->
8444 // (extract_vector_elt (VROUND X) 0)
8445 // (extract_vector_elt (VROUND X) 2)
8446 //
8447 // This is a special case since the target doesn't really support v2f32s.
8448 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8449 SelectionDAG &DAG = DCI.DAG;
8450 SDValue Op0 = N->getOperand(Num: OpNo);
8451 if (N->getValueType(ResNo: 0) == MVT::f32 && Op0.hasOneUse() &&
8452 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8453 Op0.getOperand(i: 0).getValueType() == MVT::v2f64 &&
8454 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8455 Op0.getConstantOperandVal(i: 1) == 0) {
8456 SDValue Vec = Op0.getOperand(i: 0);
8457 for (auto *U : Vec->users()) {
8458 if (U != Op0.getNode() && U->hasOneUse() &&
8459 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8460 U->getOperand(Num: 0) == Vec &&
8461 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8462 U->getConstantOperandVal(Num: 1) == 1) {
8463 SDValue OtherRound = SDValue(*U->user_begin(), 0);
8464 if (OtherRound.getOpcode() == N->getOpcode() &&
8465 OtherRound.getOperand(i: OpNo) == SDValue(U, 0) &&
8466 OtherRound.getValueType() == MVT::f32) {
8467 SDValue VRound, Chain;
8468 if (N->isStrictFPOpcode()) {
8469 Chain = MergeInputChains(N1: N, N2: OtherRound.getNode());
8470 if (!Chain)
8471 continue;
8472 VRound = DAG.getNode(Opcode: SystemZISD::STRICT_VROUND, DL: SDLoc(N),
8473 ResultTys: {MVT::v4f32, MVT::Other}, Ops: {Chain, Vec});
8474 Chain = VRound.getValue(R: 1);
8475 } else
8476 VRound = DAG.getNode(Opcode: SystemZISD::VROUND, DL: SDLoc(N),
8477 VT: MVT::v4f32, Operand: Vec);
8478 DCI.AddToWorklist(N: VRound.getNode());
8479 SDValue Extract1 =
8480 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f32,
8481 N1: VRound, N2: DAG.getConstant(Val: 2, DL: SDLoc(U), VT: MVT::i32));
8482 DCI.AddToWorklist(N: Extract1.getNode());
8483 DAG.ReplaceAllUsesOfValueWith(From: OtherRound, To: Extract1);
8484 if (Chain)
8485 DAG.ReplaceAllUsesOfValueWith(From: OtherRound.getValue(R: 1), To: Chain);
8486 SDValue Extract0 =
8487 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f32,
8488 N1: VRound, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8489 if (Chain)
8490 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8491 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8492 return Extract0;
8493 }
8494 }
8495 }
8496 }
8497 return SDValue();
8498}
8499
8500SDValue SystemZTargetLowering::combineFP_EXTEND(
8501 SDNode *N, DAGCombinerInfo &DCI) const {
8502
8503 if (!Subtarget.hasVector())
8504 return SDValue();
8505
8506 // (fpextend (extract_vector_elt X 0))
8507 // (fpextend (extract_vector_elt X 2)) ->
8508 // (extract_vector_elt (VEXTEND X) 0)
8509 // (extract_vector_elt (VEXTEND X) 1)
8510 //
8511 // This is a special case since the target doesn't really support v2f32s.
8512 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
8513 SelectionDAG &DAG = DCI.DAG;
8514 SDValue Op0 = N->getOperand(Num: OpNo);
8515 if (N->getValueType(ResNo: 0) == MVT::f64 && Op0.hasOneUse() &&
8516 Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8517 Op0.getOperand(i: 0).getValueType() == MVT::v4f32 &&
8518 Op0.getOperand(i: 1).getOpcode() == ISD::Constant &&
8519 Op0.getConstantOperandVal(i: 1) == 0) {
8520 SDValue Vec = Op0.getOperand(i: 0);
8521 for (auto *U : Vec->users()) {
8522 if (U != Op0.getNode() && U->hasOneUse() &&
8523 U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8524 U->getOperand(Num: 0) == Vec &&
8525 U->getOperand(Num: 1).getOpcode() == ISD::Constant &&
8526 U->getConstantOperandVal(Num: 1) == 2) {
8527 SDValue OtherExtend = SDValue(*U->user_begin(), 0);
8528 if (OtherExtend.getOpcode() == N->getOpcode() &&
8529 OtherExtend.getOperand(i: OpNo) == SDValue(U, 0) &&
8530 OtherExtend.getValueType() == MVT::f64) {
8531 SDValue VExtend, Chain;
8532 if (N->isStrictFPOpcode()) {
8533 Chain = MergeInputChains(N1: N, N2: OtherExtend.getNode());
8534 if (!Chain)
8535 continue;
8536 VExtend = DAG.getNode(Opcode: SystemZISD::STRICT_VEXTEND, DL: SDLoc(N),
8537 ResultTys: {MVT::v2f64, MVT::Other}, Ops: {Chain, Vec});
8538 Chain = VExtend.getValue(R: 1);
8539 } else
8540 VExtend = DAG.getNode(Opcode: SystemZISD::VEXTEND, DL: SDLoc(N),
8541 VT: MVT::v2f64, Operand: Vec);
8542 DCI.AddToWorklist(N: VExtend.getNode());
8543 SDValue Extract1 =
8544 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(U), VT: MVT::f64,
8545 N1: VExtend, N2: DAG.getConstant(Val: 1, DL: SDLoc(U), VT: MVT::i32));
8546 DCI.AddToWorklist(N: Extract1.getNode());
8547 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend, To: Extract1);
8548 if (Chain)
8549 DAG.ReplaceAllUsesOfValueWith(From: OtherExtend.getValue(R: 1), To: Chain);
8550 SDValue Extract0 =
8551 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Op0), VT: MVT::f64,
8552 N1: VExtend, N2: DAG.getConstant(Val: 0, DL: SDLoc(Op0), VT: MVT::i32));
8553 if (Chain)
8554 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: SDLoc(Op0),
8555 VTList: N->getVTList(), N1: Extract0, N2: Chain);
8556 return Extract0;
8557 }
8558 }
8559 }
8560 }
8561 return SDValue();
8562}
8563
8564SDValue SystemZTargetLowering::combineINT_TO_FP(
8565 SDNode *N, DAGCombinerInfo &DCI) const {
8566 if (DCI.Level != BeforeLegalizeTypes)
8567 return SDValue();
8568 SelectionDAG &DAG = DCI.DAG;
8569 LLVMContext &Ctx = *DAG.getContext();
8570 unsigned Opcode = N->getOpcode();
8571 EVT OutVT = N->getValueType(ResNo: 0);
8572 Type *OutLLVMTy = OutVT.getTypeForEVT(Context&: Ctx);
8573 SDValue Op = N->getOperand(Num: 0);
8574 unsigned OutScalarBits = OutLLVMTy->getScalarSizeInBits();
8575 unsigned InScalarBits = Op->getValueType(ResNo: 0).getScalarSizeInBits();
8576
8577 // Insert an extension before type-legalization to avoid scalarization, e.g.:
8578 // v2f64 = uint_to_fp v2i16
8579 // =>
8580 // v2f64 = uint_to_fp (v2i64 zero_extend v2i16)
8581 if (OutLLVMTy->isVectorTy() && OutScalarBits > InScalarBits &&
8582 OutScalarBits <= 64) {
8583 unsigned NumElts = cast<FixedVectorType>(Val: OutLLVMTy)->getNumElements();
8584 EVT ExtVT = EVT::getVectorVT(
8585 Context&: Ctx, VT: EVT::getIntegerVT(Context&: Ctx, BitWidth: OutLLVMTy->getScalarSizeInBits()), NumElements: NumElts);
8586 unsigned ExtOpcode =
8587 (Opcode == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND);
8588 SDValue ExtOp = DAG.getNode(Opcode: ExtOpcode, DL: SDLoc(N), VT: ExtVT, Operand: Op);
8589 return DAG.getNode(Opcode, DL: SDLoc(N), VT: OutVT, Operand: ExtOp);
8590 }
8591 return SDValue();
8592}
8593
8594SDValue SystemZTargetLowering::combineFCOPYSIGN(
8595 SDNode *N, DAGCombinerInfo &DCI) const {
8596 SelectionDAG &DAG = DCI.DAG;
8597 EVT VT = N->getValueType(ResNo: 0);
8598 SDValue ValOp = N->getOperand(Num: 0);
8599 SDValue SignOp = N->getOperand(Num: 1);
8600
8601 // Remove the rounding which is not needed.
8602 if (SignOp.getOpcode() == ISD::FP_ROUND) {
8603 SDValue WideOp = SignOp.getOperand(i: 0);
8604 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N), VT, N1: ValOp, N2: WideOp);
8605 }
8606
8607 return SDValue();
8608}
8609
8610SDValue SystemZTargetLowering::combineBSWAP(
8611 SDNode *N, DAGCombinerInfo &DCI) const {
8612 SelectionDAG &DAG = DCI.DAG;
8613 // Combine BSWAP (LOAD) into LRVH/LRV/LRVG/VLBR
8614 if (ISD::isNON_EXTLoad(N: N->getOperand(Num: 0).getNode()) &&
8615 N->getOperand(Num: 0).hasOneUse() &&
8616 canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0))) {
8617 SDValue Load = N->getOperand(Num: 0);
8618 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
8619
8620 // Create the byte-swapping load.
8621 SDValue Ops[] = {
8622 LD->getChain(), // Chain
8623 LD->getBasePtr() // Ptr
8624 };
8625 EVT LoadVT = N->getValueType(ResNo: 0);
8626 if (LoadVT == MVT::i16)
8627 LoadVT = MVT::i32;
8628 SDValue BSLoad =
8629 DAG.getMemIntrinsicNode(Opcode: SystemZISD::LRV, dl: SDLoc(N),
8630 VTList: DAG.getVTList(VT1: LoadVT, VT2: MVT::Other),
8631 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
8632
8633 // If this is an i16 load, insert the truncate.
8634 SDValue ResVal = BSLoad;
8635 if (N->getValueType(ResNo: 0) == MVT::i16)
8636 ResVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: MVT::i16, Operand: BSLoad);
8637
8638 // First, combine the bswap away. This makes the value produced by the
8639 // load dead.
8640 DCI.CombineTo(N, Res: ResVal);
8641
8642 // Next, combine the load away, we give it a bogus result value but a real
8643 // chain result. The result value is dead because the bswap is dead.
8644 DCI.CombineTo(N: Load.getNode(), Res0: ResVal, Res1: BSLoad.getValue(R: 1));
8645
8646 // Return N so it doesn't get rechecked!
8647 return SDValue(N, 0);
8648 }
8649
8650 // Look through bitcasts that retain the number of vector elements.
8651 SDValue Op = N->getOperand(Num: 0);
8652 if (Op.getOpcode() == ISD::BITCAST &&
8653 Op.getValueType().isVector() &&
8654 Op.getOperand(i: 0).getValueType().isVector() &&
8655 Op.getValueType().getVectorNumElements() ==
8656 Op.getOperand(i: 0).getValueType().getVectorNumElements())
8657 Op = Op.getOperand(i: 0);
8658
8659 // Push BSWAP into a vector insertion if at least one side then simplifies.
8660 if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT && Op.hasOneUse()) {
8661 SDValue Vec = Op.getOperand(i: 0);
8662 SDValue Elt = Op.getOperand(i: 1);
8663 SDValue Idx = Op.getOperand(i: 2);
8664
8665 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Vec) ||
8666 Vec.getOpcode() == ISD::BSWAP || Vec.isUndef() ||
8667 DAG.isConstantIntBuildVectorOrConstantInt(N: Elt) ||
8668 Elt.getOpcode() == ISD::BSWAP || Elt.isUndef() ||
8669 (canLoadStoreByteSwapped(VT: N->getValueType(ResNo: 0)) &&
8670 ISD::isNON_EXTLoad(N: Elt.getNode()) && Elt.hasOneUse())) {
8671 EVT VecVT = N->getValueType(ResNo: 0);
8672 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
8673 if (VecVT != Vec.getValueType()) {
8674 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8675 DCI.AddToWorklist(N: Vec.getNode());
8676 }
8677 if (EltVT != Elt.getValueType()) {
8678 Elt = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8679 DCI.AddToWorklist(N: Elt.getNode());
8680 }
8681 Vec = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Vec);
8682 DCI.AddToWorklist(N: Vec.getNode());
8683 Elt = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: EltVT, Operand: Elt);
8684 DCI.AddToWorklist(N: Elt.getNode());
8685 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N), VT: VecVT,
8686 N1: Vec, N2: Elt, N3: Idx);
8687 }
8688 }
8689
8690 // Push BSWAP into a vector shuffle if at least one side then simplifies.
8691 ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(Val&: Op);
8692 if (SV && Op.hasOneUse()) {
8693 SDValue Op0 = Op.getOperand(i: 0);
8694 SDValue Op1 = Op.getOperand(i: 1);
8695
8696 if (DAG.isConstantIntBuildVectorOrConstantInt(N: Op0) ||
8697 Op0.getOpcode() == ISD::BSWAP || Op0.isUndef() ||
8698 DAG.isConstantIntBuildVectorOrConstantInt(N: Op1) ||
8699 Op1.getOpcode() == ISD::BSWAP || Op1.isUndef()) {
8700 EVT VecVT = N->getValueType(ResNo: 0);
8701 if (VecVT != Op0.getValueType()) {
8702 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8703 DCI.AddToWorklist(N: Op0.getNode());
8704 }
8705 if (VecVT != Op1.getValueType()) {
8706 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8707 DCI.AddToWorklist(N: Op1.getNode());
8708 }
8709 Op0 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op0);
8710 DCI.AddToWorklist(N: Op0.getNode());
8711 Op1 = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT: VecVT, Operand: Op1);
8712 DCI.AddToWorklist(N: Op1.getNode());
8713 return DAG.getVectorShuffle(VT: VecVT, dl: SDLoc(N), N1: Op0, N2: Op1, Mask: SV->getMask());
8714 }
8715 }
8716
8717 return SDValue();
8718}
8719
8720SDValue SystemZTargetLowering::combineSETCC(
8721 SDNode *N, DAGCombinerInfo &DCI) const {
8722 SelectionDAG &DAG = DCI.DAG;
8723 const ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
8724 const SDValue LHS = N->getOperand(Num: 0);
8725 const SDValue RHS = N->getOperand(Num: 1);
8726 bool CmpNull = isNullConstant(V: RHS);
8727 bool CmpAllOnes = isAllOnesConstant(V: RHS);
8728 EVT VT = N->getValueType(ResNo: 0);
8729 SDLoc DL(N);
8730
8731 // Match icmp_eq/ne(bitcast(icmp(X,Y)),0/-1) reduction patterns, and
8732 // change the outer compare to a i128 compare. This will normally
8733 // allow the reduction to be recognized in adjustICmp128, and even if
8734 // not, the i128 compare will still generate better code.
8735 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && (CmpNull || CmpAllOnes)) {
8736 SDValue Src = peekThroughBitcasts(V: LHS);
8737 if (Src.getOpcode() == ISD::SETCC &&
8738 Src.getValueType().isFixedLengthVector() &&
8739 Src.getValueType().getScalarType() == MVT::i1) {
8740 EVT CmpVT = Src.getOperand(i: 0).getValueType();
8741 if (CmpVT.getSizeInBits() == 128) {
8742 EVT IntVT = CmpVT.changeVectorElementTypeToInteger();
8743 SDValue LHS =
8744 DAG.getBitcast(VT: MVT::i128, V: DAG.getSExtOrTrunc(Op: Src, DL, VT: IntVT));
8745 SDValue RHS = CmpNull ? DAG.getConstant(Val: 0, DL, VT: MVT::i128)
8746 : DAG.getAllOnesConstant(DL, VT: MVT::i128);
8747 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: LHS, N2: RHS, N3: N->getOperand(Num: 2),
8748 Flags: N->getFlags());
8749 }
8750 }
8751 }
8752
8753 return SDValue();
8754}
8755
8756static std::pair<SDValue, int> findCCUse(const SDValue &Val,
8757 unsigned Depth = 0) {
8758 // Limit depth of potentially exponential walk.
8759 if (Depth > 5)
8760 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8761
8762 switch (Val.getOpcode()) {
8763 default:
8764 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8765 case SystemZISD::IPM:
8766 if (Val.getOperand(i: 0).getOpcode() == SystemZISD::CLC ||
8767 Val.getOperand(i: 0).getOpcode() == SystemZISD::STRCMP)
8768 return std::make_pair(x: Val.getOperand(i: 0), y: SystemZ::CCMASK_ICMP);
8769 return std::make_pair(x: Val.getOperand(i: 0), y: SystemZ::CCMASK_ANY);
8770 case SystemZISD::SELECT_CCMASK: {
8771 SDValue Op4CCReg = Val.getOperand(i: 4);
8772 if (Op4CCReg.getOpcode() == SystemZISD::ICMP ||
8773 Op4CCReg.getOpcode() == SystemZISD::TM) {
8774 auto [OpCC, OpCCValid] = findCCUse(Val: Op4CCReg.getOperand(i: 0), Depth: Depth + 1);
8775 if (OpCC != SDValue())
8776 return std::make_pair(x&: OpCC, y&: OpCCValid);
8777 }
8778 auto *CCValid = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 2));
8779 if (!CCValid)
8780 return std::make_pair(x: SDValue(), y: SystemZ::CCMASK_NONE);
8781 int CCValidVal = CCValid->getZExtValue();
8782 return std::make_pair(x&: Op4CCReg, y&: CCValidVal);
8783 }
8784 case ISD::ADD:
8785 case ISD::AND:
8786 case ISD::OR:
8787 case ISD::XOR:
8788 case ISD::SHL:
8789 case ISD::SRA:
8790 case ISD::SRL:
8791 auto [Op0CC, Op0CCValid] = findCCUse(Val: Val.getOperand(i: 0), Depth: Depth + 1);
8792 if (Op0CC != SDValue())
8793 return std::make_pair(x&: Op0CC, y&: Op0CCValid);
8794 return findCCUse(Val: Val.getOperand(i: 1), Depth: Depth + 1);
8795 }
8796}
8797
8798static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
8799 SelectionDAG &DAG);
8800
8801SmallVector<SDValue, 4> static simplifyAssumingCCVal(SDValue &Val, SDValue &CC,
8802 SelectionDAG &DAG) {
8803 SDLoc DL(Val);
8804 auto Opcode = Val.getOpcode();
8805 switch (Opcode) {
8806 default:
8807 return {};
8808 case ISD::Constant:
8809 return {Val, Val, Val, Val};
8810 case SystemZISD::IPM: {
8811 SDValue IPMOp0 = Val.getOperand(i: 0);
8812 if (IPMOp0 != CC)
8813 return {};
8814 SmallVector<SDValue, 4> ShiftedCCVals;
8815 for (auto CC : {0, 1, 2, 3})
8816 ShiftedCCVals.emplace_back(
8817 Args: DAG.getConstant(Val: (CC << SystemZ::IPM_CC), DL, VT: MVT::i32));
8818 return ShiftedCCVals;
8819 }
8820 case SystemZISD::SELECT_CCMASK: {
8821 SDValue TrueVal = Val.getOperand(i: 0), FalseVal = Val.getOperand(i: 1);
8822 auto *CCValid = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 2));
8823 auto *CCMask = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 3));
8824 if (!CCValid || !CCMask)
8825 return {};
8826
8827 int CCValidVal = CCValid->getZExtValue();
8828 int CCMaskVal = CCMask->getZExtValue();
8829 // Pruning search tree early - Moving CC test and combineCCMask ahead of
8830 // recursive call to simplifyAssumingCCVal.
8831 SDValue Op4CCReg = Val.getOperand(i: 4);
8832 if (Op4CCReg != CC)
8833 combineCCMask(CCReg&: Op4CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG);
8834 if (Op4CCReg != CC)
8835 return {};
8836 const auto &&TrueSDVals = simplifyAssumingCCVal(Val&: TrueVal, CC, DAG);
8837 const auto &&FalseSDVals = simplifyAssumingCCVal(Val&: FalseVal, CC, DAG);
8838 if (TrueSDVals.empty() || FalseSDVals.empty())
8839 return {};
8840 SmallVector<SDValue, 4> MergedSDVals;
8841 for (auto &CCVal : {0, 1, 2, 3})
8842 MergedSDVals.emplace_back(Args: ((CCMaskVal & (1 << (3 - CCVal))) != 0)
8843 ? TrueSDVals[CCVal]
8844 : FalseSDVals[CCVal]);
8845 return MergedSDVals;
8846 }
8847 case ISD::ADD:
8848 case ISD::AND:
8849 case ISD::OR:
8850 case ISD::XOR:
8851 case ISD::SRA:
8852 // Avoid introducing CC spills (because ADD/AND/OR/XOR/SRA
8853 // would clobber CC).
8854 if (!Val.hasOneUse())
8855 return {};
8856 [[fallthrough]];
8857 case ISD::SHL:
8858 case ISD::SRL:
8859 SDValue Op0 = Val.getOperand(i: 0), Op1 = Val.getOperand(i: 1);
8860 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: Op0, CC, DAG);
8861 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: Op1, CC, DAG);
8862 if (Op0SDVals.empty() || Op1SDVals.empty())
8863 return {};
8864 SmallVector<SDValue, 4> BinaryOpSDVals;
8865 for (auto CCVal : {0, 1, 2, 3})
8866 BinaryOpSDVals.emplace_back(Args: DAG.getNode(
8867 Opcode, DL, VT: Val.getValueType(), N1: Op0SDVals[CCVal], N2: Op1SDVals[CCVal]));
8868 return BinaryOpSDVals;
8869 }
8870}
8871
8872static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask,
8873 SelectionDAG &DAG) {
8874 // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
8875 // set by the CCReg instruction using the CCValid / CCMask masks,
8876 // If the CCReg instruction is itself a ICMP / TM testing the condition
8877 // code set by some other instruction, see whether we can directly
8878 // use that condition code.
8879 auto *CCNode = CCReg.getNode();
8880 if (!CCNode)
8881 return false;
8882
8883 if (CCNode->getOpcode() == SystemZISD::TM) {
8884 if (CCValid != SystemZ::CCMASK_TM)
8885 return false;
8886 auto emulateTMCCMask = [](const SDValue &Op0Val, const SDValue &Op1Val) {
8887 auto *Op0Node = dyn_cast<ConstantSDNode>(Val: Op0Val.getNode());
8888 auto *Op1Node = dyn_cast<ConstantSDNode>(Val: Op1Val.getNode());
8889 if (!Op0Node || !Op1Node)
8890 return -1;
8891 auto Op0APVal = Op0Node->getAPIntValue();
8892 auto Op1APVal = Op1Node->getAPIntValue();
8893 auto Result = Op0APVal & Op1APVal;
8894 bool AllOnes = Result == Op1APVal;
8895 bool AllZeros = Result == 0;
8896 bool IsLeftMostBitSet = Result[Op1APVal.getActiveBits() - 1] != 0;
8897 return AllZeros ? 0 : AllOnes ? 3 : IsLeftMostBitSet ? 2 : 1;
8898 };
8899 SDValue Op0 = CCNode->getOperand(Num: 0);
8900 SDValue Op1 = CCNode->getOperand(Num: 1);
8901 auto [Op0CC, Op0CCValid] = findCCUse(Val: Op0);
8902 if (Op0CC == SDValue())
8903 return false;
8904 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: Op0, CC&: Op0CC, DAG);
8905 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: Op1, CC&: Op0CC, DAG);
8906 if (Op0SDVals.empty() || Op1SDVals.empty())
8907 return false;
8908 int NewCCMask = 0;
8909 for (auto CC : {0, 1, 2, 3}) {
8910 auto CCVal = emulateTMCCMask(Op0SDVals[CC], Op1SDVals[CC]);
8911 if (CCVal < 0)
8912 return false;
8913 NewCCMask <<= 1;
8914 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
8915 }
8916 NewCCMask &= Op0CCValid;
8917 CCReg = Op0CC;
8918 CCMask = NewCCMask;
8919 CCValid = Op0CCValid;
8920 return true;
8921 }
8922 if (CCNode->getOpcode() != SystemZISD::ICMP ||
8923 CCValid != SystemZ::CCMASK_ICMP)
8924 return false;
8925
8926 SDValue CmpOp0 = CCNode->getOperand(Num: 0);
8927 SDValue CmpOp1 = CCNode->getOperand(Num: 1);
8928 SDValue CmpOp2 = CCNode->getOperand(Num: 2);
8929 auto [Op0CC, Op0CCValid] = findCCUse(Val: CmpOp0);
8930 if (Op0CC != SDValue()) {
8931 const auto &&Op0SDVals = simplifyAssumingCCVal(Val&: CmpOp0, CC&: Op0CC, DAG);
8932 const auto &&Op1SDVals = simplifyAssumingCCVal(Val&: CmpOp1, CC&: Op0CC, DAG);
8933 if (Op0SDVals.empty() || Op1SDVals.empty())
8934 return false;
8935
8936 auto *CmpType = dyn_cast<ConstantSDNode>(Val&: CmpOp2);
8937 auto CmpTypeVal = CmpType->getZExtValue();
8938 const auto compareCCSigned = [&CmpTypeVal](const SDValue &Op0Val,
8939 const SDValue &Op1Val) {
8940 auto *Op0Node = dyn_cast<ConstantSDNode>(Val: Op0Val.getNode());
8941 auto *Op1Node = dyn_cast<ConstantSDNode>(Val: Op1Val.getNode());
8942 if (!Op0Node || !Op1Node)
8943 return -1;
8944 auto Op0APVal = Op0Node->getAPIntValue();
8945 auto Op1APVal = Op1Node->getAPIntValue();
8946 if (CmpTypeVal == SystemZICMP::SignedOnly)
8947 return Op0APVal == Op1APVal ? 0 : Op0APVal.slt(RHS: Op1APVal) ? 1 : 2;
8948 return Op0APVal == Op1APVal ? 0 : Op0APVal.ult(RHS: Op1APVal) ? 1 : 2;
8949 };
8950 int NewCCMask = 0;
8951 for (auto CC : {0, 1, 2, 3}) {
8952 auto CCVal = compareCCSigned(Op0SDVals[CC], Op1SDVals[CC]);
8953 if (CCVal < 0)
8954 return false;
8955 NewCCMask <<= 1;
8956 NewCCMask |= (CCMask & (1 << (3 - CCVal))) != 0;
8957 }
8958 NewCCMask &= Op0CCValid;
8959 CCMask = NewCCMask;
8960 CCReg = Op0CC;
8961 CCValid = Op0CCValid;
8962 return true;
8963 }
8964
8965 return false;
8966}
8967
8968// Merging versus split in multiple branches cost.
8969TargetLoweringBase::CondMergingParams
8970SystemZTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
8971 const Value *Lhs,
8972 const Value *Rhs,
8973 const Function *) const {
8974 const auto isFlagOutOpCC = [](const Value *V) {
8975 using namespace llvm::PatternMatch;
8976 const Value *RHSVal;
8977 const APInt *RHSC;
8978 if (const auto *I = dyn_cast<Instruction>(Val: V)) {
8979 // PatternMatch.h provides concise tree-based pattern match of llvm IR.
8980 if (match(V: I->getOperand(i: 0), P: m_And(L: m_Value(V&: RHSVal), R: m_APInt(Res&: RHSC))) ||
8981 match(V: I, P: m_Cmp(L: m_Value(V&: RHSVal), R: m_APInt(Res&: RHSC)))) {
8982 if (const auto *CB = dyn_cast<CallBase>(Val: RHSVal)) {
8983 if (CB->isInlineAsm()) {
8984 const InlineAsm *IA = cast<InlineAsm>(Val: CB->getCalledOperand());
8985 return IA && IA->getConstraintString().contains(Other: "{@cc}");
8986 }
8987 }
8988 }
8989 }
8990 return false;
8991 };
8992 // Pattern (ICmp %asm) or (ICmp (And %asm)).
8993 // Cost of longest dependency chain (ICmp, And) is 2. CostThreshold or
8994 // BaseCost can be set >=2. If cost of instruction <= CostThreshold
8995 // conditionals will be merged or else conditionals will be split.
8996 if (isFlagOutOpCC(Lhs) && isFlagOutOpCC(Rhs))
8997 return {.BaseCost: 3, .LikelyBias: 0, .UnlikelyBias: -1};
8998 // Default.
8999 return {.BaseCost: -1, .LikelyBias: -1, .UnlikelyBias: -1};
9000}
9001
9002SDValue SystemZTargetLowering::combineBR_CCMASK(SDNode *N,
9003 DAGCombinerInfo &DCI) const {
9004 SelectionDAG &DAG = DCI.DAG;
9005
9006 // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK.
9007 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
9008 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9009 if (!CCValid || !CCMask)
9010 return SDValue();
9011
9012 int CCValidVal = CCValid->getZExtValue();
9013 int CCMaskVal = CCMask->getZExtValue();
9014 SDValue Chain = N->getOperand(Num: 0);
9015 SDValue CCReg = N->getOperand(Num: 4);
9016 // If combineCMask was able to merge or simplify ccvalid or ccmask, re-emit
9017 // the modified BR_CCMASK with the new values.
9018 // In order to avoid conditional branches with full or empty cc masks, do not
9019 // do this if ccmask is 0 or equal to ccvalid.
9020 if (combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG) && CCMaskVal != 0 &&
9021 CCMaskVal != CCValidVal)
9022 return DAG.getNode(Opcode: SystemZISD::BR_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
9023 N1: Chain,
9024 N2: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
9025 N3: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32),
9026 N4: N->getOperand(Num: 3), N5: CCReg);
9027 return SDValue();
9028}
9029
9030SDValue SystemZTargetLowering::combineSELECT_CCMASK(
9031 SDNode *N, DAGCombinerInfo &DCI) const {
9032 SelectionDAG &DAG = DCI.DAG;
9033
9034 // Combine SELECT_CCMASK (ICMP (SELECT_CCMASK)) into a single SELECT_CCMASK.
9035 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9036 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3));
9037 if (!CCValid || !CCMask)
9038 return SDValue();
9039
9040 int CCValidVal = CCValid->getZExtValue();
9041 int CCMaskVal = CCMask->getZExtValue();
9042 SDValue CCReg = N->getOperand(Num: 4);
9043
9044 bool IsCombinedCCReg = combineCCMask(CCReg, CCValid&: CCValidVal, CCMask&: CCMaskVal, DAG);
9045
9046 // Populate SDVals vector for each condition code ccval for given Val, which
9047 // can again be another nested select_ccmask with the same CC.
9048 const auto constructCCSDValsFromSELECT = [&CCReg](SDValue &Val) {
9049 if (Val.getOpcode() == SystemZISD::SELECT_CCMASK) {
9050 SmallVector<SDValue, 4> Res;
9051 if (Val.getOperand(i: 4) != CCReg)
9052 return SmallVector<SDValue, 4>{};
9053 SDValue TrueVal = Val.getOperand(i: 0), FalseVal = Val.getOperand(i: 1);
9054 auto *CCMask = dyn_cast<ConstantSDNode>(Val: Val.getOperand(i: 3));
9055 if (!CCMask)
9056 return SmallVector<SDValue, 4>{};
9057
9058 int CCMaskVal = CCMask->getZExtValue();
9059 for (auto &CC : {0, 1, 2, 3})
9060 Res.emplace_back(Args&: ((CCMaskVal & (1 << (3 - CC))) != 0) ? TrueVal
9061 : FalseVal);
9062 return Res;
9063 }
9064 return SmallVector<SDValue, 4>{Val, Val, Val, Val};
9065 };
9066 // Attempting to optimize TrueVal/FalseVal in outermost select_ccmask either
9067 // with CCReg found by combineCCMask or original CCReg.
9068 SDValue TrueVal = N->getOperand(Num: 0);
9069 SDValue FalseVal = N->getOperand(Num: 1);
9070 auto &&TrueSDVals = simplifyAssumingCCVal(Val&: TrueVal, CC&: CCReg, DAG);
9071 auto &&FalseSDVals = simplifyAssumingCCVal(Val&: FalseVal, CC&: CCReg, DAG);
9072 // TrueSDVals/FalseSDVals might be empty in case of non-constant
9073 // TrueVal/FalseVal for select_ccmask, which can not be optimized further.
9074 if (TrueSDVals.empty())
9075 TrueSDVals = constructCCSDValsFromSELECT(TrueVal);
9076 if (FalseSDVals.empty())
9077 FalseSDVals = constructCCSDValsFromSELECT(FalseVal);
9078 if (!TrueSDVals.empty() && !FalseSDVals.empty()) {
9079 SmallSet<SDValue, 4> MergedSDValsSet;
9080 // Ignoring CC values outside CCValiid.
9081 for (auto CC : {0, 1, 2, 3}) {
9082 if ((CCValidVal & ((1 << (3 - CC)))) != 0)
9083 MergedSDValsSet.insert(V: ((CCMaskVal & (1 << (3 - CC))) != 0)
9084 ? TrueSDVals[CC]
9085 : FalseSDVals[CC]);
9086 }
9087 if (MergedSDValsSet.size() == 1)
9088 return *MergedSDValsSet.begin();
9089 if (MergedSDValsSet.size() == 2) {
9090 auto BeginIt = MergedSDValsSet.begin();
9091 SDValue NewTrueVal = *BeginIt, NewFalseVal = *next(x: BeginIt);
9092 if (NewTrueVal == FalseVal || NewFalseVal == TrueVal)
9093 std::swap(a&: NewTrueVal, b&: NewFalseVal);
9094 int NewCCMask = 0;
9095 for (auto CC : {0, 1, 2, 3}) {
9096 NewCCMask <<= 1;
9097 NewCCMask |= ((CCMaskVal & (1 << (3 - CC))) != 0)
9098 ? (TrueSDVals[CC] == NewTrueVal)
9099 : (FalseSDVals[CC] == NewTrueVal);
9100 }
9101 CCMaskVal = NewCCMask;
9102 CCMaskVal &= CCValidVal;
9103 TrueVal = NewTrueVal;
9104 FalseVal = NewFalseVal;
9105 IsCombinedCCReg = true;
9106 }
9107 }
9108 // If the condition is trivially false or trivially true after
9109 // combineCCMask, just collapse this SELECT_CCMASK to the indicated value
9110 // (possibly modified by constructCCSDValsFromSELECT).
9111 if (CCMaskVal == 0)
9112 return FalseVal;
9113 if (CCMaskVal == CCValidVal)
9114 return TrueVal;
9115
9116 if (IsCombinedCCReg)
9117 return DAG.getNode(
9118 Opcode: SystemZISD::SELECT_CCMASK, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: TrueVal,
9119 N2: FalseVal, N3: DAG.getTargetConstant(Val: CCValidVal, DL: SDLoc(N), VT: MVT::i32),
9120 N4: DAG.getTargetConstant(Val: CCMaskVal, DL: SDLoc(N), VT: MVT::i32), N5: CCReg);
9121
9122 return SDValue();
9123}
9124
9125SDValue SystemZTargetLowering::combineGET_CCMASK(
9126 SDNode *N, DAGCombinerInfo &DCI) const {
9127
9128 // Optimize away GET_CCMASK (SELECT_CCMASK) if the CC masks are compatible
9129 auto *CCValid = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
9130 auto *CCMask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
9131 if (!CCValid || !CCMask)
9132 return SDValue();
9133 int CCValidVal = CCValid->getZExtValue();
9134 int CCMaskVal = CCMask->getZExtValue();
9135
9136 SDValue Select = N->getOperand(Num: 0);
9137 if (Select->getOpcode() == ISD::TRUNCATE)
9138 Select = Select->getOperand(Num: 0);
9139 if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
9140 return SDValue();
9141
9142 auto *SelectCCValid = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 2));
9143 auto *SelectCCMask = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 3));
9144 if (!SelectCCValid || !SelectCCMask)
9145 return SDValue();
9146 int SelectCCValidVal = SelectCCValid->getZExtValue();
9147 int SelectCCMaskVal = SelectCCMask->getZExtValue();
9148
9149 auto *TrueVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 0));
9150 auto *FalseVal = dyn_cast<ConstantSDNode>(Val: Select->getOperand(Num: 1));
9151 if (!TrueVal || !FalseVal)
9152 return SDValue();
9153 if (TrueVal->getZExtValue() == 1 && FalseVal->getZExtValue() == 0)
9154 ;
9155 else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() == 1)
9156 SelectCCMaskVal ^= SelectCCValidVal;
9157 else
9158 return SDValue();
9159
9160 if (SelectCCValidVal & ~CCValidVal)
9161 return SDValue();
9162 if (SelectCCMaskVal != (CCMaskVal & SelectCCValidVal))
9163 return SDValue();
9164
9165 return Select->getOperand(Num: 4);
9166}
9167
9168SDValue SystemZTargetLowering::combineIntDIVREM(
9169 SDNode *N, DAGCombinerInfo &DCI) const {
9170 SelectionDAG &DAG = DCI.DAG;
9171 EVT VT = N->getValueType(ResNo: 0);
9172 // In the case where the divisor is a vector of constants a cheaper
9173 // sequence of instructions can replace the divide. BuildSDIV is called to
9174 // do this during DAG combining, but it only succeeds when it can build a
9175 // multiplication node. The only option for SystemZ is ISD::SMUL_LOHI, and
9176 // since it is not Legal but Custom it can only happen before
9177 // legalization. Therefore we must scalarize this early before Combine
9178 // 1. For widened vectors, this is already the result of type legalization.
9179 if (DCI.Level == BeforeLegalizeTypes && VT.isVector() && isTypeLegal(VT) &&
9180 DAG.isConstantIntBuildVectorOrConstantInt(N: N->getOperand(Num: 1)))
9181 return DAG.UnrollVectorOp(N);
9182 return SDValue();
9183}
9184
9185
9186// Transform a right shift of a multiply-and-add into a multiply-and-add-high.
9187// This is closely modeled after the common-code combineShiftToMULH.
9188SDValue SystemZTargetLowering::combineShiftToMulAddHigh(
9189 SDNode *N, DAGCombinerInfo &DCI) const {
9190 SelectionDAG &DAG = DCI.DAG;
9191 SDLoc DL(N);
9192
9193 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
9194 "SRL or SRA node is required here!");
9195
9196 if (!Subtarget.hasVector())
9197 return SDValue();
9198
9199 // Check the shift amount. Proceed with the transformation if the shift
9200 // amount is constant.
9201 ConstantSDNode *ShiftAmtSrc = isConstOrConstSplat(N: N->getOperand(Num: 1));
9202 if (!ShiftAmtSrc)
9203 return SDValue();
9204
9205 // The operation feeding into the shift must be an add.
9206 SDValue ShiftOperand = N->getOperand(Num: 0);
9207 if (ShiftOperand.getOpcode() != ISD::ADD)
9208 return SDValue();
9209
9210 // One operand of the add must be a multiply.
9211 SDValue MulOp = ShiftOperand.getOperand(i: 0);
9212 SDValue AddOp = ShiftOperand.getOperand(i: 1);
9213 if (MulOp.getOpcode() != ISD::MUL) {
9214 if (AddOp.getOpcode() != ISD::MUL)
9215 return SDValue();
9216 std::swap(a&: MulOp, b&: AddOp);
9217 }
9218
9219 // All operands must be equivalent extend nodes.
9220 SDValue LeftOp = MulOp.getOperand(i: 0);
9221 SDValue RightOp = MulOp.getOperand(i: 1);
9222
9223 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND;
9224 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND;
9225
9226 if (!IsSignExt && !IsZeroExt)
9227 return SDValue();
9228
9229 EVT NarrowVT = LeftOp.getOperand(i: 0).getValueType();
9230 unsigned NarrowVTSize = NarrowVT.getScalarSizeInBits();
9231
9232 SDValue MulhRightOp;
9233 if (ConstantSDNode *Constant = isConstOrConstSplat(N: RightOp)) {
9234 unsigned ActiveBits = IsSignExt
9235 ? Constant->getAPIntValue().getSignificantBits()
9236 : Constant->getAPIntValue().getActiveBits();
9237 if (ActiveBits > NarrowVTSize)
9238 return SDValue();
9239 MulhRightOp = DAG.getConstant(
9240 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
9241 VT: NarrowVT);
9242 } else {
9243 if (LeftOp.getOpcode() != RightOp.getOpcode())
9244 return SDValue();
9245 // Check that the two extend nodes are the same type.
9246 if (NarrowVT != RightOp.getOperand(i: 0).getValueType())
9247 return SDValue();
9248 MulhRightOp = RightOp.getOperand(i: 0);
9249 }
9250
9251 SDValue MulhAddOp;
9252 if (ConstantSDNode *Constant = isConstOrConstSplat(N: AddOp)) {
9253 unsigned ActiveBits = IsSignExt
9254 ? Constant->getAPIntValue().getSignificantBits()
9255 : Constant->getAPIntValue().getActiveBits();
9256 if (ActiveBits > NarrowVTSize)
9257 return SDValue();
9258 MulhAddOp = DAG.getConstant(
9259 Val: Constant->getAPIntValue().trunc(width: NarrowVT.getScalarSizeInBits()), DL,
9260 VT: NarrowVT);
9261 } else {
9262 if (LeftOp.getOpcode() != AddOp.getOpcode())
9263 return SDValue();
9264 // Check that the two extend nodes are the same type.
9265 if (NarrowVT != AddOp.getOperand(i: 0).getValueType())
9266 return SDValue();
9267 MulhAddOp = AddOp.getOperand(i: 0);
9268 }
9269
9270 EVT WideVT = LeftOp.getValueType();
9271 // Proceed with the transformation if the wide types match.
9272 assert((WideVT == RightOp.getValueType()) &&
9273 "Cannot have a multiply node with two different operand types.");
9274 assert((WideVT == AddOp.getValueType()) &&
9275 "Cannot have an add node with two different operand types.");
9276
9277 // Proceed with the transformation if the wide type is twice as large
9278 // as the narrow type.
9279 if (WideVT.getScalarSizeInBits() != 2 * NarrowVTSize)
9280 return SDValue();
9281
9282 // Check the shift amount with the narrow type size.
9283 // Proceed with the transformation if the shift amount is the width
9284 // of the narrow type.
9285 unsigned ShiftAmt = ShiftAmtSrc->getZExtValue();
9286 if (ShiftAmt != NarrowVTSize)
9287 return SDValue();
9288
9289 // Proceed if we support the multiply-and-add-high operation.
9290 if (!(NarrowVT == MVT::v16i8 || NarrowVT == MVT::v8i16 ||
9291 NarrowVT == MVT::v4i32 ||
9292 (Subtarget.hasVectorEnhancements3() &&
9293 (NarrowVT == MVT::v2i64 || NarrowVT == MVT::i128))))
9294 return SDValue();
9295
9296 // Emit the VMAH (signed) or VMALH (unsigned) operation.
9297 SDValue Result = DAG.getNode(Opcode: IsSignExt ? SystemZISD::VMAH : SystemZISD::VMALH,
9298 DL, VT: NarrowVT, N1: LeftOp.getOperand(i: 0),
9299 N2: MulhRightOp, N3: MulhAddOp);
9300 bool IsSigned = N->getOpcode() == ISD::SRA;
9301 return DAG.getExtOrTrunc(IsSigned, Op: Result, DL, VT: WideVT);
9302}
9303
9304// Op is an operand of a multiplication. Check whether this can be folded
9305// into an even/odd widening operation; if so, return the opcode to be used
9306// and update Op to the appropriate sub-operand. Note that the caller must
9307// verify that *both* operands of the multiplication support the operation.
9308static unsigned detectEvenOddMultiplyOperand(const SelectionDAG &DAG,
9309 const SystemZSubtarget &Subtarget,
9310 SDValue &Op) {
9311 EVT VT = Op.getValueType();
9312
9313 // Check for (sign/zero_extend_vector_inreg (vector_shuffle)) corresponding
9314 // to selecting the even or odd vector elements.
9315 if (VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
9316 (Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ||
9317 Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG)) {
9318 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
9319 unsigned NumElts = VT.getVectorNumElements();
9320 Op = Op.getOperand(i: 0);
9321 if (Op.getValueType().getVectorNumElements() == 2 * NumElts &&
9322 Op.getOpcode() == ISD::VECTOR_SHUFFLE) {
9323 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
9324 ArrayRef<int> ShuffleMask = SVN->getMask();
9325 bool CanUseEven = true, CanUseOdd = true;
9326 for (unsigned Elt = 0; Elt < NumElts; Elt++) {
9327 if (ShuffleMask[Elt] == -1)
9328 continue;
9329 if (unsigned(ShuffleMask[Elt]) != 2 * Elt)
9330 CanUseEven = false;
9331 if (unsigned(ShuffleMask[Elt]) != 2 * Elt + 1)
9332 CanUseOdd = false;
9333 }
9334 Op = Op.getOperand(i: 0);
9335 if (CanUseEven)
9336 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9337 if (CanUseOdd)
9338 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9339 }
9340 }
9341
9342 // For z17, we can also support the v2i64->i128 case, which looks like
9343 // (sign/zero_extend (extract_vector_elt X 0/1))
9344 if (VT == MVT::i128 && Subtarget.hasVectorEnhancements3() &&
9345 (Op.getOpcode() == ISD::SIGN_EXTEND ||
9346 Op.getOpcode() == ISD::ZERO_EXTEND)) {
9347 bool IsSigned = Op.getOpcode() == ISD::SIGN_EXTEND;
9348 Op = Op.getOperand(i: 0);
9349 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
9350 Op.getOperand(i: 0).getValueType() == MVT::v2i64 &&
9351 Op.getOperand(i: 1).getOpcode() == ISD::Constant) {
9352 unsigned Elem = Op.getConstantOperandVal(i: 1);
9353 Op = Op.getOperand(i: 0);
9354 if (Elem == 0)
9355 return IsSigned ? SystemZISD::VME : SystemZISD::VMLE;
9356 if (Elem == 1)
9357 return IsSigned ? SystemZISD::VMO : SystemZISD::VMLO;
9358 }
9359 }
9360
9361 return 0;
9362}
9363
9364SDValue SystemZTargetLowering::combineMUL(
9365 SDNode *N, DAGCombinerInfo &DCI) const {
9366 SelectionDAG &DAG = DCI.DAG;
9367
9368 // Detect even/odd widening multiplication.
9369 SDValue Op0 = N->getOperand(Num: 0);
9370 SDValue Op1 = N->getOperand(Num: 1);
9371 unsigned OpcodeCand0 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op0);
9372 unsigned OpcodeCand1 = detectEvenOddMultiplyOperand(DAG, Subtarget, Op&: Op1);
9373 if (OpcodeCand0 && OpcodeCand0 == OpcodeCand1)
9374 return DAG.getNode(Opcode: OpcodeCand0, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1);
9375
9376 return SDValue();
9377}
9378
9379SDValue SystemZTargetLowering::combineINTRINSIC(
9380 SDNode *N, DAGCombinerInfo &DCI) const {
9381 SelectionDAG &DAG = DCI.DAG;
9382
9383 unsigned Id = N->getConstantOperandVal(Num: 1);
9384 switch (Id) {
9385 // VECTOR LOAD (RIGHTMOST) WITH LENGTH with a length operand of 15
9386 // or larger is simply a vector load.
9387 case Intrinsic::s390_vll:
9388 case Intrinsic::s390_vlrl:
9389 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2)))
9390 if (C->getZExtValue() >= 15)
9391 return DAG.getLoad(VT: N->getValueType(ResNo: 0), dl: SDLoc(N), Chain: N->getOperand(Num: 0),
9392 Ptr: N->getOperand(Num: 3), PtrInfo: MachinePointerInfo());
9393 break;
9394 // Likewise for VECTOR STORE (RIGHTMOST) WITH LENGTH.
9395 case Intrinsic::s390_vstl:
9396 case Intrinsic::s390_vstrl:
9397 if (auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 3)))
9398 if (C->getZExtValue() >= 15)
9399 return DAG.getStore(Chain: N->getOperand(Num: 0), dl: SDLoc(N), Val: N->getOperand(Num: 2),
9400 Ptr: N->getOperand(Num: 4), PtrInfo: MachinePointerInfo());
9401 break;
9402 }
9403
9404 return SDValue();
9405}
9406
9407SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
9408 if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
9409 return N->getOperand(Num: 0);
9410 return N;
9411}
9412
9413SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
9414 DAGCombinerInfo &DCI) const {
9415 switch(N->getOpcode()) {
9416 default: break;
9417 case ISD::ZERO_EXTEND: return combineZERO_EXTEND(N, DCI);
9418 case ISD::SIGN_EXTEND: return combineSIGN_EXTEND(N, DCI);
9419 case ISD::SIGN_EXTEND_INREG: return combineSIGN_EXTEND_INREG(N, DCI);
9420 case SystemZISD::MERGE_HIGH:
9421 case SystemZISD::MERGE_LOW: return combineMERGE(N, DCI);
9422 case ISD::LOAD: return combineLOAD(N, DCI);
9423 case ISD::STORE: return combineSTORE(N, DCI);
9424 case ISD::VECTOR_SHUFFLE: return combineVECTOR_SHUFFLE(N, DCI);
9425 case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI);
9426 case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI);
9427 case ISD::STRICT_FP_ROUND:
9428 case ISD::FP_ROUND: return combineFP_ROUND(N, DCI);
9429 case ISD::STRICT_FP_EXTEND:
9430 case ISD::FP_EXTEND: return combineFP_EXTEND(N, DCI);
9431 case ISD::SINT_TO_FP:
9432 case ISD::UINT_TO_FP: return combineINT_TO_FP(N, DCI);
9433 case ISD::FCOPYSIGN: return combineFCOPYSIGN(N, DCI);
9434 case ISD::BSWAP: return combineBSWAP(N, DCI);
9435 case ISD::SETCC: return combineSETCC(N, DCI);
9436 case SystemZISD::BR_CCMASK: return combineBR_CCMASK(N, DCI);
9437 case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI);
9438 case SystemZISD::GET_CCMASK: return combineGET_CCMASK(N, DCI);
9439 case ISD::SRL:
9440 case ISD::SRA: return combineShiftToMulAddHigh(N, DCI);
9441 case ISD::MUL: return combineMUL(N, DCI);
9442 case ISD::SDIV:
9443 case ISD::UDIV:
9444 case ISD::SREM:
9445 case ISD::UREM: return combineIntDIVREM(N, DCI);
9446 case ISD::INTRINSIC_W_CHAIN:
9447 case ISD::INTRINSIC_VOID: return combineINTRINSIC(N, DCI);
9448 }
9449
9450 return SDValue();
9451}
9452
9453// Return the demanded elements for the OpNo source operand of Op. DemandedElts
9454// are for Op.
9455static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts,
9456 unsigned OpNo) {
9457 EVT VT = Op.getValueType();
9458 unsigned NumElts = (VT.isVector() ? VT.getVectorNumElements() : 1);
9459 APInt SrcDemE;
9460 unsigned Opcode = Op.getOpcode();
9461 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9462 unsigned Id = Op.getConstantOperandVal(i: 0);
9463 switch (Id) {
9464 case Intrinsic::s390_vpksh: // PACKS
9465 case Intrinsic::s390_vpksf:
9466 case Intrinsic::s390_vpksg:
9467 case Intrinsic::s390_vpkshs: // PACKS_CC
9468 case Intrinsic::s390_vpksfs:
9469 case Intrinsic::s390_vpksgs:
9470 case Intrinsic::s390_vpklsh: // PACKLS
9471 case Intrinsic::s390_vpklsf:
9472 case Intrinsic::s390_vpklsg:
9473 case Intrinsic::s390_vpklshs: // PACKLS_CC
9474 case Intrinsic::s390_vpklsfs:
9475 case Intrinsic::s390_vpklsgs:
9476 // VECTOR PACK truncates the elements of two source vectors into one.
9477 SrcDemE = DemandedElts;
9478 if (OpNo == 2)
9479 SrcDemE.lshrInPlace(ShiftAmt: NumElts / 2);
9480 SrcDemE = SrcDemE.trunc(width: NumElts / 2);
9481 break;
9482 // VECTOR UNPACK extends half the elements of the source vector.
9483 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9484 case Intrinsic::s390_vuphh:
9485 case Intrinsic::s390_vuphf:
9486 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9487 case Intrinsic::s390_vuplhh:
9488 case Intrinsic::s390_vuplhf:
9489 SrcDemE = APInt(NumElts * 2, 0);
9490 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: 0);
9491 break;
9492 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9493 case Intrinsic::s390_vuplhw:
9494 case Intrinsic::s390_vuplf:
9495 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9496 case Intrinsic::s390_vupllh:
9497 case Intrinsic::s390_vupllf:
9498 SrcDemE = APInt(NumElts * 2, 0);
9499 SrcDemE.insertBits(SubBits: DemandedElts, bitPosition: NumElts);
9500 break;
9501 case Intrinsic::s390_vpdi: {
9502 // VECTOR PERMUTE DWORD IMMEDIATE selects one element from each source.
9503 SrcDemE = APInt(NumElts, 0);
9504 if (!DemandedElts[OpNo - 1])
9505 break;
9506 unsigned Mask = Op.getConstantOperandVal(i: 3);
9507 unsigned MaskBit = ((OpNo - 1) ? 1 : 4);
9508 // Demand input element 0 or 1, given by the mask bit value.
9509 SrcDemE.setBit((Mask & MaskBit)? 1 : 0);
9510 break;
9511 }
9512 case Intrinsic::s390_vsldb: {
9513 // VECTOR SHIFT LEFT DOUBLE BY BYTE
9514 assert(VT == MVT::v16i8 && "Unexpected type.");
9515 unsigned FirstIdx = Op.getConstantOperandVal(i: 3);
9516 assert (FirstIdx > 0 && FirstIdx < 16 && "Unused operand.");
9517 unsigned NumSrc0Els = 16 - FirstIdx;
9518 SrcDemE = APInt(NumElts, 0);
9519 if (OpNo == 1) {
9520 APInt DemEls = DemandedElts.trunc(width: NumSrc0Els);
9521 SrcDemE.insertBits(SubBits: DemEls, bitPosition: FirstIdx);
9522 } else {
9523 APInt DemEls = DemandedElts.lshr(shiftAmt: NumSrc0Els);
9524 SrcDemE.insertBits(SubBits: DemEls, bitPosition: 0);
9525 }
9526 break;
9527 }
9528 case Intrinsic::s390_vperm:
9529 SrcDemE = APInt::getAllOnes(numBits: NumElts);
9530 break;
9531 default:
9532 llvm_unreachable("Unhandled intrinsic.");
9533 break;
9534 }
9535 } else {
9536 switch (Opcode) {
9537 case SystemZISD::JOIN_DWORDS:
9538 // Scalar operand.
9539 SrcDemE = APInt(1, 1);
9540 break;
9541 case SystemZISD::SELECT_CCMASK:
9542 SrcDemE = DemandedElts;
9543 break;
9544 default:
9545 llvm_unreachable("Unhandled opcode.");
9546 break;
9547 }
9548 }
9549 return SrcDemE;
9550}
9551
9552static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known,
9553 const APInt &DemandedElts,
9554 const SelectionDAG &DAG, unsigned Depth,
9555 unsigned OpNo) {
9556 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9557 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9558 KnownBits LHSKnown =
9559 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9560 KnownBits RHSKnown =
9561 DAG.computeKnownBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9562 Known = LHSKnown.intersectWith(RHS: RHSKnown);
9563}
9564
9565void
9566SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
9567 KnownBits &Known,
9568 const APInt &DemandedElts,
9569 const SelectionDAG &DAG,
9570 unsigned Depth) const {
9571 Known.resetAll();
9572
9573 // Intrinsic CC result is returned in the two low bits.
9574 unsigned Tmp0, Tmp1; // not used
9575 if (Op.getResNo() == 1 && isIntrinsicWithCC(Op, Opcode&: Tmp0, CCValid&: Tmp1)) {
9576 Known.Zero.setBitsFrom(2);
9577 return;
9578 }
9579 EVT VT = Op.getValueType();
9580 if (Op.getResNo() != 0 || VT == MVT::Untyped)
9581 return;
9582 assert (Known.getBitWidth() == VT.getScalarSizeInBits() &&
9583 "KnownBits does not match VT in bitwidth");
9584 assert ((!VT.isVector() ||
9585 (DemandedElts.getBitWidth() == VT.getVectorNumElements())) &&
9586 "DemandedElts does not match VT number of elements");
9587 unsigned BitWidth = Known.getBitWidth();
9588 unsigned Opcode = Op.getOpcode();
9589 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9590 bool IsLogical = false;
9591 unsigned Id = Op.getConstantOperandVal(i: 0);
9592 switch (Id) {
9593 case Intrinsic::s390_vpksh: // PACKS
9594 case Intrinsic::s390_vpksf:
9595 case Intrinsic::s390_vpksg:
9596 case Intrinsic::s390_vpkshs: // PACKS_CC
9597 case Intrinsic::s390_vpksfs:
9598 case Intrinsic::s390_vpksgs:
9599 case Intrinsic::s390_vpklsh: // PACKLS
9600 case Intrinsic::s390_vpklsf:
9601 case Intrinsic::s390_vpklsg:
9602 case Intrinsic::s390_vpklshs: // PACKLS_CC
9603 case Intrinsic::s390_vpklsfs:
9604 case Intrinsic::s390_vpklsgs:
9605 case Intrinsic::s390_vpdi:
9606 case Intrinsic::s390_vsldb:
9607 case Intrinsic::s390_vperm:
9608 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 1);
9609 break;
9610 case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
9611 case Intrinsic::s390_vuplhh:
9612 case Intrinsic::s390_vuplhf:
9613 case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
9614 case Intrinsic::s390_vupllh:
9615 case Intrinsic::s390_vupllf:
9616 IsLogical = true;
9617 [[fallthrough]];
9618 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9619 case Intrinsic::s390_vuphh:
9620 case Intrinsic::s390_vuphf:
9621 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9622 case Intrinsic::s390_vuplhw:
9623 case Intrinsic::s390_vuplf: {
9624 SDValue SrcOp = Op.getOperand(i: 1);
9625 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 0);
9626 Known = DAG.computeKnownBits(Op: SrcOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9627 if (IsLogical) {
9628 Known = Known.zext(BitWidth);
9629 } else
9630 Known = Known.sext(BitWidth);
9631 break;
9632 }
9633 default:
9634 break;
9635 }
9636 } else {
9637 switch (Opcode) {
9638 case SystemZISD::JOIN_DWORDS:
9639 case SystemZISD::SELECT_CCMASK:
9640 computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, OpNo: 0);
9641 break;
9642 case SystemZISD::REPLICATE: {
9643 SDValue SrcOp = Op.getOperand(i: 0);
9644 Known = DAG.computeKnownBits(Op: SrcOp, Depth: Depth + 1);
9645 if (Known.getBitWidth() < BitWidth && isa<ConstantSDNode>(Val: SrcOp))
9646 Known = Known.sext(BitWidth); // VREPI sign extends the immedate.
9647 break;
9648 }
9649 default:
9650 break;
9651 }
9652 }
9653
9654 // Known has the width of the source operand(s). Adjust if needed to match
9655 // the passed bitwidth.
9656 if (Known.getBitWidth() != BitWidth)
9657 Known = Known.anyextOrTrunc(BitWidth);
9658}
9659
9660static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts,
9661 const SelectionDAG &DAG, unsigned Depth,
9662 unsigned OpNo) {
9663 APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
9664 unsigned LHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo), DemandedElts: Src0DemE, Depth: Depth + 1);
9665 if (LHS == 1) return 1; // Early out.
9666 APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo: OpNo + 1);
9667 unsigned RHS = DAG.ComputeNumSignBits(Op: Op.getOperand(i: OpNo + 1), DemandedElts: Src1DemE, Depth: Depth + 1);
9668 if (RHS == 1) return 1; // Early out.
9669 unsigned Common = std::min(a: LHS, b: RHS);
9670 unsigned SrcBitWidth = Op.getOperand(i: OpNo).getScalarValueSizeInBits();
9671 EVT VT = Op.getValueType();
9672 unsigned VTBits = VT.getScalarSizeInBits();
9673 if (SrcBitWidth > VTBits) { // PACK
9674 unsigned SrcExtraBits = SrcBitWidth - VTBits;
9675 if (Common > SrcExtraBits)
9676 return (Common - SrcExtraBits);
9677 return 1;
9678 }
9679 assert (SrcBitWidth == VTBits && "Expected operands of same bitwidth.");
9680 return Common;
9681}
9682
9683unsigned
9684SystemZTargetLowering::ComputeNumSignBitsForTargetNode(
9685 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9686 unsigned Depth) const {
9687 if (Op.getResNo() != 0)
9688 return 1;
9689 unsigned Opcode = Op.getOpcode();
9690 if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
9691 unsigned Id = Op.getConstantOperandVal(i: 0);
9692 switch (Id) {
9693 case Intrinsic::s390_vpksh: // PACKS
9694 case Intrinsic::s390_vpksf:
9695 case Intrinsic::s390_vpksg:
9696 case Intrinsic::s390_vpkshs: // PACKS_CC
9697 case Intrinsic::s390_vpksfs:
9698 case Intrinsic::s390_vpksgs:
9699 case Intrinsic::s390_vpklsh: // PACKLS
9700 case Intrinsic::s390_vpklsf:
9701 case Intrinsic::s390_vpklsg:
9702 case Intrinsic::s390_vpklshs: // PACKLS_CC
9703 case Intrinsic::s390_vpklsfs:
9704 case Intrinsic::s390_vpklsgs:
9705 case Intrinsic::s390_vpdi:
9706 case Intrinsic::s390_vsldb:
9707 case Intrinsic::s390_vperm:
9708 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 1);
9709 case Intrinsic::s390_vuphb: // VECTOR UNPACK HIGH
9710 case Intrinsic::s390_vuphh:
9711 case Intrinsic::s390_vuphf:
9712 case Intrinsic::s390_vuplb: // VECTOR UNPACK LOW
9713 case Intrinsic::s390_vuplhw:
9714 case Intrinsic::s390_vuplf: {
9715 SDValue PackedOp = Op.getOperand(i: 1);
9716 APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, OpNo: 1);
9717 unsigned Tmp = DAG.ComputeNumSignBits(Op: PackedOp, DemandedElts: SrcDemE, Depth: Depth + 1);
9718 EVT VT = Op.getValueType();
9719 unsigned VTBits = VT.getScalarSizeInBits();
9720 Tmp += VTBits - PackedOp.getScalarValueSizeInBits();
9721 return Tmp;
9722 }
9723 default:
9724 break;
9725 }
9726 } else {
9727 switch (Opcode) {
9728 case SystemZISD::SELECT_CCMASK:
9729 return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, OpNo: 0);
9730 default:
9731 break;
9732 }
9733 }
9734
9735 return 1;
9736}
9737
9738bool SystemZTargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
9739 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
9740 UndefPoisonKind Kind, unsigned Depth) const {
9741 switch (Op->getOpcode()) {
9742 case SystemZISD::PCREL_WRAPPER:
9743 case SystemZISD::PCREL_OFFSET:
9744 return true;
9745 }
9746 return false;
9747}
9748
9749unsigned
9750SystemZTargetLowering::getStackProbeSize(const MachineFunction &MF) const {
9751 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
9752 unsigned StackAlign = TFI->getStackAlignment();
9753 assert(StackAlign >=1 && isPowerOf2_32(StackAlign) &&
9754 "Unexpected stack alignment");
9755 // The default stack probe size is 4096 if the function has no
9756 // stack-probe-size attribute.
9757 unsigned StackProbeSize =
9758 MF.getFunction().getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
9759 // Round down to the stack alignment.
9760 StackProbeSize &= ~(StackAlign - 1);
9761 return StackProbeSize ? StackProbeSize : StackAlign;
9762}
9763
9764//===----------------------------------------------------------------------===//
9765// Custom insertion
9766//===----------------------------------------------------------------------===//
9767
9768// Force base value Base into a register before MI. Return the register.
9769static Register forceReg(MachineInstr &MI, MachineOperand &Base,
9770 const SystemZInstrInfo *TII) {
9771 MachineBasicBlock *MBB = MI.getParent();
9772 MachineFunction &MF = *MBB->getParent();
9773 MachineRegisterInfo &MRI = MF.getRegInfo();
9774
9775 if (Base.isReg()) {
9776 // Copy Base into a new virtual register to help register coalescing in
9777 // cases with multiple uses.
9778 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9779 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::COPY), DestReg: Reg)
9780 .add(MO: Base);
9781 return Reg;
9782 }
9783
9784 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
9785 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::LA), DestReg: Reg)
9786 .add(MO: Base)
9787 .addImm(Val: 0)
9788 .addReg(RegNo: 0);
9789 return Reg;
9790}
9791
9792// The CC operand of MI might be missing a kill marker because there
9793// were multiple uses of CC, and ISel didn't know which to mark.
9794// Figure out whether MI should have had a kill marker.
9795static bool checkCCKill(MachineInstr &MI, MachineBasicBlock *MBB) {
9796 // Scan forward through BB for a use/def of CC.
9797 MachineBasicBlock::iterator miI(std::next(x: MachineBasicBlock::iterator(MI)));
9798 for (MachineBasicBlock::iterator miE = MBB->end(); miI != miE; ++miI) {
9799 const MachineInstr &MI = *miI;
9800 if (MI.readsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9801 return false;
9802 if (MI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr))
9803 break; // Should have kill-flag - update below.
9804 }
9805
9806 // If we hit the end of the block, check whether CC is live into a
9807 // successor.
9808 if (miI == MBB->end()) {
9809 for (const MachineBasicBlock *Succ : MBB->successors())
9810 if (Succ->isLiveIn(Reg: SystemZ::CC))
9811 return false;
9812 }
9813
9814 return true;
9815}
9816
9817// Return true if it is OK for this Select pseudo-opcode to be cascaded
9818// together with other Select pseudo-opcodes into a single basic-block with
9819// a conditional jump around it.
9820static bool isSelectPseudo(MachineInstr &MI) {
9821 switch (MI.getOpcode()) {
9822 case SystemZ::Select32:
9823 case SystemZ::Select64:
9824 case SystemZ::Select128:
9825 case SystemZ::SelectF32:
9826 case SystemZ::SelectF64:
9827 case SystemZ::SelectF128:
9828 case SystemZ::SelectVR32:
9829 case SystemZ::SelectVR64:
9830 case SystemZ::SelectVR128:
9831 return true;
9832
9833 default:
9834 return false;
9835 }
9836}
9837
9838// Helper function, which inserts PHI functions into SinkMBB:
9839// %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
9840// where %FalseValue(i) and %TrueValue(i) are taken from Selects.
9841static void createPHIsForSelects(SmallVector<MachineInstr*, 8> &Selects,
9842 MachineBasicBlock *TrueMBB,
9843 MachineBasicBlock *FalseMBB,
9844 MachineBasicBlock *SinkMBB) {
9845 MachineFunction *MF = TrueMBB->getParent();
9846 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
9847
9848 MachineInstr *FirstMI = Selects.front();
9849 unsigned CCValid = FirstMI->getOperand(i: 3).getImm();
9850 unsigned CCMask = FirstMI->getOperand(i: 4).getImm();
9851
9852 MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
9853
9854 // As we are creating the PHIs, we have to be careful if there is more than
9855 // one. Later Selects may reference the results of earlier Selects, but later
9856 // PHIs have to reference the individual true/false inputs from earlier PHIs.
9857 // That also means that PHI construction must work forward from earlier to
9858 // later, and that the code must maintain a mapping from earlier PHI's
9859 // destination registers, and the registers that went into the PHI.
9860 DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
9861
9862 for (auto *MI : Selects) {
9863 Register DestReg = MI->getOperand(i: 0).getReg();
9864 Register TrueReg = MI->getOperand(i: 1).getReg();
9865 Register FalseReg = MI->getOperand(i: 2).getReg();
9866
9867 // If this Select we are generating is the opposite condition from
9868 // the jump we generated, then we have to swap the operands for the
9869 // PHI that is going to be generated.
9870 if (MI->getOperand(i: 4).getImm() == (CCValid ^ CCMask))
9871 std::swap(a&: TrueReg, b&: FalseReg);
9872
9873 if (auto It = RegRewriteTable.find(Val: TrueReg); It != RegRewriteTable.end())
9874 TrueReg = It->second.first;
9875
9876 if (auto It = RegRewriteTable.find(Val: FalseReg); It != RegRewriteTable.end())
9877 FalseReg = It->second.second;
9878
9879 DebugLoc DL = MI->getDebugLoc();
9880 BuildMI(BB&: *SinkMBB, I: SinkInsertionPoint, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg)
9881 .addReg(RegNo: TrueReg).addMBB(MBB: TrueMBB)
9882 .addReg(RegNo: FalseReg).addMBB(MBB: FalseMBB);
9883
9884 // Add this PHI to the rewrite table.
9885 RegRewriteTable[DestReg] = std::make_pair(x&: TrueReg, y&: FalseReg);
9886 }
9887
9888 MF->getProperties().resetNoPHIs();
9889}
9890
9891MachineBasicBlock *
9892SystemZTargetLowering::emitAdjCallStack(MachineInstr &MI,
9893 MachineBasicBlock *BB) const {
9894 MachineFunction &MF = *BB->getParent();
9895 MachineFrameInfo &MFI = MF.getFrameInfo();
9896 auto *TFL = Subtarget.getFrameLowering<SystemZFrameLowering>();
9897 assert(TFL->hasReservedCallFrame(MF) &&
9898 "ADJSTACKDOWN and ADJSTACKUP should be no-ops");
9899 (void)TFL;
9900 // Get the MaxCallFrameSize value and erase MI since it serves no further
9901 // purpose as the call frame is statically reserved in the prolog. Set
9902 // AdjustsStack as MI is *not* mapped as a frame instruction.
9903 uint32_t NumBytes = MI.getOperand(i: 0).getImm();
9904 if (NumBytes > MFI.getMaxCallFrameSize())
9905 MFI.setMaxCallFrameSize(NumBytes);
9906 MFI.setAdjustsStack(true);
9907
9908 MI.eraseFromParent();
9909 return BB;
9910}
9911
9912// Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI.
9913MachineBasicBlock *
9914SystemZTargetLowering::emitSelect(MachineInstr &MI,
9915 MachineBasicBlock *MBB) const {
9916 assert(isSelectPseudo(MI) && "Bad call to emitSelect()");
9917 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
9918
9919 unsigned CCValid = MI.getOperand(i: 3).getImm();
9920 unsigned CCMask = MI.getOperand(i: 4).getImm();
9921
9922 // If we have a sequence of Select* pseudo instructions using the
9923 // same condition code value, we want to expand all of them into
9924 // a single pair of basic blocks using the same condition.
9925 SmallVector<MachineInstr*, 8> Selects;
9926 SmallVector<MachineInstr*, 8> DbgValues;
9927 Selects.push_back(Elt: &MI);
9928 unsigned Count = 0;
9929 for (MachineInstr &NextMI : llvm::make_range(
9930 x: std::next(x: MachineBasicBlock::iterator(MI)), y: MBB->end())) {
9931 if (isSelectPseudo(MI&: NextMI)) {
9932 assert(NextMI.getOperand(3).getImm() == CCValid &&
9933 "Bad CCValid operands since CC was not redefined.");
9934 if (NextMI.getOperand(i: 4).getImm() == CCMask ||
9935 NextMI.getOperand(i: 4).getImm() == (CCValid ^ CCMask)) {
9936 Selects.push_back(Elt: &NextMI);
9937 continue;
9938 }
9939 break;
9940 }
9941 if (NextMI.definesRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
9942 NextMI.usesCustomInsertionHook())
9943 break;
9944 bool User = false;
9945 for (auto *SelMI : Selects)
9946 if (NextMI.readsVirtualRegister(Reg: SelMI->getOperand(i: 0).getReg())) {
9947 User = true;
9948 break;
9949 }
9950 if (NextMI.isDebugInstr()) {
9951 if (User) {
9952 assert(NextMI.isDebugValue() && "Unhandled debug opcode.");
9953 DbgValues.push_back(Elt: &NextMI);
9954 }
9955 } else if (User || ++Count > 20)
9956 break;
9957 }
9958
9959 MachineInstr *LastMI = Selects.back();
9960 bool CCKilled = (LastMI->killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) ||
9961 checkCCKill(MI&: *LastMI, MBB));
9962 MachineBasicBlock *StartMBB = MBB;
9963 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI: LastMI, MBB);
9964 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
9965
9966 // Unless CC was killed in the last Select instruction, mark it as
9967 // live-in to both FalseMBB and JoinMBB.
9968 if (!CCKilled) {
9969 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
9970 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
9971 }
9972
9973 // StartMBB:
9974 // BRC CCMask, JoinMBB
9975 // # fallthrough to FalseMBB
9976 MBB = StartMBB;
9977 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
9978 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
9979 MBB->addSuccessor(Succ: JoinMBB);
9980 MBB->addSuccessor(Succ: FalseMBB);
9981
9982 // FalseMBB:
9983 // # fallthrough to JoinMBB
9984 MBB = FalseMBB;
9985 MBB->addSuccessor(Succ: JoinMBB);
9986
9987 // JoinMBB:
9988 // %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ]
9989 // ...
9990 MBB = JoinMBB;
9991 createPHIsForSelects(Selects, TrueMBB: StartMBB, FalseMBB, SinkMBB: MBB);
9992 for (auto *SelMI : Selects)
9993 SelMI->eraseFromParent();
9994
9995 MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();
9996 for (auto *DbgMI : DbgValues)
9997 MBB->splice(Where: InsertPos, Other: StartMBB, From: DbgMI);
9998
9999 return JoinMBB;
10000}
10001
10002// Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI.
10003// StoreOpcode is the store to use and Invert says whether the store should
10004// happen when the condition is false rather than true. If a STORE ON
10005// CONDITION is available, STOCOpcode is its opcode, otherwise it is 0.
10006MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI,
10007 MachineBasicBlock *MBB,
10008 unsigned StoreOpcode,
10009 unsigned STOCOpcode,
10010 bool Invert) const {
10011 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10012
10013 Register SrcReg = MI.getOperand(i: 0).getReg();
10014 MachineOperand Base = MI.getOperand(i: 1);
10015 int64_t Disp = MI.getOperand(i: 2).getImm();
10016 Register IndexReg = MI.getOperand(i: 3).getReg();
10017 unsigned CCValid = MI.getOperand(i: 4).getImm();
10018 unsigned CCMask = MI.getOperand(i: 5).getImm();
10019 DebugLoc DL = MI.getDebugLoc();
10020
10021 StoreOpcode = TII->getOpcodeForOffset(Opcode: StoreOpcode, Offset: Disp);
10022
10023 // ISel pattern matching also adds a load memory operand of the same
10024 // address, so take special care to find the storing memory operand.
10025 MachineMemOperand *MMO = nullptr;
10026 for (auto *I : MI.memoperands())
10027 if (I->isStore()) {
10028 MMO = I;
10029 break;
10030 }
10031
10032 // Use STOCOpcode if possible. We could use different store patterns in
10033 // order to avoid matching the index register, but the performance trade-offs
10034 // might be more complicated in that case.
10035 if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
10036 if (Invert)
10037 CCMask ^= CCValid;
10038
10039 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: STOCOpcode))
10040 .addReg(RegNo: SrcReg)
10041 .add(MO: Base)
10042 .addImm(Val: Disp)
10043 .addImm(Val: CCValid)
10044 .addImm(Val: CCMask)
10045 .addMemOperand(MMO);
10046
10047 MI.eraseFromParent();
10048 return MBB;
10049 }
10050
10051 // Get the condition needed to branch around the store.
10052 if (!Invert)
10053 CCMask ^= CCValid;
10054
10055 MachineBasicBlock *StartMBB = MBB;
10056 MachineBasicBlock *JoinMBB = SystemZ::splitBlockBefore(MI, MBB);
10057 MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10058
10059 // Unless CC was killed in the CondStore instruction, mark it as
10060 // live-in to both FalseMBB and JoinMBB.
10061 if (!MI.killsRegister(Reg: SystemZ::CC, /*TRI=*/nullptr) &&
10062 !checkCCKill(MI, MBB: JoinMBB)) {
10063 FalseMBB->addLiveIn(PhysReg: SystemZ::CC);
10064 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10065 }
10066
10067 // StartMBB:
10068 // BRC CCMask, JoinMBB
10069 // # fallthrough to FalseMBB
10070 MBB = StartMBB;
10071 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10072 .addImm(Val: CCValid).addImm(Val: CCMask).addMBB(MBB: JoinMBB);
10073 MBB->addSuccessor(Succ: JoinMBB);
10074 MBB->addSuccessor(Succ: FalseMBB);
10075
10076 // FalseMBB:
10077 // store %SrcReg, %Disp(%Index,%Base)
10078 // # fallthrough to JoinMBB
10079 MBB = FalseMBB;
10080 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: StoreOpcode))
10081 .addReg(RegNo: SrcReg)
10082 .add(MO: Base)
10083 .addImm(Val: Disp)
10084 .addReg(RegNo: IndexReg)
10085 .addMemOperand(MMO);
10086 MBB->addSuccessor(Succ: JoinMBB);
10087
10088 MI.eraseFromParent();
10089 return JoinMBB;
10090}
10091
10092// Implement EmitInstrWithCustomInserter for pseudo [SU]Cmp128Hi instruction MI.
10093MachineBasicBlock *
10094SystemZTargetLowering::emitICmp128Hi(MachineInstr &MI,
10095 MachineBasicBlock *MBB,
10096 bool Unsigned) const {
10097 MachineFunction &MF = *MBB->getParent();
10098 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10099 MachineRegisterInfo &MRI = MF.getRegInfo();
10100
10101 // Synthetic instruction to compare 128-bit values.
10102 // Sets CC 1 if Op0 > Op1, sets a different CC otherwise.
10103 Register Op0 = MI.getOperand(i: 0).getReg();
10104 Register Op1 = MI.getOperand(i: 1).getReg();
10105
10106 MachineBasicBlock *StartMBB = MBB;
10107 MachineBasicBlock *JoinMBB = SystemZ::splitBlockAfter(MI, MBB);
10108 MachineBasicBlock *HiEqMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10109
10110 // StartMBB:
10111 //
10112 // Use VECTOR ELEMENT COMPARE [LOGICAL] to compare the high parts.
10113 // Swap the inputs to get:
10114 // CC 1 if high(Op0) > high(Op1)
10115 // CC 2 if high(Op0) < high(Op1)
10116 // CC 0 if high(Op0) == high(Op1)
10117 //
10118 // If CC != 0, we'd done, so jump over the next instruction.
10119 //
10120 // VEC[L]G Op1, Op0
10121 // JNE JoinMBB
10122 // # fallthrough to HiEqMBB
10123 MBB = StartMBB;
10124 int HiOpcode = Unsigned? SystemZ::VECLG : SystemZ::VECG;
10125 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: HiOpcode))
10126 .addReg(RegNo: Op1).addReg(RegNo: Op0);
10127 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::BRC))
10128 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: JoinMBB);
10129 MBB->addSuccessor(Succ: JoinMBB);
10130 MBB->addSuccessor(Succ: HiEqMBB);
10131
10132 // HiEqMBB:
10133 //
10134 // Otherwise, use VECTOR COMPARE HIGH LOGICAL.
10135 // Since we already know the high parts are equal, the CC
10136 // result will only depend on the low parts:
10137 // CC 1 if low(Op0) > low(Op1)
10138 // CC 3 if low(Op0) <= low(Op1)
10139 //
10140 // VCHLGS Tmp, Op0, Op1
10141 // # fallthrough to JoinMBB
10142 MBB = HiEqMBB;
10143 Register Temp = MRI.createVirtualRegister(RegClass: &SystemZ::VR128BitRegClass);
10144 BuildMI(BB: MBB, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SystemZ::VCHLGS), DestReg: Temp)
10145 .addReg(RegNo: Op0).addReg(RegNo: Op1);
10146 MBB->addSuccessor(Succ: JoinMBB);
10147
10148 // Mark CC as live-in to JoinMBB.
10149 JoinMBB->addLiveIn(PhysReg: SystemZ::CC);
10150
10151 MI.eraseFromParent();
10152 return JoinMBB;
10153}
10154
10155// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_LOADW_* or
10156// ATOMIC_SWAPW instruction MI. BinOpcode is the instruction that performs
10157// the binary operation elided by "*", or 0 for ATOMIC_SWAPW. Invert says
10158// whether the field should be inverted after performing BinOpcode (e.g. for
10159// NAND).
10160MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
10161 MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode,
10162 bool Invert) const {
10163 MachineFunction &MF = *MBB->getParent();
10164 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10165 MachineRegisterInfo &MRI = MF.getRegInfo();
10166
10167 // Extract the operands. Base can be a register or a frame index.
10168 // Src2 can be a register or immediate.
10169 Register Dest = MI.getOperand(i: 0).getReg();
10170 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10171 int64_t Disp = MI.getOperand(i: 2).getImm();
10172 MachineOperand Src2 = earlyUseOperand(Op: MI.getOperand(i: 3));
10173 Register BitShift = MI.getOperand(i: 4).getReg();
10174 Register NegBitShift = MI.getOperand(i: 5).getReg();
10175 unsigned BitSize = MI.getOperand(i: 6).getImm();
10176 DebugLoc DL = MI.getDebugLoc();
10177
10178 // Get the right opcodes for the displacement.
10179 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10180 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10181 assert(LOpcode && CSOpcode && "Displacement out of range");
10182
10183 // Create virtual registers for temporary results.
10184 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10185 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10186 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10187 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10188 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10189
10190 // Insert a basic block for the main loop.
10191 MachineBasicBlock *StartMBB = MBB;
10192 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10193 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10194
10195 // StartMBB:
10196 // ...
10197 // %OrigVal = L Disp(%Base)
10198 // # fall through to LoopMBB
10199 MBB = StartMBB;
10200 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10201 MBB->addSuccessor(Succ: LoopMBB);
10202
10203 // LoopMBB:
10204 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ]
10205 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10206 // %RotatedNewVal = OP %RotatedOldVal, %Src2
10207 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10208 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10209 // JNE LoopMBB
10210 // # fall through to DoneMBB
10211 MBB = LoopMBB;
10212 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10213 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
10214 .addReg(RegNo: Dest).addMBB(MBB: LoopMBB);
10215 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
10216 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
10217 if (Invert) {
10218 // Perform the operation normally and then invert every bit of the field.
10219 Register Tmp = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10220 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: Tmp).addReg(RegNo: RotatedOldVal).add(MO: Src2);
10221 // XILF with the upper BitSize bits set.
10222 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::XILF), DestReg: RotatedNewVal)
10223 .addReg(RegNo: Tmp).addImm(Val: -1U << (32 - BitSize));
10224 } else if (BinOpcode)
10225 // A simply binary operation.
10226 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: BinOpcode), DestReg: RotatedNewVal)
10227 .addReg(RegNo: RotatedOldVal)
10228 .add(MO: Src2);
10229 else
10230 // Use RISBG to rotate Src2 into position and use it to replace the
10231 // field in RotatedOldVal.
10232 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedNewVal)
10233 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2.getReg())
10234 .addImm(Val: 32).addImm(Val: 31 + BitSize).addImm(Val: 32 - BitSize);
10235 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
10236 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
10237 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
10238 .addReg(RegNo: OldVal)
10239 .addReg(RegNo: NewVal)
10240 .add(MO: Base)
10241 .addImm(Val: Disp);
10242 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10243 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10244 MBB->addSuccessor(Succ: LoopMBB);
10245 MBB->addSuccessor(Succ: DoneMBB);
10246
10247 MI.eraseFromParent();
10248 return DoneMBB;
10249}
10250
10251// Implement EmitInstrWithCustomInserter for subword pseudo
10252// ATOMIC_LOADW_{,U}{MIN,MAX} instruction MI. CompareOpcode is the
10253// instruction that should be used to compare the current field with the
10254// minimum or maximum value. KeepOldMask is the BRC condition-code mask
10255// for when the current field should be kept.
10256MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
10257 MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode,
10258 unsigned KeepOldMask) const {
10259 MachineFunction &MF = *MBB->getParent();
10260 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10261 MachineRegisterInfo &MRI = MF.getRegInfo();
10262
10263 // Extract the operands. Base can be a register or a frame index.
10264 Register Dest = MI.getOperand(i: 0).getReg();
10265 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10266 int64_t Disp = MI.getOperand(i: 2).getImm();
10267 Register Src2 = MI.getOperand(i: 3).getReg();
10268 Register BitShift = MI.getOperand(i: 4).getReg();
10269 Register NegBitShift = MI.getOperand(i: 5).getReg();
10270 unsigned BitSize = MI.getOperand(i: 6).getImm();
10271 DebugLoc DL = MI.getDebugLoc();
10272
10273 // Get the right opcodes for the displacement.
10274 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10275 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10276 assert(LOpcode && CSOpcode && "Displacement out of range");
10277
10278 // Create virtual registers for temporary results.
10279 Register OrigVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10280 Register OldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10281 Register NewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10282 Register RotatedOldVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10283 Register RotatedAltVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10284 Register RotatedNewVal = MRI.createVirtualRegister(RegClass: &SystemZ::GR32BitRegClass);
10285
10286 // Insert 3 basic blocks for the loop.
10287 MachineBasicBlock *StartMBB = MBB;
10288 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10289 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10290 MachineBasicBlock *UseAltMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10291 MachineBasicBlock *UpdateMBB = SystemZ::emitBlockAfter(MBB: UseAltMBB);
10292
10293 // StartMBB:
10294 // ...
10295 // %OrigVal = L Disp(%Base)
10296 // # fall through to LoopMBB
10297 MBB = StartMBB;
10298 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigVal).add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10299 MBB->addSuccessor(Succ: LoopMBB);
10300
10301 // LoopMBB:
10302 // %OldVal = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ]
10303 // %RotatedOldVal = RLL %OldVal, 0(%BitShift)
10304 // CompareOpcode %RotatedOldVal, %Src2
10305 // BRC KeepOldMask, UpdateMBB
10306 MBB = LoopMBB;
10307 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10308 .addReg(RegNo: OrigVal).addMBB(MBB: StartMBB)
10309 .addReg(RegNo: Dest).addMBB(MBB: UpdateMBB);
10310 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: RotatedOldVal)
10311 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: 0);
10312 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CompareOpcode))
10313 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2);
10314 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10315 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: KeepOldMask).addMBB(MBB: UpdateMBB);
10316 MBB->addSuccessor(Succ: UpdateMBB);
10317 MBB->addSuccessor(Succ: UseAltMBB);
10318
10319 // UseAltMBB:
10320 // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0
10321 // # fall through to UpdateMBB
10322 MBB = UseAltMBB;
10323 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RotatedAltVal)
10324 .addReg(RegNo: RotatedOldVal).addReg(RegNo: Src2)
10325 .addImm(Val: 32).addImm(Val: 31 + BitSize).addImm(Val: 0);
10326 MBB->addSuccessor(Succ: UpdateMBB);
10327
10328 // UpdateMBB:
10329 // %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ],
10330 // [ %RotatedAltVal, UseAltMBB ]
10331 // %NewVal = RLL %RotatedNewVal, 0(%NegBitShift)
10332 // %Dest = CS %OldVal, %NewVal, Disp(%Base)
10333 // JNE LoopMBB
10334 // # fall through to DoneMBB
10335 MBB = UpdateMBB;
10336 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RotatedNewVal)
10337 .addReg(RegNo: RotatedOldVal).addMBB(MBB: LoopMBB)
10338 .addReg(RegNo: RotatedAltVal).addMBB(MBB: UseAltMBB);
10339 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: NewVal)
10340 .addReg(RegNo: RotatedNewVal).addReg(RegNo: NegBitShift).addImm(Val: 0);
10341 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: Dest)
10342 .addReg(RegNo: OldVal)
10343 .addReg(RegNo: NewVal)
10344 .add(MO: Base)
10345 .addImm(Val: Disp);
10346 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10347 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10348 MBB->addSuccessor(Succ: LoopMBB);
10349 MBB->addSuccessor(Succ: DoneMBB);
10350
10351 MI.eraseFromParent();
10352 return DoneMBB;
10353}
10354
10355// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_CMP_SWAPW
10356// instruction MI.
10357MachineBasicBlock *
10358SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
10359 MachineBasicBlock *MBB) const {
10360 MachineFunction &MF = *MBB->getParent();
10361 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10362 MachineRegisterInfo &MRI = MF.getRegInfo();
10363
10364 // Extract the operands. Base can be a register or a frame index.
10365 Register Dest = MI.getOperand(i: 0).getReg();
10366 MachineOperand Base = earlyUseOperand(Op: MI.getOperand(i: 1));
10367 int64_t Disp = MI.getOperand(i: 2).getImm();
10368 Register CmpVal = MI.getOperand(i: 3).getReg();
10369 Register OrigSwapVal = MI.getOperand(i: 4).getReg();
10370 Register BitShift = MI.getOperand(i: 5).getReg();
10371 Register NegBitShift = MI.getOperand(i: 6).getReg();
10372 int64_t BitSize = MI.getOperand(i: 7).getImm();
10373 DebugLoc DL = MI.getDebugLoc();
10374
10375 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
10376
10377 // Get the right opcodes for the displacement and zero-extension.
10378 unsigned LOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::L, Offset: Disp);
10379 unsigned CSOpcode = TII->getOpcodeForOffset(Opcode: SystemZ::CS, Offset: Disp);
10380 unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
10381 assert(LOpcode && CSOpcode && "Displacement out of range");
10382
10383 // Create virtual registers for temporary results.
10384 Register OrigOldVal = MRI.createVirtualRegister(RegClass: RC);
10385 Register OldVal = MRI.createVirtualRegister(RegClass: RC);
10386 Register SwapVal = MRI.createVirtualRegister(RegClass: RC);
10387 Register StoreVal = MRI.createVirtualRegister(RegClass: RC);
10388 Register OldValRot = MRI.createVirtualRegister(RegClass: RC);
10389 Register RetryOldVal = MRI.createVirtualRegister(RegClass: RC);
10390 Register RetrySwapVal = MRI.createVirtualRegister(RegClass: RC);
10391
10392 // Insert 2 basic blocks for the loop.
10393 MachineBasicBlock *StartMBB = MBB;
10394 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10395 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10396 MachineBasicBlock *SetMBB = SystemZ::emitBlockAfter(MBB: LoopMBB);
10397
10398 // StartMBB:
10399 // ...
10400 // %OrigOldVal = L Disp(%Base)
10401 // # fall through to LoopMBB
10402 MBB = StartMBB;
10403 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: LOpcode), DestReg: OrigOldVal)
10404 .add(MO: Base)
10405 .addImm(Val: Disp)
10406 .addReg(RegNo: 0);
10407 MBB->addSuccessor(Succ: LoopMBB);
10408
10409 // LoopMBB:
10410 // %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
10411 // %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
10412 // %OldValRot = RLL %OldVal, BitSize(%BitShift)
10413 // ^^ The low BitSize bits contain the field
10414 // of interest.
10415 // %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
10416 // ^^ Replace the upper 32-BitSize bits of the
10417 // swap value with those that we loaded and rotated.
10418 // %Dest = LL[CH] %OldValRot
10419 // CR %Dest, %CmpVal
10420 // JNE DoneMBB
10421 // # Fall through to SetMBB
10422 MBB = LoopMBB;
10423 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: OldVal)
10424 .addReg(RegNo: OrigOldVal).addMBB(MBB: StartMBB)
10425 .addReg(RegNo: RetryOldVal).addMBB(MBB: SetMBB);
10426 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: SwapVal)
10427 .addReg(RegNo: OrigSwapVal).addMBB(MBB: StartMBB)
10428 .addReg(RegNo: RetrySwapVal).addMBB(MBB: SetMBB);
10429 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: OldValRot)
10430 .addReg(RegNo: OldVal).addReg(RegNo: BitShift).addImm(Val: BitSize);
10431 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RISBG32), DestReg: RetrySwapVal)
10432 .addReg(RegNo: SwapVal).addReg(RegNo: OldValRot).addImm(Val: 32).addImm(Val: 63 - BitSize).addImm(Val: 0);
10433 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: ZExtOpcode), DestReg: Dest)
10434 .addReg(RegNo: OldValRot);
10435 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CR))
10436 .addReg(RegNo: Dest).addReg(RegNo: CmpVal);
10437 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10438 .addImm(Val: SystemZ::CCMASK_ICMP)
10439 .addImm(Val: SystemZ::CCMASK_CMP_NE).addMBB(MBB: DoneMBB);
10440 MBB->addSuccessor(Succ: DoneMBB);
10441 MBB->addSuccessor(Succ: SetMBB);
10442
10443 // SetMBB:
10444 // %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
10445 // ^^ Rotate the new field to its proper position.
10446 // %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
10447 // JNE LoopMBB
10448 // # fall through to ExitMBB
10449 MBB = SetMBB;
10450 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::RLL), DestReg: StoreVal)
10451 .addReg(RegNo: RetrySwapVal).addReg(RegNo: NegBitShift).addImm(Val: -BitSize);
10452 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: CSOpcode), DestReg: RetryOldVal)
10453 .addReg(RegNo: OldVal)
10454 .addReg(RegNo: StoreVal)
10455 .add(MO: Base)
10456 .addImm(Val: Disp);
10457 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10458 .addImm(Val: SystemZ::CCMASK_CS).addImm(Val: SystemZ::CCMASK_CS_NE).addMBB(MBB: LoopMBB);
10459 MBB->addSuccessor(Succ: LoopMBB);
10460 MBB->addSuccessor(Succ: DoneMBB);
10461
10462 // If the CC def wasn't dead in the ATOMIC_CMP_SWAPW, mark CC as live-in
10463 // to the block after the loop. At this point, CC may have been defined
10464 // either by the CR in LoopMBB or by the CS in SetMBB.
10465 if (!MI.registerDefIsDead(Reg: SystemZ::CC, /*TRI=*/nullptr))
10466 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10467
10468 MI.eraseFromParent();
10469 return DoneMBB;
10470}
10471
10472// Emit a move from two GR64s to a GR128.
10473MachineBasicBlock *
10474SystemZTargetLowering::emitPair128(MachineInstr &MI,
10475 MachineBasicBlock *MBB) const {
10476 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10477 const DebugLoc &DL = MI.getDebugLoc();
10478
10479 Register Dest = MI.getOperand(i: 0).getReg();
10480 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest)
10481 .add(MO: MI.getOperand(i: 1))
10482 .addImm(Val: SystemZ::subreg_h64)
10483 .add(MO: MI.getOperand(i: 2))
10484 .addImm(Val: SystemZ::subreg_l64);
10485 MI.eraseFromParent();
10486 return MBB;
10487}
10488
10489// Emit an extension from a GR64 to a GR128. ClearEven is true
10490// if the high register of the GR128 value must be cleared or false if
10491// it's "don't care".
10492MachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI,
10493 MachineBasicBlock *MBB,
10494 bool ClearEven) const {
10495 MachineFunction &MF = *MBB->getParent();
10496 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10497 MachineRegisterInfo &MRI = MF.getRegInfo();
10498 DebugLoc DL = MI.getDebugLoc();
10499
10500 Register Dest = MI.getOperand(i: 0).getReg();
10501 Register Src = MI.getOperand(i: 1).getReg();
10502 Register In128 = MRI.createVirtualRegister(RegClass: &SystemZ::GR128BitRegClass);
10503
10504 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: In128);
10505 if (ClearEven) {
10506 Register NewIn128 = MRI.createVirtualRegister(RegClass: &SystemZ::GR128BitRegClass);
10507 Register Zero64 = MRI.createVirtualRegister(RegClass: &SystemZ::GR64BitRegClass);
10508
10509 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LLILL), DestReg: Zero64)
10510 .addImm(Val: 0);
10511 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: NewIn128)
10512 .addReg(RegNo: In128).addReg(RegNo: Zero64).addImm(Val: SystemZ::subreg_h64);
10513 In128 = NewIn128;
10514 }
10515 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Dest)
10516 .addReg(RegNo: In128).addReg(RegNo: Src).addImm(Val: SystemZ::subreg_l64);
10517
10518 MI.eraseFromParent();
10519 return MBB;
10520}
10521
10522MachineBasicBlock *
10523SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
10524 MachineBasicBlock *MBB,
10525 unsigned Opcode, bool IsMemset) const {
10526 MachineFunction &MF = *MBB->getParent();
10527 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10528 MachineRegisterInfo &MRI = MF.getRegInfo();
10529 DebugLoc DL = MI.getDebugLoc();
10530
10531 MachineOperand DestBase = earlyUseOperand(Op: MI.getOperand(i: 0));
10532 uint64_t DestDisp = MI.getOperand(i: 1).getImm();
10533 MachineOperand SrcBase = MachineOperand::CreateReg(Reg: 0U, isDef: false);
10534 uint64_t SrcDisp;
10535
10536 // Fold the displacement Disp if it is out of range.
10537 auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
10538 if (!isUInt<12>(x: Disp)) {
10539 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10540 unsigned Opcode = TII->getOpcodeForOffset(Opcode: SystemZ::LA, Offset: Disp);
10541 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode), DestReg: Reg)
10542 .add(MO: Base).addImm(Val: Disp).addReg(RegNo: 0);
10543 Base = MachineOperand::CreateReg(Reg, isDef: false);
10544 Disp = 0;
10545 }
10546 };
10547
10548 if (!IsMemset) {
10549 SrcBase = earlyUseOperand(Op: MI.getOperand(i: 2));
10550 SrcDisp = MI.getOperand(i: 3).getImm();
10551 } else {
10552 SrcBase = DestBase;
10553 SrcDisp = DestDisp++;
10554 foldDisplIfNeeded(DestBase, DestDisp);
10555 }
10556
10557 MachineOperand &LengthMO = MI.getOperand(i: IsMemset ? 2 : 4);
10558 bool IsImmForm = LengthMO.isImm();
10559 bool IsRegForm = !IsImmForm;
10560
10561 // Build and insert one Opcode of Length, with special treatment for memset.
10562 auto insertMemMemOp = [&](MachineBasicBlock *InsMBB,
10563 MachineBasicBlock::iterator InsPos,
10564 MachineOperand DBase, uint64_t DDisp,
10565 MachineOperand SBase, uint64_t SDisp,
10566 unsigned Length) -> void {
10567 assert(Length > 0 && Length <= 256 && "Building memory op with bad length.");
10568 if (IsMemset) {
10569 MachineOperand ByteMO = earlyUseOperand(Op: MI.getOperand(i: 3));
10570 if (ByteMO.isImm())
10571 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::MVI))
10572 .add(MO: SBase).addImm(Val: SDisp).add(MO: ByteMO);
10573 else
10574 BuildMI(BB&: *InsMBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode: SystemZ::STC))
10575 .add(MO: ByteMO).add(MO: SBase).addImm(Val: SDisp).addReg(RegNo: 0);
10576 if (--Length == 0)
10577 return;
10578 }
10579 BuildMI(BB&: *MBB, I: InsPos, MIMD: DL, MCID: TII->get(Opcode))
10580 .add(MO: DBase).addImm(Val: DDisp).addImm(Val: Length)
10581 .add(MO: SBase).addImm(Val: SDisp)
10582 .setMemRefs(MI.memoperands());
10583 };
10584
10585 bool NeedsLoop = false;
10586 uint64_t ImmLength = 0;
10587 Register LenAdjReg = SystemZ::NoRegister;
10588 if (IsImmForm) {
10589 ImmLength = LengthMO.getImm();
10590 ImmLength += IsMemset ? 2 : 1; // Add back the subtracted adjustment.
10591 if (ImmLength == 0) {
10592 MI.eraseFromParent();
10593 return MBB;
10594 }
10595 if (Opcode == SystemZ::CLC) {
10596 if (ImmLength > 3 * 256)
10597 // A two-CLC sequence is a clear win over a loop, not least because
10598 // it needs only one branch. A three-CLC sequence needs the same
10599 // number of branches as a loop (i.e. 2), but is shorter. That
10600 // brings us to lengths greater than 768 bytes. It seems relatively
10601 // likely that a difference will be found within the first 768 bytes,
10602 // so we just optimize for the smallest number of branch
10603 // instructions, in order to avoid polluting the prediction buffer
10604 // too much.
10605 NeedsLoop = true;
10606 } else if (ImmLength > 6 * 256)
10607 // The heuristic we use is to prefer loops for anything that would
10608 // require 7 or more MVCs. With these kinds of sizes there isn't much
10609 // to choose between straight-line code and looping code, since the
10610 // time will be dominated by the MVCs themselves.
10611 NeedsLoop = true;
10612 } else {
10613 NeedsLoop = true;
10614 LenAdjReg = LengthMO.getReg();
10615 }
10616
10617 // When generating more than one CLC, all but the last will need to
10618 // branch to the end when a difference is found.
10619 MachineBasicBlock *EndMBB =
10620 (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
10621 ? SystemZ::splitBlockAfter(MI, MBB)
10622 : nullptr);
10623
10624 if (NeedsLoop) {
10625 Register StartCountReg =
10626 MRI.createVirtualRegister(RegClass: &SystemZ::GR64BitRegClass);
10627 if (IsImmForm) {
10628 TII->loadImmediate(MBB&: *MBB, MBBI: MI, Reg: StartCountReg, Value: ImmLength / 256);
10629 ImmLength &= 255;
10630 } else {
10631 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SRLG), DestReg: StartCountReg)
10632 .addReg(RegNo: LenAdjReg)
10633 .addReg(RegNo: 0)
10634 .addImm(Val: 8);
10635 }
10636
10637 bool HaveSingleBase = DestBase.isIdenticalTo(Other: SrcBase);
10638 auto loadZeroAddress = [&]() -> MachineOperand {
10639 Register Reg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10640 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LGHI), DestReg: Reg).addImm(Val: 0);
10641 return MachineOperand::CreateReg(Reg, isDef: false);
10642 };
10643 if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
10644 DestBase = loadZeroAddress();
10645 if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
10646 SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
10647
10648 MachineBasicBlock *StartMBB = nullptr;
10649 MachineBasicBlock *LoopMBB = nullptr;
10650 MachineBasicBlock *NextMBB = nullptr;
10651 MachineBasicBlock *DoneMBB = nullptr;
10652 MachineBasicBlock *AllDoneMBB = nullptr;
10653
10654 Register StartSrcReg = forceReg(MI, Base&: SrcBase, TII);
10655 Register StartDestReg =
10656 (HaveSingleBase ? StartSrcReg : forceReg(MI, Base&: DestBase, TII));
10657
10658 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
10659 Register ThisSrcReg = MRI.createVirtualRegister(RegClass: RC);
10660 Register ThisDestReg =
10661 (HaveSingleBase ? ThisSrcReg : MRI.createVirtualRegister(RegClass: RC));
10662 Register NextSrcReg = MRI.createVirtualRegister(RegClass: RC);
10663 Register NextDestReg =
10664 (HaveSingleBase ? NextSrcReg : MRI.createVirtualRegister(RegClass: RC));
10665 RC = &SystemZ::GR64BitRegClass;
10666 Register ThisCountReg = MRI.createVirtualRegister(RegClass: RC);
10667 Register NextCountReg = MRI.createVirtualRegister(RegClass: RC);
10668
10669 if (IsRegForm) {
10670 AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10671 StartMBB = SystemZ::emitBlockAfter(MBB);
10672 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10673 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10674 DoneMBB = SystemZ::emitBlockAfter(MBB: NextMBB);
10675
10676 // MBB:
10677 // # Jump to AllDoneMBB if LenAdjReg means 0, or fall thru to StartMBB.
10678 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10679 .addReg(RegNo: LenAdjReg).addImm(Val: IsMemset ? -2 : -1);
10680 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10681 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10682 .addMBB(MBB: AllDoneMBB);
10683 MBB->addSuccessor(Succ: AllDoneMBB);
10684 if (!IsMemset)
10685 MBB->addSuccessor(Succ: StartMBB);
10686 else {
10687 // MemsetOneCheckMBB:
10688 // # Jump to MemsetOneMBB for a memset of length 1, or
10689 // # fall thru to StartMBB.
10690 MachineBasicBlock *MemsetOneCheckMBB = SystemZ::emitBlockAfter(MBB);
10691 MachineBasicBlock *MemsetOneMBB = SystemZ::emitBlockAfter(MBB: &*MF.rbegin());
10692 MBB->addSuccessor(Succ: MemsetOneCheckMBB);
10693 MBB = MemsetOneCheckMBB;
10694 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10695 .addReg(RegNo: LenAdjReg).addImm(Val: -1);
10696 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10697 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10698 .addMBB(MBB: MemsetOneMBB);
10699 MBB->addSuccessor(Succ: MemsetOneMBB, Prob: {10, 100});
10700 MBB->addSuccessor(Succ: StartMBB, Prob: {90, 100});
10701
10702 // MemsetOneMBB:
10703 // # Jump back to AllDoneMBB after a single MVI or STC.
10704 MBB = MemsetOneMBB;
10705 insertMemMemOp(MBB, MBB->end(),
10706 MachineOperand::CreateReg(Reg: StartDestReg, isDef: false), DestDisp,
10707 MachineOperand::CreateReg(Reg: StartSrcReg, isDef: false), SrcDisp,
10708 1);
10709 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: AllDoneMBB);
10710 MBB->addSuccessor(Succ: AllDoneMBB);
10711 }
10712
10713 // StartMBB:
10714 // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB.
10715 MBB = StartMBB;
10716 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10717 .addReg(RegNo: StartCountReg).addImm(Val: 0);
10718 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10719 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
10720 .addMBB(MBB: DoneMBB);
10721 MBB->addSuccessor(Succ: DoneMBB);
10722 MBB->addSuccessor(Succ: LoopMBB);
10723 }
10724 else {
10725 StartMBB = MBB;
10726 DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10727 LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10728 NextMBB = (EndMBB ? SystemZ::emitBlockAfter(MBB: LoopMBB) : LoopMBB);
10729
10730 // StartMBB:
10731 // # fall through to LoopMBB
10732 MBB->addSuccessor(Succ: LoopMBB);
10733
10734 DestBase = MachineOperand::CreateReg(Reg: NextDestReg, isDef: false);
10735 SrcBase = MachineOperand::CreateReg(Reg: NextSrcReg, isDef: false);
10736 if (EndMBB && !ImmLength)
10737 // If the loop handled the whole CLC range, DoneMBB will be empty with
10738 // CC live-through into EndMBB, so add it as live-in.
10739 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10740 }
10741
10742 // LoopMBB:
10743 // %ThisDestReg = phi [ %StartDestReg, StartMBB ],
10744 // [ %NextDestReg, NextMBB ]
10745 // %ThisSrcReg = phi [ %StartSrcReg, StartMBB ],
10746 // [ %NextSrcReg, NextMBB ]
10747 // %ThisCountReg = phi [ %StartCountReg, StartMBB ],
10748 // [ %NextCountReg, NextMBB ]
10749 // ( PFD 2, 768+DestDisp(%ThisDestReg) )
10750 // Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg)
10751 // ( JLH EndMBB )
10752 //
10753 // The prefetch is used only for MVC. The JLH is used only for CLC.
10754 MBB = LoopMBB;
10755 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisDestReg)
10756 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10757 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10758 if (!HaveSingleBase)
10759 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisSrcReg)
10760 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10761 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10762 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: ThisCountReg)
10763 .addReg(RegNo: StartCountReg).addMBB(MBB: StartMBB)
10764 .addReg(RegNo: NextCountReg).addMBB(MBB: NextMBB);
10765 if (Opcode == SystemZ::MVC)
10766 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PFD))
10767 .addImm(Val: SystemZ::PFD_WRITE)
10768 .addReg(RegNo: ThisDestReg).addImm(Val: DestDisp - IsMemset + 768).addReg(RegNo: 0);
10769 insertMemMemOp(MBB, MBB->end(),
10770 MachineOperand::CreateReg(Reg: ThisDestReg, isDef: false), DestDisp,
10771 MachineOperand::CreateReg(Reg: ThisSrcReg, isDef: false), SrcDisp, 256);
10772 if (EndMBB) {
10773 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10774 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10775 .addMBB(MBB: EndMBB);
10776 MBB->addSuccessor(Succ: EndMBB);
10777 MBB->addSuccessor(Succ: NextMBB);
10778 }
10779
10780 // NextMBB:
10781 // %NextDestReg = LA 256(%ThisDestReg)
10782 // %NextSrcReg = LA 256(%ThisSrcReg)
10783 // %NextCountReg = AGHI %ThisCountReg, -1
10784 // CGHI %NextCountReg, 0
10785 // JLH LoopMBB
10786 // # fall through to DoneMBB
10787 //
10788 // The AGHI, CGHI and JLH should be converted to BRCTG by later passes.
10789 MBB = NextMBB;
10790 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextDestReg)
10791 .addReg(RegNo: ThisDestReg).addImm(Val: 256).addReg(RegNo: 0);
10792 if (!HaveSingleBase)
10793 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::LA), DestReg: NextSrcReg)
10794 .addReg(RegNo: ThisSrcReg).addImm(Val: 256).addReg(RegNo: 0);
10795 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::AGHI), DestReg: NextCountReg)
10796 .addReg(RegNo: ThisCountReg).addImm(Val: -1);
10797 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
10798 .addReg(RegNo: NextCountReg).addImm(Val: 0);
10799 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10800 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10801 .addMBB(MBB: LoopMBB);
10802 MBB->addSuccessor(Succ: LoopMBB);
10803 MBB->addSuccessor(Succ: DoneMBB);
10804
10805 MBB = DoneMBB;
10806 if (IsRegForm) {
10807 // DoneMBB:
10808 // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
10809 // # Use EXecute Relative Long for the remainder of the bytes. The target
10810 // instruction of the EXRL will have a length field of 1 since 0 is an
10811 // illegal value. The number of bytes processed becomes (%LenAdjReg &
10812 // 0xff) + 1.
10813 // # Fall through to AllDoneMBB.
10814 Register RemSrcReg = MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10815 Register RemDestReg = HaveSingleBase ? RemSrcReg
10816 : MRI.createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
10817 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemDestReg)
10818 .addReg(RegNo: StartDestReg).addMBB(MBB: StartMBB)
10819 .addReg(RegNo: NextDestReg).addMBB(MBB: NextMBB);
10820 if (!HaveSingleBase)
10821 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: RemSrcReg)
10822 .addReg(RegNo: StartSrcReg).addMBB(MBB: StartMBB)
10823 .addReg(RegNo: NextSrcReg).addMBB(MBB: NextMBB);
10824 if (IsMemset)
10825 insertMemMemOp(MBB, MBB->end(),
10826 MachineOperand::CreateReg(Reg: RemDestReg, isDef: false), DestDisp,
10827 MachineOperand::CreateReg(Reg: RemSrcReg, isDef: false), SrcDisp, 1);
10828 MachineInstrBuilder EXRL_MIB =
10829 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::EXRL_Pseudo))
10830 .addImm(Val: Opcode)
10831 .addReg(RegNo: LenAdjReg)
10832 .addReg(RegNo: RemDestReg).addImm(Val: DestDisp)
10833 .addReg(RegNo: RemSrcReg).addImm(Val: SrcDisp);
10834 MBB->addSuccessor(Succ: AllDoneMBB);
10835 MBB = AllDoneMBB;
10836 if (Opcode != SystemZ::MVC) {
10837 EXRL_MIB.addReg(RegNo: SystemZ::CC, Flags: RegState::ImplicitDefine);
10838 if (EndMBB)
10839 MBB->addLiveIn(PhysReg: SystemZ::CC);
10840 }
10841 }
10842 MF.getProperties().resetNoPHIs();
10843 }
10844
10845 // Handle any remaining bytes with straight-line code.
10846 while (ImmLength > 0) {
10847 uint64_t ThisLength = std::min(a: ImmLength, b: uint64_t(256));
10848 // The previous iteration might have created out-of-range displacements.
10849 // Apply them using LA/LAY if so.
10850 foldDisplIfNeeded(DestBase, DestDisp);
10851 foldDisplIfNeeded(SrcBase, SrcDisp);
10852 insertMemMemOp(MBB, MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
10853 DestDisp += ThisLength;
10854 SrcDisp += ThisLength;
10855 ImmLength -= ThisLength;
10856 // If there's another CLC to go, branch to the end if a difference
10857 // was found.
10858 if (EndMBB && ImmLength > 0) {
10859 MachineBasicBlock *NextMBB = SystemZ::splitBlockBefore(MI, MBB);
10860 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10861 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_NE)
10862 .addMBB(MBB: EndMBB);
10863 MBB->addSuccessor(Succ: EndMBB);
10864 MBB->addSuccessor(Succ: NextMBB);
10865 MBB = NextMBB;
10866 }
10867 }
10868 if (EndMBB) {
10869 MBB->addSuccessor(Succ: EndMBB);
10870 MBB = EndMBB;
10871 MBB->addLiveIn(PhysReg: SystemZ::CC);
10872 }
10873
10874 MI.eraseFromParent();
10875 return MBB;
10876}
10877
10878// Decompose string pseudo-instruction MI into a loop that continually performs
10879// Opcode until CC != 3.
10880MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
10881 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
10882 MachineFunction &MF = *MBB->getParent();
10883 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10884 MachineRegisterInfo &MRI = MF.getRegInfo();
10885 DebugLoc DL = MI.getDebugLoc();
10886
10887 uint64_t End1Reg = MI.getOperand(i: 0).getReg();
10888 uint64_t Start1Reg = MI.getOperand(i: 1).getReg();
10889 uint64_t Start2Reg = MI.getOperand(i: 2).getReg();
10890 uint64_t CharReg = MI.getOperand(i: 3).getReg();
10891
10892 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass;
10893 uint64_t This1Reg = MRI.createVirtualRegister(RegClass: RC);
10894 uint64_t This2Reg = MRI.createVirtualRegister(RegClass: RC);
10895 uint64_t End2Reg = MRI.createVirtualRegister(RegClass: RC);
10896
10897 MachineBasicBlock *StartMBB = MBB;
10898 MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
10899 MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
10900
10901 // StartMBB:
10902 // # fall through to LoopMBB
10903 MBB->addSuccessor(Succ: LoopMBB);
10904
10905 // LoopMBB:
10906 // %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ]
10907 // %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ]
10908 // R0L = %CharReg
10909 // %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L
10910 // JO LoopMBB
10911 // # fall through to DoneMBB
10912 //
10913 // The load of R0L can be hoisted by post-RA LICM.
10914 MBB = LoopMBB;
10915
10916 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This1Reg)
10917 .addReg(RegNo: Start1Reg).addMBB(MBB: StartMBB)
10918 .addReg(RegNo: End1Reg).addMBB(MBB: LoopMBB);
10919 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: This2Reg)
10920 .addReg(RegNo: Start2Reg).addMBB(MBB: StartMBB)
10921 .addReg(RegNo: End2Reg).addMBB(MBB: LoopMBB);
10922 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: SystemZ::R0L).addReg(RegNo: CharReg);
10923 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode))
10924 .addReg(RegNo: End1Reg, Flags: RegState::Define).addReg(RegNo: End2Reg, Flags: RegState::Define)
10925 .addReg(RegNo: This1Reg).addReg(RegNo: This2Reg);
10926 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
10927 .addImm(Val: SystemZ::CCMASK_ANY).addImm(Val: SystemZ::CCMASK_3).addMBB(MBB: LoopMBB);
10928 MBB->addSuccessor(Succ: LoopMBB);
10929 MBB->addSuccessor(Succ: DoneMBB);
10930
10931 DoneMBB->addLiveIn(PhysReg: SystemZ::CC);
10932
10933 MI.eraseFromParent();
10934 return DoneMBB;
10935}
10936
10937// Update TBEGIN instruction with final opcode and register clobbers.
10938MachineBasicBlock *SystemZTargetLowering::emitTransactionBegin(
10939 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode,
10940 bool NoFloat) const {
10941 MachineFunction &MF = *MBB->getParent();
10942 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
10943 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10944
10945 // Update opcode.
10946 MI.setDesc(TII->get(Opcode));
10947
10948 // We cannot handle a TBEGIN that clobbers the stack or frame pointer.
10949 // Make sure to add the corresponding GRSM bits if they are missing.
10950 uint64_t Control = MI.getOperand(i: 2).getImm();
10951 static const unsigned GPRControlBit[16] = {
10952 0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000,
10953 0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100
10954 };
10955 Control |= GPRControlBit[15];
10956 if (TFI->hasFP(MF))
10957 Control |= GPRControlBit[11];
10958 MI.getOperand(i: 2).setImm(Control);
10959
10960 // Add GPR clobbers.
10961 for (int I = 0; I < 16; I++) {
10962 if ((Control & GPRControlBit[I]) == 0) {
10963 unsigned Reg = SystemZMC::GR64Regs[I];
10964 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
10965 }
10966 }
10967
10968 // Add FPR/VR clobbers.
10969 if (!NoFloat && (Control & 4) != 0) {
10970 if (Subtarget.hasVector()) {
10971 for (unsigned Reg : SystemZMC::VR128Regs) {
10972 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
10973 }
10974 } else {
10975 for (unsigned Reg : SystemZMC::FP64Regs) {
10976 MI.addOperand(Op: MachineOperand::CreateReg(Reg, isDef: true, isImp: true));
10977 }
10978 }
10979 }
10980
10981 return MBB;
10982}
10983
10984MachineBasicBlock *SystemZTargetLowering::emitLoadAndTestCmp0(
10985 MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
10986 MachineFunction &MF = *MBB->getParent();
10987 MachineRegisterInfo *MRI = &MF.getRegInfo();
10988 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
10989 DebugLoc DL = MI.getDebugLoc();
10990
10991 Register SrcReg = MI.getOperand(i: 0).getReg();
10992
10993 // Create new virtual register of the same class as source.
10994 const TargetRegisterClass *RC = MRI->getRegClass(Reg: SrcReg);
10995 Register DstReg = MRI->createVirtualRegister(RegClass: RC);
10996
10997 // Replace pseudo with a normal load-and-test that models the def as
10998 // well.
10999 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode), DestReg: DstReg)
11000 .addReg(RegNo: SrcReg)
11001 .setMIFlags(MI.getFlags());
11002 MI.eraseFromParent();
11003
11004 return MBB;
11005}
11006
11007MachineBasicBlock *SystemZTargetLowering::emitProbedAlloca(
11008 MachineInstr &MI, MachineBasicBlock *MBB) const {
11009 MachineFunction &MF = *MBB->getParent();
11010 MachineRegisterInfo *MRI = &MF.getRegInfo();
11011 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11012 DebugLoc DL = MI.getDebugLoc();
11013 const unsigned ProbeSize = getStackProbeSize(MF);
11014 Register DstReg = MI.getOperand(i: 0).getReg();
11015 Register SizeReg = MI.getOperand(i: 2).getReg();
11016
11017 MachineBasicBlock *StartMBB = MBB;
11018 MachineBasicBlock *DoneMBB = SystemZ::splitBlockAfter(MI, MBB);
11019 MachineBasicBlock *LoopTestMBB = SystemZ::emitBlockAfter(MBB: StartMBB);
11020 MachineBasicBlock *LoopBodyMBB = SystemZ::emitBlockAfter(MBB: LoopTestMBB);
11021 MachineBasicBlock *TailTestMBB = SystemZ::emitBlockAfter(MBB: LoopBodyMBB);
11022 MachineBasicBlock *TailMBB = SystemZ::emitBlockAfter(MBB: TailTestMBB);
11023
11024 MachineMemOperand *VolLdMMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
11025 F: MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, Size: 8, BaseAlignment: Align(1));
11026
11027 Register PHIReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11028 Register IncReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11029
11030 // LoopTestMBB
11031 // BRC TailTestMBB
11032 // # fallthrough to LoopBodyMBB
11033 StartMBB->addSuccessor(Succ: LoopTestMBB);
11034 MBB = LoopTestMBB;
11035 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::PHI), DestReg: PHIReg)
11036 .addReg(RegNo: SizeReg)
11037 .addMBB(MBB: StartMBB)
11038 .addReg(RegNo: IncReg)
11039 .addMBB(MBB: LoopBodyMBB);
11040 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CLGFI))
11041 .addReg(RegNo: PHIReg)
11042 .addImm(Val: ProbeSize);
11043 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11044 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_LT)
11045 .addMBB(MBB: TailTestMBB);
11046 MBB->addSuccessor(Succ: LoopBodyMBB);
11047 MBB->addSuccessor(Succ: TailTestMBB);
11048
11049 // LoopBodyMBB: Allocate and probe by means of a volatile compare.
11050 // J LoopTestMBB
11051 MBB = LoopBodyMBB;
11052 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: IncReg)
11053 .addReg(RegNo: PHIReg)
11054 .addImm(Val: ProbeSize);
11055 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGFI), DestReg: SystemZ::R15D)
11056 .addReg(RegNo: SystemZ::R15D)
11057 .addImm(Val: ProbeSize);
11058 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG)).addReg(RegNo: SystemZ::R15D)
11059 .addReg(RegNo: SystemZ::R15D).addImm(Val: ProbeSize - 8).addReg(RegNo: 0)
11060 .setMemRefs(VolLdMMO);
11061 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::J)).addMBB(MBB: LoopTestMBB);
11062 MBB->addSuccessor(Succ: LoopTestMBB);
11063
11064 // TailTestMBB
11065 // BRC DoneMBB
11066 // # fallthrough to TailMBB
11067 MBB = TailTestMBB;
11068 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CGHI))
11069 .addReg(RegNo: PHIReg)
11070 .addImm(Val: 0);
11071 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::BRC))
11072 .addImm(Val: SystemZ::CCMASK_ICMP).addImm(Val: SystemZ::CCMASK_CMP_EQ)
11073 .addMBB(MBB: DoneMBB);
11074 MBB->addSuccessor(Succ: TailMBB);
11075 MBB->addSuccessor(Succ: DoneMBB);
11076
11077 // TailMBB
11078 // # fallthrough to DoneMBB
11079 MBB = TailMBB;
11080 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::SLGR), DestReg: SystemZ::R15D)
11081 .addReg(RegNo: SystemZ::R15D)
11082 .addReg(RegNo: PHIReg);
11083 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: SystemZ::CG)).addReg(RegNo: SystemZ::R15D)
11084 .addReg(RegNo: SystemZ::R15D).addImm(Val: -8).addReg(RegNo: PHIReg)
11085 .setMemRefs(VolLdMMO);
11086 MBB->addSuccessor(Succ: DoneMBB);
11087
11088 // DoneMBB
11089 MBB = DoneMBB;
11090 BuildMI(BB&: *MBB, I: MBB->begin(), MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
11091 .addReg(RegNo: SystemZ::R15D);
11092
11093 MI.eraseFromParent();
11094 return DoneMBB;
11095}
11096
11097SDValue SystemZTargetLowering::
11098getBackchainAddress(SDValue SP, SelectionDAG &DAG) const {
11099 MachineFunction &MF = DAG.getMachineFunction();
11100 auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
11101 SDLoc DL(SP);
11102 return DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: SP,
11103 N2: DAG.getIntPtrConstant(Val: TFL->getBackchainOffset(MF), DL));
11104}
11105
11106// Replace a _STACKGUARD_DAG pseudo with a _STACKGUARD pseudo, adding
11107// a dead early-clobber def reg that will be used as a scratch register
11108// when the pseudo is expanded.
11109MachineBasicBlock *SystemZTargetLowering::emitStackGuardPseudo(
11110 MachineInstr &MI, MachineBasicBlock *MBB, unsigned PseudoOp) const {
11111 MachineRegisterInfo *MRI = &MBB->getParent()->getRegInfo();
11112 const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
11113 DebugLoc DL = MI.getDebugLoc();
11114 Register AddrReg = MRI->createVirtualRegister(RegClass: &SystemZ::ADDR64BitRegClass);
11115 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PseudoOp), DestReg: AddrReg)
11116 .addFrameIndex(Idx: MI.getOperand(i: 0).getIndex())
11117 .addImm(Val: MI.getOperand(i: 1).getImm());
11118 MI.eraseFromParent();
11119 return MBB;
11120}
11121
11122MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
11123 MachineInstr &MI, MachineBasicBlock *MBB) const {
11124 switch (MI.getOpcode()) {
11125 case SystemZ::ADJCALLSTACKDOWN:
11126 case SystemZ::ADJCALLSTACKUP:
11127 return emitAdjCallStack(MI, BB: MBB);
11128
11129 case SystemZ::Select32:
11130 case SystemZ::Select64:
11131 case SystemZ::Select128:
11132 case SystemZ::SelectF32:
11133 case SystemZ::SelectF64:
11134 case SystemZ::SelectF128:
11135 case SystemZ::SelectVR32:
11136 case SystemZ::SelectVR64:
11137 case SystemZ::SelectVR128:
11138 return emitSelect(MI, MBB);
11139
11140 case SystemZ::CondStore8Mux:
11141 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: false);
11142 case SystemZ::CondStore8MuxInv:
11143 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STCMux, STOCOpcode: 0, Invert: true);
11144 case SystemZ::CondStore16Mux:
11145 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: false);
11146 case SystemZ::CondStore16MuxInv:
11147 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STHMux, STOCOpcode: 0, Invert: true);
11148 case SystemZ::CondStore32Mux:
11149 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: false);
11150 case SystemZ::CondStore32MuxInv:
11151 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STMux, STOCOpcode: SystemZ::STOCMux, Invert: true);
11152 case SystemZ::CondStore8:
11153 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: false);
11154 case SystemZ::CondStore8Inv:
11155 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STC, STOCOpcode: 0, Invert: true);
11156 case SystemZ::CondStore16:
11157 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: false);
11158 case SystemZ::CondStore16Inv:
11159 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STH, STOCOpcode: 0, Invert: true);
11160 case SystemZ::CondStore32:
11161 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: false);
11162 case SystemZ::CondStore32Inv:
11163 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::ST, STOCOpcode: SystemZ::STOC, Invert: true);
11164 case SystemZ::CondStore64:
11165 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: false);
11166 case SystemZ::CondStore64Inv:
11167 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STG, STOCOpcode: SystemZ::STOCG, Invert: true);
11168 case SystemZ::CondStoreF32:
11169 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: false);
11170 case SystemZ::CondStoreF32Inv:
11171 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STE, STOCOpcode: 0, Invert: true);
11172 case SystemZ::CondStoreF64:
11173 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: false);
11174 case SystemZ::CondStoreF64Inv:
11175 return emitCondStore(MI, MBB, StoreOpcode: SystemZ::STD, STOCOpcode: 0, Invert: true);
11176
11177 case SystemZ::SCmp128Hi:
11178 return emitICmp128Hi(MI, MBB, Unsigned: false);
11179 case SystemZ::UCmp128Hi:
11180 return emitICmp128Hi(MI, MBB, Unsigned: true);
11181
11182 case SystemZ::PAIR128:
11183 return emitPair128(MI, MBB);
11184 case SystemZ::AEXT128:
11185 return emitExt128(MI, MBB, ClearEven: false);
11186 case SystemZ::ZEXT128:
11187 return emitExt128(MI, MBB, ClearEven: true);
11188
11189 case SystemZ::ATOMIC_SWAPW:
11190 return emitAtomicLoadBinary(MI, MBB, BinOpcode: 0);
11191
11192 case SystemZ::ATOMIC_LOADW_AR:
11193 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AR);
11194 case SystemZ::ATOMIC_LOADW_AFI:
11195 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::AFI);
11196
11197 case SystemZ::ATOMIC_LOADW_SR:
11198 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::SR);
11199
11200 case SystemZ::ATOMIC_LOADW_NR:
11201 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR);
11202 case SystemZ::ATOMIC_LOADW_NILH:
11203 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH);
11204
11205 case SystemZ::ATOMIC_LOADW_OR:
11206 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OR);
11207 case SystemZ::ATOMIC_LOADW_OILH:
11208 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::OILH);
11209
11210 case SystemZ::ATOMIC_LOADW_XR:
11211 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XR);
11212 case SystemZ::ATOMIC_LOADW_XILF:
11213 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::XILF);
11214
11215 case SystemZ::ATOMIC_LOADW_NRi:
11216 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NR, Invert: true);
11217 case SystemZ::ATOMIC_LOADW_NILHi:
11218 return emitAtomicLoadBinary(MI, MBB, BinOpcode: SystemZ::NILH, Invert: true);
11219
11220 case SystemZ::ATOMIC_LOADW_MIN:
11221 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
11222 case SystemZ::ATOMIC_LOADW_MAX:
11223 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
11224 case SystemZ::ATOMIC_LOADW_UMIN:
11225 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_LE);
11226 case SystemZ::ATOMIC_LOADW_UMAX:
11227 return emitAtomicLoadMinMax(MI, MBB, CompareOpcode: SystemZ::CLR, KeepOldMask: SystemZ::CCMASK_CMP_GE);
11228
11229 case SystemZ::ATOMIC_CMP_SWAPW:
11230 return emitAtomicCmpSwapW(MI, MBB);
11231 case SystemZ::MVCImm:
11232 case SystemZ::MVCReg:
11233 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC);
11234 case SystemZ::NCImm:
11235 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::NC);
11236 case SystemZ::OCImm:
11237 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::OC);
11238 case SystemZ::XCImm:
11239 case SystemZ::XCReg:
11240 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::XC);
11241 case SystemZ::CLCImm:
11242 case SystemZ::CLCReg:
11243 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::CLC);
11244 case SystemZ::MemsetImmImm:
11245 case SystemZ::MemsetImmReg:
11246 case SystemZ::MemsetRegImm:
11247 case SystemZ::MemsetRegReg:
11248 return emitMemMemWrapper(MI, MBB, Opcode: SystemZ::MVC, IsMemset: true/*IsMemset*/);
11249 case SystemZ::CLSTLoop:
11250 return emitStringWrapper(MI, MBB, Opcode: SystemZ::CLST);
11251 case SystemZ::MVSTLoop:
11252 return emitStringWrapper(MI, MBB, Opcode: SystemZ::MVST);
11253 case SystemZ::SRSTLoop:
11254 return emitStringWrapper(MI, MBB, Opcode: SystemZ::SRST);
11255 case SystemZ::TBEGIN:
11256 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: false);
11257 case SystemZ::TBEGIN_nofloat:
11258 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGIN, NoFloat: true);
11259 case SystemZ::TBEGINC:
11260 return emitTransactionBegin(MI, MBB, Opcode: SystemZ::TBEGINC, NoFloat: true);
11261 case SystemZ::LTEBRCompare_Pseudo:
11262 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTEBR);
11263 case SystemZ::LTDBRCompare_Pseudo:
11264 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTDBR);
11265 case SystemZ::LTXBRCompare_Pseudo:
11266 return emitLoadAndTestCmp0(MI, MBB, Opcode: SystemZ::LTXBR);
11267
11268 case SystemZ::PROBED_ALLOCA:
11269 return emitProbedAlloca(MI, MBB);
11270 case SystemZ::EH_SjLj_SetJmp:
11271 return emitEHSjLjSetJmp(MI, MBB);
11272 case SystemZ::EH_SjLj_LongJmp:
11273 return emitEHSjLjLongJmp(MI, MBB);
11274
11275 case TargetOpcode::STACKMAP:
11276 case TargetOpcode::PATCHPOINT:
11277 return emitPatchPoint(MI, MBB);
11278
11279 case SystemZ::MOV_STACKGUARD_DAG:
11280 return emitStackGuardPseudo(MI, MBB, PseudoOp: SystemZ::MOV_STACKGUARD);
11281
11282 case SystemZ::CMP_STACKGUARD_DAG:
11283 return emitStackGuardPseudo(MI, MBB, PseudoOp: SystemZ::CMP_STACKGUARD);
11284
11285 default:
11286 llvm_unreachable("Unexpected instr type to insert");
11287 }
11288}
11289
11290// This is only used by the isel schedulers, and is needed only to prevent
11291// compiler from crashing when list-ilp is used.
11292const TargetRegisterClass *
11293SystemZTargetLowering::getRepRegClassFor(MVT VT) const {
11294 if (VT == MVT::Untyped)
11295 return &SystemZ::ADDR128BitRegClass;
11296 return TargetLowering::getRepRegClassFor(VT);
11297}
11298
11299SDValue SystemZTargetLowering::lowerGET_ROUNDING(SDValue Op,
11300 SelectionDAG &DAG) const {
11301 SDLoc dl(Op);
11302 /*
11303 The rounding method is in FPC Byte 3 bits 6-7, and has the following
11304 settings:
11305 00 Round to nearest
11306 01 Round to 0
11307 10 Round to +inf
11308 11 Round to -inf
11309
11310 FLT_ROUNDS, on the other hand, expects the following:
11311 -1 Undefined
11312 0 Round to 0
11313 1 Round to nearest
11314 2 Round to +inf
11315 3 Round to -inf
11316 */
11317
11318 // Save FPC to register.
11319 SDValue Chain = Op.getOperand(i: 0);
11320 SDValue EFPC(
11321 DAG.getMachineNode(Opcode: SystemZ::EFPC, dl, ResultTys: {MVT::i32, MVT::Other}, Ops: Chain), 0);
11322 Chain = EFPC.getValue(R: 1);
11323
11324 // Transform as necessary
11325 SDValue CWD1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: EFPC,
11326 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
11327 // RetVal = (CWD1 ^ (CWD1 >> 1)) ^ 1
11328 SDValue CWD2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD1,
11329 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: CWD1,
11330 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32)));
11331
11332 SDValue RetVal = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD2,
11333 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11334 RetVal = DAG.getZExtOrTrunc(Op: RetVal, DL: dl, VT: Op.getValueType());
11335
11336 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl);
11337}
11338
11339SDValue SystemZTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
11340 SelectionDAG &DAG) const {
11341 EVT VT = Op.getValueType();
11342 Op = Op.getOperand(i: 0);
11343 EVT OpVT = Op.getValueType();
11344
11345 assert(OpVT.isVector() && "Operand type for VECREDUCE_ADD is not a vector.");
11346
11347 SDLoc DL(Op);
11348
11349 // load a 0 vector for the third operand of VSUM.
11350 SDValue Zero = DAG.getSplatBuildVector(VT: OpVT, DL, Op: DAG.getConstant(Val: 0, DL, VT));
11351
11352 // execute VSUM.
11353 switch (OpVT.getScalarSizeInBits()) {
11354 case 8:
11355 case 16:
11356 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::v4i32, N1: Op, N2: Zero);
11357 [[fallthrough]];
11358 case 32:
11359 case 64:
11360 Op = DAG.getNode(Opcode: SystemZISD::VSUM, DL, VT: MVT::i128, N1: Op,
11361 N2: DAG.getBitcast(VT: Op.getValueType(), V: Zero));
11362 break;
11363 case 128:
11364 break; // VSUM over v1i128 should not happen and would be a noop
11365 default:
11366 llvm_unreachable("Unexpected scalar size.");
11367 }
11368 // Cast to original vector type, retrieve last element.
11369 return DAG.getNode(
11370 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: DAG.getBitcast(VT: OpVT, V: Op),
11371 N2: DAG.getConstant(Val: OpVT.getVectorNumElements() - 1, DL, VT: MVT::i32));
11372}
11373
11374static void printFunctionArgExts(const Function *F, raw_fd_ostream &OS) {
11375 FunctionType *FT = F->getFunctionType();
11376 const AttributeList &Attrs = F->getAttributes();
11377 if (Attrs.hasRetAttrs())
11378 OS << Attrs.getAsString(Index: AttributeList::ReturnIndex) << " ";
11379 OS << *F->getReturnType() << " @" << F->getName() << "(";
11380 for (unsigned I = 0, E = FT->getNumParams(); I != E; ++I) {
11381 if (I)
11382 OS << ", ";
11383 OS << *FT->getParamType(i: I);
11384 AttributeSet ArgAttrs = Attrs.getParamAttrs(ArgNo: I);
11385 for (auto A : {Attribute::SExt, Attribute::ZExt, Attribute::NoExt})
11386 if (ArgAttrs.hasAttribute(Kind: A))
11387 OS << " " << Attribute::getNameFromAttrKind(AttrKind: A);
11388 }
11389 OS << ")\n";
11390}
11391
11392bool SystemZTargetLowering::isInternal(const Function *Fn) const {
11393 std::map<const Function *, bool>::iterator Itr = IsInternalCache.find(x: Fn);
11394 if (Itr == IsInternalCache.end())
11395 Itr = IsInternalCache
11396 .insert(x: std::pair<const Function *, bool>(
11397 Fn, (Fn->hasLocalLinkage() && !Fn->hasAddressTaken())))
11398 .first;
11399 return Itr->second;
11400}
11401
11402void SystemZTargetLowering::
11403verifyNarrowIntegerArgs_Call(const SmallVectorImpl<ISD::OutputArg> &Outs,
11404 const Function *F, SDValue Callee) const {
11405 // Temporarily only do the check when explicitly requested, until it can be
11406 // enabled by default.
11407 if (!EnableIntArgExtCheck)
11408 return;
11409
11410 bool IsInternal = false;
11411 const Function *CalleeFn = nullptr;
11412 if (auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
11413 if ((CalleeFn = dyn_cast<Function>(Val: G->getGlobal())))
11414 IsInternal = isInternal(Fn: CalleeFn);
11415 if (!IsInternal && !verifyNarrowIntegerArgs(Outs)) {
11416 errs() << "ERROR: Missing extension attribute of passed "
11417 << "value in call to function:\n" << "Callee: ";
11418 if (CalleeFn != nullptr)
11419 printFunctionArgExts(F: CalleeFn, OS&: errs());
11420 else
11421 errs() << "-\n";
11422 errs() << "Caller: ";
11423 printFunctionArgExts(F, OS&: errs());
11424 llvm_unreachable("");
11425 }
11426}
11427
11428void SystemZTargetLowering::
11429verifyNarrowIntegerArgs_Ret(const SmallVectorImpl<ISD::OutputArg> &Outs,
11430 const Function *F) const {
11431 // Temporarily only do the check when explicitly requested, until it can be
11432 // enabled by default.
11433 if (!EnableIntArgExtCheck)
11434 return;
11435
11436 if (!isInternal(Fn: F) && !verifyNarrowIntegerArgs(Outs)) {
11437 errs() << "ERROR: Missing extension attribute of returned "
11438 << "value from function:\n";
11439 printFunctionArgExts(F, OS&: errs());
11440 llvm_unreachable("");
11441 }
11442}
11443
11444// Verify that narrow integer arguments are extended as required by the ABI.
11445// Return false if an error is found.
11446bool SystemZTargetLowering::verifyNarrowIntegerArgs(
11447 const SmallVectorImpl<ISD::OutputArg> &Outs) const {
11448 if (!Subtarget.isTargetELF())
11449 return true;
11450
11451 if (EnableIntArgExtCheck.getNumOccurrences()) {
11452 if (!EnableIntArgExtCheck)
11453 return true;
11454 } else if (!getTargetMachine().Options.VerifyArgABICompliance)
11455 return true;
11456
11457 for (unsigned i = 0; i < Outs.size(); ++i) {
11458 MVT VT = Outs[i].VT;
11459 ISD::ArgFlagsTy Flags = Outs[i].Flags;
11460 if (VT.isInteger()) {
11461 assert((VT == MVT::i32 || VT.getSizeInBits() >= 64) &&
11462 "Unexpected integer argument VT.");
11463 if (VT == MVT::i32 &&
11464 !Flags.isSExt() && !Flags.isZExt() && !Flags.isNoExt())
11465 return false;
11466 }
11467 }
11468
11469 return true;
11470}
11471
11472void SystemZTargetLowering::insertSSPDeclarations(
11473 Module &M, const LibcallLoweringInfo &Libcalls) const {
11474 StringRef GuardMode = M.getStackProtectorGuard();
11475
11476 // In the TLS case, no symbol needs to be inserted.
11477 if (GuardMode == "tls" || GuardMode.empty())
11478 return;
11479
11480 // Otherwise (in the global case), insert the appropriate global variable.
11481 TargetLowering::insertSSPDeclarations(M, Libcalls);
11482}
11483